diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 27c8e97dd2..e74f6a080b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -38,6 +38,9 @@ jobs: - name: 📥 Download deps run: pnpm install --frozen-lockfile --filter trigger.dev... + - name: 📀 Generate Prisma Client + run: pnpm run generate + - name: 🔧 Build v3 cli monorepo dependencies run: pnpm run build --filter trigger.dev^... diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index 55bf20ad51..d5b53a5bee 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -54,6 +54,9 @@ jobs: - name: 📥 Download deps run: pnpm install --frozen-lockfile + - name: 📀 Generate Prisma Client + run: pnpm run generate + - name: 🏗️ Build run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev" diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 5d19bc4e92..99019a0e1b 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -24,6 +24,13 @@ jobs: node-version: 20.11.1 cache: "pnpm" + # ..to avoid rate limits when pulling images + - name: 🐳 Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: 📥 Download deps run: pnpm install --frozen-lockfile diff --git a/.npmrc b/.npmrc index 8dbd39f189..fac274c900 100644 --- a/.npmrc +++ b/.npmrc @@ -1,3 +1,4 @@ link-workspace-packages=false public-hoist-pattern[]=*prisma* -prefer-workspace-packages=true \ No newline at end of file +prefer-workspace-packages=true +update-notifier=false \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index f0736642d3..309b27aac2 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -116,6 +116,14 @@ "command": "pnpm exec trigger dev", "cwd": "${workspaceFolder}/references/hello-world", "sourceMaps": true + }, + { + "type": "node-terminal", + "request": "launch", + "name": "Debug RunEngine tests", + "command": "pnpm run test --filter @internal/run-engine", + "cwd": "${workspaceFolder}", + "sourceMaps": true } ] } diff --git a/apps/coordinator/package.json b/apps/coordinator/package.json index c860adb1c8..3b4240bd37 100644 --- a/apps/coordinator/package.json +++ b/apps/coordinator/package.json @@ -23,10 +23,8 @@ "tinyexec": "^0.3.0" }, "devDependencies": { - "@types/node": "^18", "dotenv": "^16.4.2", "esbuild": "^0.19.11", - "tsx": "^4.7.0", - "typescript": "^5.3.3" + "tsx": "^4.7.0" } } \ No newline at end of file diff --git a/apps/coordinator/src/checkpointer.ts b/apps/coordinator/src/checkpointer.ts index bf82a6702c..f6468e5c2e 100644 --- a/apps/coordinator/src/checkpointer.ts +++ b/apps/coordinator/src/checkpointer.ts @@ -193,7 +193,7 @@ export class Checkpointer { const start = performance.now(); this.#logger.log(`checkpointAndPush() start`, { start, opts }); - let interval: NodeJS.Timer | undefined; + let interval: NodeJS.Timeout | undefined; if (opts.shouldHeartbeat) { interval = setInterval(() => { diff --git a/apps/docker-provider/package.json b/apps/docker-provider/package.json index 56d8f89b7e..f3e4015ef0 100644 --- a/apps/docker-provider/package.json +++ b/apps/docker-provider/package.json @@ -20,10 +20,8 @@ "execa": "^8.0.1" }, "devDependencies": { - "@types/node": "^18.19.8", "dotenv": "^16.4.2", "esbuild": "^0.19.11", - "tsx": "^4.7.0", - "typescript": "^5.3.3" + "tsx": "^4.7.0" } } \ No newline at end of file diff --git a/apps/kubernetes-provider/package.json b/apps/kubernetes-provider/package.json index 3b62f65449..6cb26e2c70 100644 --- a/apps/kubernetes-provider/package.json +++ b/apps/kubernetes-provider/package.json @@ -23,7 +23,6 @@ "devDependencies": { "dotenv": "^16.4.2", "esbuild": "^0.19.11", - "tsx": "^4.7.0", - "typescript": "^5.3.3" + "tsx": "^4.7.0" } } \ No newline at end of file diff --git a/apps/proxy/package.json b/apps/proxy/package.json index d72311dcf8..80646e60a0 100644 --- a/apps/proxy/package.json +++ b/apps/proxy/package.json @@ -9,7 +9,6 @@ }, "devDependencies": { "@cloudflare/workers-types": "^4.20240512.0", - "typescript": "^5.0.4", "wrangler": "^3.57.1" }, "dependencies": { diff --git a/apps/webapp/app/consts.ts b/apps/webapp/app/consts.ts index 8b27c3c955..e349bc086b 100644 --- a/apps/webapp/app/consts.ts +++ b/apps/webapp/app/consts.ts @@ -1,6 +1,5 @@ export const LIVE_ENVIRONMENT = "live"; export const DEV_ENVIRONMENT = "development"; -export const CURRENT_DEPLOYMENT_LABEL = "current"; export const MAX_LIVE_PROJECTS = 1; export const DEFAULT_MAX_CONCURRENT_RUNS = 10; export const MAX_CONCURRENT_RUNS_LIMIT = 20; diff --git a/apps/webapp/app/entry.server.tsx b/apps/webapp/app/entry.server.tsx index 4a4330c9f2..a937c24ba7 100644 --- a/apps/webapp/app/entry.server.tsx +++ b/apps/webapp/app/entry.server.tsx @@ -205,6 +205,8 @@ process.on("uncaughtException", (error, origin) => { const sqsEventConsumer = singleton("sqsEventConsumer", getSharedSqsEventConsumer); +singleton("RunEngineEventBusHandlers", registerRunEngineEventBusHandlers); + export { apiRateLimiter } from "./services/apiRateLimit.server"; export { socketIo } from "./v3/handleSocketIo.server"; export { wss } from "./v3/handleWebsockets.server"; @@ -214,6 +216,7 @@ import { eventLoopMonitor } from "./eventLoopMonitor.server"; import { env } from "./env.server"; import { logger } from "./services/logger.server"; import { Prisma } from "./db.server"; +import { registerRunEngineEventBusHandlers } from "./v3/runEngineHandlers.server"; if (env.EVENT_LOOP_MONITOR_ENABLED === "1") { eventLoopMonitor.enable(); diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 166b1531d5..b12fb114b0 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -91,6 +91,33 @@ const EnvironmentSchema = z.object({ REDIS_PASSWORD: z.string().optional(), REDIS_TLS_DISABLED: z.string().optional(), + // Valkey options (used in Run Engine 2.0+) + VALKEY_HOST: z + .string() + .nullish() + .default(process.env.REDIS_HOST ?? null), + VALKEY_READER_HOST: z + .string() + .nullish() + .default(process.env.REDIS_READER_HOST ?? null), + VALKEY_READER_PORT: z.coerce + .number() + .nullish() + .default(process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : null), + VALKEY_PORT: z.coerce + .number() + .nullish() + .default(process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : null), + VALKEY_USERNAME: z + .string() + .nullish() + .default(process.env.REDIS_USERNAME ?? null), + VALKEY_PASSWORD: z + .string() + .nullish() + .default(process.env.REDIS_PASSWORD ?? null), + VALKEY_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"), + DEFAULT_ENV_EXECUTION_CONCURRENCY_LIMIT: z.coerce.number().int().default(10), DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT: z.coerce.number().int().default(10), DEFAULT_DEV_ENV_EXECUTION_ATTEMPTS: z.coerce.number().int().positive().default(1), @@ -162,6 +189,7 @@ const EnvironmentSchema = z.object({ SHARED_QUEUE_CONSUMER_POOL_SIZE: z.coerce.number().int().default(10), SHARED_QUEUE_CONSUMER_INTERVAL_MS: z.coerce.number().int().default(100), SHARED_QUEUE_CONSUMER_NEXT_TICK_INTERVAL_MS: z.coerce.number().int().default(100), + MANAGED_WORKER_SECRET: z.string().default("managed-secret"), // Development OTEL environment variables DEV_OTEL_EXPORTER_OTLP_ENDPOINT: z.string().optional(), @@ -260,6 +288,11 @@ const EnvironmentSchema = z.object({ MAX_BATCH_V2_TRIGGER_ITEMS: z.coerce.number().int().default(500), REALTIME_STREAM_VERSION: z.enum(["v1", "v2"]).default("v1"), + + // Run Engine 2.0 + RUN_ENGINE_WORKER_COUNT: z.coerce.number().int().default(4), + RUN_ENGINE_TASKS_PER_WORKER: z.coerce.number().int().default(10), + RUN_ENGINE_WORKER_POLL_INTERVAL: z.coerce.number().int().default(100), }); export type Environment = z.infer; diff --git a/apps/webapp/app/presenters/v3/DeploymentListPresenter.server.ts b/apps/webapp/app/presenters/v3/DeploymentListPresenter.server.ts index c08454e4a6..d7d0add22a 100644 --- a/apps/webapp/app/presenters/v3/DeploymentListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/DeploymentListPresenter.server.ts @@ -1,4 +1,4 @@ -import { WorkerDeploymentStatus } from "@trigger.dev/database"; +import { WorkerDeploymentStatus, WorkerInstanceGroupType } from "@trigger.dev/database"; import { sqlDatabaseSchema, PrismaClient, prisma } from "~/db.server"; import { Organization } from "~/models/organization.server"; import { Project } from "~/models/project.server"; @@ -95,29 +95,31 @@ export class DeploymentListPresenter { userName: string | null; userDisplayName: string | null; userAvatarUrl: string | null; + type: WorkerInstanceGroupType; }[] >` - SELECT - wd."id", - wd."shortCode", - wd."version", - (SELECT COUNT(*) FROM ${sqlDatabaseSchema}."BackgroundWorkerTask" WHERE "BackgroundWorkerTask"."workerId" = wd."workerId") AS "tasksCount", - wd."environmentId", - wd."status", - u."id" AS "userId", - u."name" AS "userName", - u."displayName" AS "userDisplayName", - u."avatarUrl" AS "userAvatarUrl", + SELECT + wd."id", + wd."shortCode", + wd."version", + (SELECT COUNT(*) FROM ${sqlDatabaseSchema}."BackgroundWorkerTask" WHERE "BackgroundWorkerTask"."workerId" = wd."workerId") AS "tasksCount", + wd."environmentId", + wd."status", + u."id" AS "userId", + u."name" AS "userName", + u."displayName" AS "userDisplayName", + u."avatarUrl" AS "userAvatarUrl", wd."builtAt", - wd."deployedAt" -FROM + wd."deployedAt", + wd."type" +FROM ${sqlDatabaseSchema}."WorkerDeployment" as wd -INNER JOIN - ${sqlDatabaseSchema}."User" as u ON wd."triggeredById" = u."id" -WHERE +INNER JOIN + ${sqlDatabaseSchema}."User" as u ON wd."triggeredById" = u."id" +WHERE wd."projectId" = ${project.id} -ORDER BY - string_to_array(wd."version", '.')::int[] DESC +ORDER BY + string_to_array(wd."version", '.')::int[] DESC LIMIT ${pageSize} OFFSET ${pageSize * (page - 1)};`; return { @@ -146,6 +148,7 @@ LIMIT ${pageSize} OFFSET ${pageSize * (page - 1)};`; isCurrent: label?.label === "current", isDeployed: deployment.status === "DEPLOYED", isLatest: page === 1 && index === 0, + type: deployment.type, environment: { id: environment.id, type: environment.type, diff --git a/apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts b/apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts index 04b12394ef..d9a4c3cd7d 100644 --- a/apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts @@ -68,6 +68,7 @@ export class DeploymentPresenter { imageReference: true, externalBuildData: true, projectId: true, + type: true, environment: { select: { id: true, @@ -154,6 +155,7 @@ export class DeploymentPresenter { organizationId: project.organizationId, errorData: DeploymentPresenter.prepareErrorData(deployment.errorData), isBuilt: !!deployment.builtAt, + type: deployment.type, }, }; } diff --git a/apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts b/apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts index 5df64c9ae3..c340903076 100644 --- a/apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts @@ -39,7 +39,7 @@ export class RunStreamPresenter { traceId: run.traceId, }); - let pinger: NodeJS.Timer | undefined = undefined; + let pinger: NodeJS.Timeout | undefined = undefined; const { unsubscribe, eventEmitter } = await eventRepository.subscribeToTrace(run.traceId); diff --git a/apps/webapp/app/presenters/v3/SpanPresenter.server.ts b/apps/webapp/app/presenters/v3/SpanPresenter.server.ts index d8516ca6ac..c8a9b50529 100644 --- a/apps/webapp/app/presenters/v3/SpanPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/SpanPresenter.server.ts @@ -9,7 +9,7 @@ import { eventRepository } from "~/v3/eventRepository.server"; import { machinePresetFromName } from "~/v3/machinePresets.server"; import { FINAL_ATTEMPT_STATUSES, isFailedRunStatus, isFinalRunStatus } from "~/v3/taskStatus"; import { BasePresenter } from "./basePresenter.server"; -import { getMaxDuration } from "~/v3/utils/maxDuration"; +import { getMaxDuration } from "@trigger.dev/core/v3/apps"; type Result = Awaited>; export type Span = NonNullable["span"]>; @@ -83,6 +83,12 @@ export class SpanPresenter extends BasePresenter { sdkVersion: true, }, }, + engine: true, + masterQueue: true, + secondaryMasterQueue: true, + error: true, + output: true, + outputType: true, //status + duration status: true, startedAt: true, @@ -183,13 +189,33 @@ export class SpanPresenter extends BasePresenter { }) : null; + const finishedData = + run.engine === "V2" + ? run + : isFinished + ? await this._replica.taskRunAttempt.findFirst({ + select: { + output: true, + outputType: true, + error: true, + }, + where: { + status: { in: FINAL_ATTEMPT_STATUSES }, + taskRunId: run.id, + }, + orderBy: { + createdAt: "desc", + }, + }) + : null; + const output = - finishedAttempt === null + finishedData === null ? undefined - : finishedAttempt.outputType === "application/store" - ? `/resources/packets/${run.runtimeEnvironment.id}/${finishedAttempt.output}` - : typeof finishedAttempt.output !== "undefined" && finishedAttempt.output !== null - ? await prettyPrintPacket(finishedAttempt.output, finishedAttempt.outputType ?? undefined) + : finishedData.outputType === "application/store" + ? `/resources/packets/${run.runtimeEnvironment.id}/${finishedData.output}` + : typeof finishedData.output !== "undefined" && finishedData.output !== null + ? await prettyPrintPacket(finishedData.output, finishedData.outputType ?? undefined) : undefined; const payload = @@ -200,14 +226,14 @@ export class SpanPresenter extends BasePresenter { : undefined; let error: TaskRunError | undefined = undefined; - if (finishedAttempt?.error) { - const result = TaskRunError.safeParse(finishedAttempt.error); + if (finishedData?.error) { + const result = TaskRunError.safeParse(finishedData.error); if (result.success) { error = result.data; } else { error = { type: "CUSTOM_ERROR", - raw: JSON.stringify(finishedAttempt.error), + raw: JSON.stringify(finishedData.error), }; } } @@ -320,6 +346,9 @@ export class SpanPresenter extends BasePresenter { metadata, maxDurationInSeconds: getMaxDuration(run.maxDurationInSeconds), batch: run.batch ? { friendlyId: run.batch.friendlyId } : undefined, + engine: run.engine, + masterQueue: run.masterQueue, + secondaryMasterQueue: run.secondaryMasterQueue, }; } diff --git a/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts b/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts index cb1aa77b04..c4260bf312 100644 --- a/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts @@ -19,8 +19,8 @@ import { import { logger } from "~/services/logger.server"; import { BasePresenter } from "./basePresenter.server"; import { TaskRunStatus } from "~/database-types"; -import { CURRENT_DEPLOYMENT_LABEL } from "~/consts"; import { concurrencyTracker } from "~/v3/services/taskRunConcurrencyTracker.server"; +import { CURRENT_DEPLOYMENT_LABEL } from "@trigger.dev/core/v3/apps"; export type Task = { slug: string; diff --git a/apps/webapp/app/presenters/v3/TasksStreamPresenter.server.ts b/apps/webapp/app/presenters/v3/TasksStreamPresenter.server.ts index c7318c3400..b01587d77f 100644 --- a/apps/webapp/app/presenters/v3/TasksStreamPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/TasksStreamPresenter.server.ts @@ -58,7 +58,7 @@ export class TasksStreamPresenter { projectSlug, }); - let pinger: NodeJS.Timer | undefined = undefined; + let pinger: NodeJS.Timeout | undefined = undefined; const subscriber = await projectPubSub.subscribe(`project:${project.id}:*`); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.deployments.$deploymentParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.deployments.$deploymentParam/route.tsx index 7678f53633..f309c5869f 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.deployments.$deploymentParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.deployments.$deploymentParam/route.tsx @@ -30,6 +30,7 @@ import { DeploymentPresenter } from "~/presenters/v3/DeploymentPresenter.server" import { requireUserId } from "~/services/session.server"; import { cn } from "~/utils/cn"; import { v3DeploymentParams, v3DeploymentsPath } from "~/utils/pathBuilder"; +import { capitalizeWord } from "~/utils/string"; export const loader = async ({ request, params }: LoaderFunctionArgs) => { const userId = await requireUserId(request); @@ -155,6 +156,10 @@ export default function Page() { CLI Version {deployment.cliVersion ? deployment.cliVersion : "–"} + + Worker type + {capitalizeWord(deployment.type)} + Started at diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.deployments/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.deployments/route.tsx index 411698cdb6..682890d8d8 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.deployments/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.deployments/route.tsx @@ -6,6 +6,7 @@ import { } from "@heroicons/react/20/solid"; import { Outlet, useLocation, useParams } from "@remix-run/react"; import { LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { WorkerInstanceGroupType } from "@trigger.dev/database"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; import { UserAvatar } from "~/components/UserProfilePhoto"; @@ -321,7 +322,11 @@ function DeploymentActionsCell({ const location = useLocation(); const project = useProject(); - const canRollback = !deployment.isCurrent && deployment.isDeployed; + const canRollback = + deployment.type === WorkerInstanceGroupType.MANAGED && + !deployment.isCurrent && + deployment.isDeployed; + const canRetryIndexing = deployment.isLatest && deploymentIndexingIsRetryable(deployment); if (!canRollback && !canRetryIndexing) { diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx index 6b975a1a14..af8371031f 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx @@ -65,7 +65,7 @@ import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; import { useReplaceSearchParams } from "~/hooks/useReplaceSearchParams"; import { Shortcut, useShortcutKeys } from "~/hooks/useShortcutKeys"; -import { useUser } from "~/hooks/useUser"; +import { useHasAdminAccess, useUser } from "~/hooks/useUser"; import { Run, RunPresenter } from "~/presenters/v3/RunPresenter.server"; import { requireUserId } from "~/services/session.server"; import { cn } from "~/utils/cn"; @@ -462,8 +462,10 @@ function TasksTreeView({ maximumLiveReloadingSetting, rootRun, }: TasksTreeViewProps) { + const isAdmin = useHasAdminAccess(); const [filterText, setFilterText] = useState(""); const [errorsOnly, setErrorsOnly] = useState(false); + const [showDebug, setShowDebug] = useState(false); const [showDurations, setShowDurations] = useState(true); const [scale, setScale] = useState(0); const parentRef = useRef(null); @@ -483,7 +485,7 @@ function TasksTreeView({ scrollToNode, virtualizer, } = useTree({ - tree: events, + tree: showDebug ? events : events.filter((event) => !event.data.isDebug), selectedId, // collapsedIds, onSelectedIdChanged, @@ -508,6 +510,14 @@ function TasksTreeView({
+ {isAdmin && ( + setShowDebug(e.valueOf())} + /> + )} { ); } - logger.error("Failed to start a test run", { error: e }); + logger.error("Failed to start a test run", { error: e instanceof Error ? e.message : e }); return redirectBackWithErrorMessage( request, diff --git a/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.ts b/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.ts index dddbc007be..5ee92606ec 100644 --- a/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.ts +++ b/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.ts @@ -3,6 +3,7 @@ import { z } from "zod"; import { prisma } from "~/db.server"; import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; import { marqs } from "~/v3/marqs/index.server"; +import { updateEnvConcurrencyLimits } from "~/v3/runQueue.server"; const ParamsSchema = z.object({ environmentId: z.string(), @@ -60,7 +61,7 @@ export async function action({ request, params }: ActionFunctionArgs) { }, }); - await marqs?.updateEnvConcurrencyLimits(environment); + await updateEnvConcurrencyLimits(environment); return json({ success: true }); } diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts index 51d292eb05..d6491bcc45 100644 --- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts @@ -3,6 +3,7 @@ import { z } from "zod"; import { prisma } from "~/db.server"; import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; import { marqs } from "~/v3/marqs/index.server"; +import { updateEnvConcurrencyLimits } from "~/v3/runQueue.server"; const ParamsSchema = z.object({ organizationId: z.string(), @@ -97,7 +98,7 @@ export async function action({ request, params }: ActionFunctionArgs) { }, }); - await marqs?.updateEnvConcurrencyLimits({ ...modifiedEnvironment, organization }); + await updateEnvConcurrencyLimits({ ...modifiedEnvironment, organization }); } return json({ success: true }); diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.environments.staging.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.environments.staging.ts index c4088257af..8483058f32 100644 --- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.environments.staging.ts +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.environments.staging.ts @@ -4,6 +4,7 @@ import { prisma } from "~/db.server"; import { createEnvironment } from "~/models/organization.server"; import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; import { marqs } from "~/v3/marqs/index.server"; +import { updateEnvConcurrencyLimits } from "~/v3/runQueue.server"; const ParamsSchema = z.object({ organizationId: z.string(), @@ -58,10 +59,10 @@ export async function action({ request, params }: ActionFunctionArgs) { if (!stagingEnvironment) { const staging = await createEnvironment(organization, project, "STAGING"); - await marqs?.updateEnvConcurrencyLimits({ ...staging, organization, project }); + await updateEnvConcurrencyLimits({ ...staging, organization, project }); created++; } else { - await marqs?.updateEnvConcurrencyLimits({ ...stagingEnvironment, organization, project }); + await updateEnvConcurrencyLimits({ ...stagingEnvironment, organization, project }); } } diff --git a/apps/webapp/app/routes/admin.api.v1.workers.ts b/apps/webapp/app/routes/admin.api.v1.workers.ts new file mode 100644 index 0000000000..185c9cc4d0 --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.workers.ts @@ -0,0 +1,65 @@ +import { ActionFunctionArgs, json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; +import { WorkerGroupService } from "~/v3/services/worker/workerGroupService.server"; + +const RequestBodySchema = z.object({ + name: z.string().optional(), + description: z.string().optional(), + projectId: z.string().optional(), + makeDefault: z.boolean().optional(), +}); + +export async function action({ request }: ActionFunctionArgs) { + // Next authenticate the request + const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request); + + if (!authenticationResult) { + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const user = await prisma.user.findUnique({ + where: { + id: authenticationResult.userId, + }, + }); + + if (!user) { + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + if (!user.admin) { + return json({ error: "You must be an admin to perform this action" }, { status: 403 }); + } + + try { + const rawBody = await request.json(); + const { name, description, projectId, makeDefault } = RequestBodySchema.parse(rawBody ?? {}); + + const service = new WorkerGroupService(); + const { workerGroup, token } = await service.createWorkerGroup({ + name, + description, + }); + + if (makeDefault && projectId) { + await prisma.project.update({ + where: { + id: projectId, + }, + data: { + defaultWorkerGroupId: workerGroup.id, + engine: "V2", + }, + }); + } + + return json({ + token, + workerGroup, + }); + } catch (error) { + return json({ error: error instanceof Error ? error.message : error }, { status: 400 }); + } +} diff --git a/apps/webapp/app/routes/api.v1.deployments.latest.ts b/apps/webapp/app/routes/api.v1.deployments.latest.ts new file mode 100644 index 0000000000..6f31f58fcc --- /dev/null +++ b/apps/webapp/app/routes/api.v1.deployments.latest.ts @@ -0,0 +1,41 @@ +import { LoaderFunctionArgs, json } from "@remix-run/server-runtime"; +import { WorkerInstanceGroupType } from "@trigger.dev/database"; +import { prisma } from "~/db.server"; +import { authenticateApiRequest } from "~/services/apiAuth.server"; +import { logger } from "~/services/logger.server"; + +export async function loader({ request }: LoaderFunctionArgs) { + // Next authenticate the request + const authenticationResult = await authenticateApiRequest(request); + + if (!authenticationResult) { + logger.info("Invalid or missing api key", { url: request.url }); + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const authenticatedEnv = authenticationResult.environment; + + const deployment = await prisma.workerDeployment.findFirst({ + where: { + type: WorkerInstanceGroupType.UNMANAGED, + environmentId: authenticatedEnv.id, + }, + orderBy: { + createdAt: "desc", + }, + }); + + if (!deployment) { + return json({ error: "Deployment not found" }, { status: 404 }); + } + + return json({ + id: deployment.friendlyId, + status: deployment.status, + contentHash: deployment.contentHash, + shortCode: deployment.shortCode, + version: deployment.version, + imageReference: deployment.imageReference, + errorData: deployment.errorData, + }); +} diff --git a/apps/webapp/app/routes/api.v1.deployments.ts b/apps/webapp/app/routes/api.v1.deployments.ts index 2f4b9bdd54..c3dcfb13d0 100644 --- a/apps/webapp/app/routes/api.v1.deployments.ts +++ b/apps/webapp/app/routes/api.v1.deployments.ts @@ -6,6 +6,7 @@ import { import { env } from "~/env.server"; import { authenticateApiRequest } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; +import { ServiceValidationError } from "~/v3/services/baseService.server"; import { InitializeDeploymentService } from "~/v3/services/initializeDeployment.server"; export async function action({ request, params }: ActionFunctionArgs) { @@ -33,18 +34,30 @@ export async function action({ request, params }: ActionFunctionArgs) { const service = new InitializeDeploymentService(); - const { deployment, imageTag } = await service.call(authenticatedEnv, body.data); - - const responseBody: InitializeDeploymentResponseBody = { - id: deployment.friendlyId, - contentHash: deployment.contentHash, - shortCode: deployment.shortCode, - version: deployment.version, - externalBuildData: - deployment.externalBuildData as InitializeDeploymentResponseBody["externalBuildData"], - imageTag, - registryHost: body.data.registryHost ?? env.DEPLOY_REGISTRY_HOST, - }; - - return json(responseBody, { status: 200 }); + try { + const { deployment, imageTag } = await service.call(authenticatedEnv, body.data); + + const responseBody: InitializeDeploymentResponseBody = { + id: deployment.friendlyId, + contentHash: deployment.contentHash, + shortCode: deployment.shortCode, + version: deployment.version, + externalBuildData: + deployment.externalBuildData as InitializeDeploymentResponseBody["externalBuildData"], + imageTag, + registryHost: body.data.registryHost ?? env.DEPLOY_REGISTRY_HOST, + }; + + return json(responseBody, { status: 200 }); + } catch (error) { + if (error instanceof ServiceValidationError) { + return json({ error: error.message }, { status: 400 }); + } else if (error instanceof Error) { + logger.error("Error initializing deployment", { error: error.message }); + return json({ error: `Internal server error: ${error.message}` }, { status: 500 }); + } else { + logger.error("Error initializing deployment", { error: String(error) }); + return json({ error: "Internal server error" }, { status: 500 }); + } + } } diff --git a/apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts b/apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts index e6e3398e69..96eec3ba31 100644 --- a/apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts +++ b/apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts @@ -1,5 +1,9 @@ import { json } from "@remix-run/server-runtime"; -import { generateJWT as internal_generateJWT, TriggerTaskRequestBody } from "@trigger.dev/core/v3"; +import { + generateJWT as internal_generateJWT, + RunEngineVersionSchema, + TriggerTaskRequestBody, +} from "@trigger.dev/core/v3"; import { TaskRun } from "@trigger.dev/database"; import { z } from "zod"; import { env } from "~/env.server"; @@ -21,6 +25,7 @@ export const HeadersSchema = z.object({ "x-trigger-span-parent-as-link": z.coerce.number().nullish(), "x-trigger-worker": z.string().nullish(), "x-trigger-client": z.string().nullish(), + "x-trigger-engine-version": RunEngineVersionSchema.nullish(), traceparent: z.string().optional(), tracestate: z.string().optional(), }); @@ -49,6 +54,7 @@ const { action, loader } = createActionApiRoute( tracestate, "x-trigger-worker": isFromWorker, "x-trigger-client": triggerClient, + "x-trigger-engine-version": engineVersion, } = headers; const service = new TriggerTaskService(); @@ -74,14 +80,20 @@ const { action, loader } = createActionApiRoute( const idempotencyKeyExpiresAt = resolveIdempotencyKeyTTL(idempotencyKeyTTL); - const run = await service.call(params.taskId, authentication.environment, body, { - idempotencyKey: idempotencyKey ?? undefined, - idempotencyKeyExpiresAt: idempotencyKeyExpiresAt, - triggerVersion: triggerVersion ?? undefined, - traceContext, - spanParentAsLink: spanParentAsLink === 1, - oneTimeUseToken, - }); + const run = await service.call( + params.taskId, + authentication.environment, + body, + { + idempotencyKey: idempotencyKey ?? undefined, + idempotencyKeyExpiresAt: idempotencyKeyExpiresAt, + triggerVersion: triggerVersion ?? undefined, + traceContext, + spanParentAsLink: spanParentAsLink === 1, + oneTimeUseToken, + }, + engineVersion ?? undefined + ); if (!run) { return json({ error: "Task not found" }, { status: 404 }); diff --git a/apps/webapp/app/routes/api.v1.tasks.batch.ts b/apps/webapp/app/routes/api.v1.tasks.batch.ts index 5b1b89d2d9..591d04f0ce 100644 --- a/apps/webapp/app/routes/api.v1.tasks.batch.ts +++ b/apps/webapp/app/routes/api.v1.tasks.batch.ts @@ -1,23 +1,23 @@ import { json } from "@remix-run/server-runtime"; import { - BatchTriggerTaskResponse, BatchTriggerTaskV2RequestBody, BatchTriggerTaskV2Response, generateJWT, } from "@trigger.dev/core/v3"; import { env } from "~/env.server"; +import { AuthenticatedEnvironment, getOneTimeUseToken } from "~/services/apiAuth.server"; +import { logger } from "~/services/logger.server"; import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; -import { HeadersSchema } from "./api.v1.tasks.$taskId.trigger"; import { resolveIdempotencyKeyTTL } from "~/utils/idempotencyKeys.server"; +import { determineEngineVersion } from "~/v3/engineVersion.server"; +import { ServiceValidationError } from "~/v3/services/baseService.server"; import { BatchProcessingStrategy, BatchTriggerV2Service, } from "~/v3/services/batchTriggerV2.server"; -import { ServiceValidationError } from "~/v3/services/baseService.server"; +import { BatchTriggerV3Service } from "~/v3/services/batchTriggerV3.server"; import { OutOfEntitlementError } from "~/v3/services/triggerTask.server"; -import { AuthenticatedEnvironment, getOneTimeUseToken } from "~/services/apiAuth.server"; -import { logger } from "~/services/logger.server"; -import { z } from "zod"; +import { HeadersSchema } from "./api.v1.tasks.$taskId.trigger"; const { action, loader } = createActionApiRoute( { @@ -58,6 +58,7 @@ const { action, loader } = createActionApiRoute( "x-trigger-span-parent-as-link": spanParentAsLink, "x-trigger-worker": isFromWorker, "x-trigger-client": triggerClient, + "x-trigger-engine-version": engineVersion, "batch-processing-strategy": batchProcessingStrategy, traceparent, tracestate, @@ -87,7 +88,15 @@ const { action, loader } = createActionApiRoute( resolveIdempotencyKeyTTL(idempotencyKeyTTL) ?? new Date(Date.now() + 24 * 60 * 60 * 1000 * 30); - const service = new BatchTriggerV2Service(batchProcessingStrategy ?? undefined); + const version = await determineEngineVersion({ + environment: authentication.environment, + version: engineVersion ?? undefined, + }); + + const service = + version === "V1" + ? new BatchTriggerV2Service(batchProcessingStrategy ?? undefined) + : new BatchTriggerV3Service(batchProcessingStrategy ?? undefined); try { const batch = await service.call(authentication.environment, body, { diff --git a/apps/webapp/app/routes/api.v1.worker-actions.connect.ts b/apps/webapp/app/routes/api.v1.worker-actions.connect.ts new file mode 100644 index 0000000000..024526a147 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.worker-actions.connect.ts @@ -0,0 +1,19 @@ +import { json, TypedResponse } from "@remix-run/server-runtime"; +import { WorkerApiConnectRequestBody, WorkerApiConnectResponseBody } from "@trigger.dev/worker"; +import { createActionWorkerApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + +export const action = createActionWorkerApiRoute( + { + body: WorkerApiConnectRequestBody, + }, + async ({ authenticatedWorker, body }): Promise> => { + await authenticatedWorker.connect(body.metadata); + return json({ + ok: true, + workerGroup: { + type: authenticatedWorker.type, + name: authenticatedWorker.name, + }, + }); + } +); diff --git a/apps/webapp/app/routes/api.v1.worker-actions.deployments.$deploymentFriendlyId.dequeue.ts b/apps/webapp/app/routes/api.v1.worker-actions.deployments.$deploymentFriendlyId.dequeue.ts new file mode 100644 index 0000000000..fa56f895b2 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.worker-actions.deployments.$deploymentFriendlyId.dequeue.ts @@ -0,0 +1,58 @@ +import { json, TypedResponse } from "@remix-run/server-runtime"; +import { CURRENT_DEPLOYMENT_LABEL } from "@trigger.dev/core/v3/apps"; +import { WorkerApiDequeueResponseBody } from "@trigger.dev/worker"; +import { z } from "zod"; +import { $replica, prisma } from "~/db.server"; +import { createLoaderWorkerApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + +export const loader = createLoaderWorkerApiRoute( + { + params: z.object({ + deploymentFriendlyId: z.string(), + }), + }, + async ({ authenticatedWorker, params }): Promise> => { + const deployment = await $replica.workerDeployment.findUnique({ + where: { + friendlyId: params.deploymentFriendlyId, + }, + include: { + worker: true, + }, + }); + + if (!deployment) { + throw new Error("Deployment not found"); + } + + if (!deployment.worker) { + throw new Error("Worker not found"); + } + + const dequeuedMessages = (await isCurrentDeployment(deployment.id, deployment.environmentId)) + ? await authenticatedWorker.dequeueFromEnvironment( + deployment.worker.id, + deployment.environmentId + ) + : await authenticatedWorker.dequeueFromVersion(deployment.worker.id); + + return json(dequeuedMessages); + } +); + +async function isCurrentDeployment(deploymentId: string, environmentId: string): Promise { + const promotion = await prisma.workerDeploymentPromotion.findUnique({ + where: { + environmentId_label: { + environmentId, + label: CURRENT_DEPLOYMENT_LABEL, + }, + }, + }); + + if (!promotion) { + return false; + } + + return promotion.deploymentId === deploymentId; +} diff --git a/apps/webapp/app/routes/api.v1.worker-actions.dequeue.ts b/apps/webapp/app/routes/api.v1.worker-actions.dequeue.ts new file mode 100644 index 0000000000..fd19968fb1 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.worker-actions.dequeue.ts @@ -0,0 +1,10 @@ +import { json, TypedResponse } from "@remix-run/server-runtime"; +import { WorkerApiDequeueResponseBody } from "@trigger.dev/worker"; +import { createLoaderWorkerApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + +export const loader = createLoaderWorkerApiRoute( + {}, + async ({ authenticatedWorker }): Promise> => { + return json(await authenticatedWorker.dequeue()); + } +); diff --git a/apps/webapp/app/routes/api.v1.worker-actions.heartbeat.ts b/apps/webapp/app/routes/api.v1.worker-actions.heartbeat.ts new file mode 100644 index 0000000000..babe12d5ea --- /dev/null +++ b/apps/webapp/app/routes/api.v1.worker-actions.heartbeat.ts @@ -0,0 +1,13 @@ +import { json, TypedResponse } from "@remix-run/server-runtime"; +import { WorkerApiHeartbeatResponseBody, WorkerApiHeartbeatRequestBody } from "@trigger.dev/worker"; +import { createActionWorkerApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + +export const action = createActionWorkerApiRoute( + { + body: WorkerApiHeartbeatRequestBody, + }, + async ({ authenticatedWorker }): Promise> => { + await authenticatedWorker.heartbeatWorkerInstance(); + return json({ ok: true }); + } +); diff --git a/apps/webapp/app/routes/api.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.complete.ts b/apps/webapp/app/routes/api.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.complete.ts new file mode 100644 index 0000000000..4e33f04fec --- /dev/null +++ b/apps/webapp/app/routes/api.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.complete.ts @@ -0,0 +1,33 @@ +import { json, TypedResponse } from "@remix-run/server-runtime"; +import { + WorkerApiRunAttemptCompleteRequestBody, + WorkerApiRunAttemptCompleteResponseBody, +} from "@trigger.dev/worker"; +import { z } from "zod"; +import { createActionWorkerApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + +export const action = createActionWorkerApiRoute( + { + body: WorkerApiRunAttemptCompleteRequestBody, + params: z.object({ + runFriendlyId: z.string(), + snapshotFriendlyId: z.string(), + }), + }, + async ({ + authenticatedWorker, + body, + params, + }): Promise> => { + const { completion } = body; + const { runFriendlyId, snapshotFriendlyId } = params; + + const completeResult = await authenticatedWorker.completeRunAttempt({ + runFriendlyId, + snapshotFriendlyId, + completion, + }); + + return json({ result: completeResult }); + } +); diff --git a/apps/webapp/app/routes/api.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts b/apps/webapp/app/routes/api.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts new file mode 100644 index 0000000000..d8137e9b90 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts @@ -0,0 +1,32 @@ +import { json, TypedResponse } from "@remix-run/server-runtime"; +import { + WorkerApiRunAttemptStartRequestBody, + WorkerApiRunAttemptStartResponseBody, +} from "@trigger.dev/worker"; +import { z } from "zod"; +import { createActionWorkerApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + +export const action = createActionWorkerApiRoute( + { + body: WorkerApiRunAttemptStartRequestBody, + params: z.object({ + runFriendlyId: z.string(), + snapshotFriendlyId: z.string(), + }), + }, + async ({ + authenticatedWorker, + body, + params, + }): Promise> => { + const { runFriendlyId, snapshotFriendlyId } = params; + + const runExecutionData = await authenticatedWorker.startRunAttempt({ + runFriendlyId, + snapshotFriendlyId, + isWarmStart: body.isWarmStart, + }); + + return json(runExecutionData); + } +); diff --git a/apps/webapp/app/routes/api.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.heartbeat.ts b/apps/webapp/app/routes/api.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.heartbeat.ts new file mode 100644 index 0000000000..96cd8c7e45 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.heartbeat.ts @@ -0,0 +1,26 @@ +import { json, TypedResponse } from "@remix-run/server-runtime"; +import { WorkloadHeartbeatResponseBody } from "@trigger.dev/worker"; +import { z } from "zod"; +import { createActionWorkerApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + +export const action = createActionWorkerApiRoute( + { + params: z.object({ + runFriendlyId: z.string(), + snapshotFriendlyId: z.string(), + }), + }, + async ({ + authenticatedWorker, + params, + }): Promise> => { + const { runFriendlyId, snapshotFriendlyId } = params; + + await authenticatedWorker.heartbeatRun({ + runFriendlyId, + snapshotFriendlyId, + }); + + return json({ ok: true }); + } +); diff --git a/apps/webapp/app/routes/api.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.wait.duration.ts b/apps/webapp/app/routes/api.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.wait.duration.ts new file mode 100644 index 0000000000..f8676f6454 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.wait.duration.ts @@ -0,0 +1,32 @@ +import { json, TypedResponse } from "@remix-run/server-runtime"; +import { + WorkerApiWaitForDurationRequestBody, + WorkerApiWaitForDurationResponseBody, +} from "@trigger.dev/worker"; +import { z } from "zod"; +import { createActionWorkerApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + +export const action = createActionWorkerApiRoute( + { + body: WorkerApiWaitForDurationRequestBody, + params: z.object({ + runFriendlyId: z.string(), + snapshotFriendlyId: z.string(), + }), + }, + async ({ + authenticatedWorker, + body, + params, + }): Promise> => { + const { runFriendlyId, snapshotFriendlyId } = params; + + const waitResult = await authenticatedWorker.waitForDuration({ + runFriendlyId, + snapshotFriendlyId, + date: body.date, + }); + + return json(waitResult); + } +); diff --git a/apps/webapp/app/routes/api.v1.worker-actions.runs.$runFriendlyId.snapshots.latest.ts b/apps/webapp/app/routes/api.v1.worker-actions.runs.$runFriendlyId.snapshots.latest.ts new file mode 100644 index 0000000000..37422bf42a --- /dev/null +++ b/apps/webapp/app/routes/api.v1.worker-actions.runs.$runFriendlyId.snapshots.latest.ts @@ -0,0 +1,28 @@ +import { json, TypedResponse } from "@remix-run/server-runtime"; +import { WorkerApiRunLatestSnapshotResponseBody } from "@trigger.dev/worker"; +import { z } from "zod"; +import { createLoaderWorkerApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + +export const loader = createLoaderWorkerApiRoute( + { + params: z.object({ + runFriendlyId: z.string(), + }), + }, + async ({ + authenticatedWorker, + params, + }): Promise> => { + const { runFriendlyId } = params; + + const executionData = await authenticatedWorker.getLatestSnapshot({ + runFriendlyId, + }); + + if (!executionData) { + throw new Error("Failed to retrieve latest snapshot"); + } + + return json({ execution: executionData }); + } +); diff --git a/apps/webapp/app/routes/api.v1.workers.ts b/apps/webapp/app/routes/api.v1.workers.ts new file mode 100644 index 0000000000..4008d64f1a --- /dev/null +++ b/apps/webapp/app/routes/api.v1.workers.ts @@ -0,0 +1,73 @@ +import { json, TypedResponse } from "@remix-run/server-runtime"; +import { + WorkersCreateRequestBody, + WorkersCreateResponseBody, + WorkersListResponseBody, +} from "@trigger.dev/core/v3"; +import { + createActionApiRoute, + createLoaderApiRoute, +} from "~/services/routeBuilders/apiBuilder.server"; +import { WorkerGroupService } from "~/v3/services/worker/workerGroupService.server"; + +export const loader = createLoaderApiRoute( + { + corsStrategy: "all", + findResource: async () => 1, // This is a dummy function, we don't need to find a resource + }, + async ({ + authentication, + }): Promise> => { + if (authentication.environment.project.engine !== "V2") { + return json({ error: "Not supported for V1 projects" }, { status: 400 }); + } + + const service = new WorkerGroupService(); + const workers = await service.listWorkerGroups({ + projectId: authentication.environment.projectId, + }); + + return json( + workers.map((w) => ({ + type: w.type, + name: w.name, + description: w.description, + isDefault: w.id === authentication.environment.project.defaultWorkerGroupId, + updatedAt: w.updatedAt, + })) + ); + } +); + +export const { action } = createActionApiRoute( + { + corsStrategy: "all", + body: WorkersCreateRequestBody, + }, + async ({ + authentication, + body, + }): Promise> => { + if (authentication.environment.project.engine !== "V2") { + return json({ error: "Not supported" }, { status: 400 }); + } + + const service = new WorkerGroupService(); + const { workerGroup, token } = await service.createWorkerGroup({ + projectId: authentication.environment.projectId, + organizationId: authentication.environment.organizationId, + name: body.name, + description: body.description, + }); + + return json({ + token: { + plaintext: token.plaintext, + }, + workerGroup: { + name: workerGroup.name, + description: workerGroup.description, + }, + }); + } +); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx index 55c9dd38dd..1594e82cfd 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx @@ -49,6 +49,7 @@ import { TaskRunStatusCombo } from "~/components/runs/v3/TaskRunStatus"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; import { useSearchParams } from "~/hooks/useSearchParam"; +import { useHasAdminAccess } from "~/hooks/useUser"; import { redirectWithErrorMessage } from "~/models/message.server"; import { Span, SpanPresenter, SpanRun } from "~/presenters/v3/SpanPresenter.server"; import { logger } from "~/services/logger.server"; @@ -415,6 +416,7 @@ function RunBody({ }) { const organization = useOrganization(); const project = useProject(); + const isAdmin = useHasAdminAccess(); const { value, replace } = useSearchParams(); const tab = value("tab"); @@ -632,6 +634,22 @@ function RunBody({ )} + + Engine version + {run.engine} + + {isAdmin && ( + <> + + Primary master queue + {run.masterQueue} + + + Secondary master queue + {run.secondaryMasterQueue} + + + )} Test run @@ -769,12 +787,13 @@ function RunBody({
+ {run.error && } + {run.payload !== undefined && ( )} - {run.error !== undefined ? ( - - ) : run.output !== undefined ? ( + + {run.error === undefined && run.output !== undefined ? ( ) : null}
diff --git a/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts b/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts index dc35a0cd24..241fef98f7 100644 --- a/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts +++ b/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts @@ -17,6 +17,10 @@ import { PersonalAccessTokenAuthenticationResult, } from "../personalAccessToken.server"; import { safeJsonParse } from "~/utils/json"; +import { + AuthenticatedWorkerInstance, + WorkerGroupTokenService, +} from "~/v3/services/worker/workerGroupTokenService.server"; type ApiKeyRouteBuilderOptions< TParamsSchema extends z.AnyZodObject | undefined = undefined, @@ -640,3 +644,239 @@ async function wrapResponse( }) : response; } + +type WorkerLoaderRouteBuilderOptions< + TParamsSchema extends z.AnyZodObject | undefined = undefined, + TSearchParamsSchema extends z.AnyZodObject | undefined = undefined, + THeadersSchema extends z.AnyZodObject | undefined = undefined +> = { + params?: TParamsSchema; + searchParams?: TSearchParamsSchema; + headers?: THeadersSchema; +}; + +type WorkerLoaderHandlerFunction< + TParamsSchema extends z.AnyZodObject | undefined, + TSearchParamsSchema extends z.AnyZodObject | undefined, + THeadersSchema extends z.AnyZodObject | undefined = undefined +> = (args: { + params: TParamsSchema extends z.AnyZodObject ? z.infer : undefined; + searchParams: TSearchParamsSchema extends z.AnyZodObject + ? z.infer + : undefined; + authenticatedWorker: AuthenticatedWorkerInstance; + request: Request; + headers: THeadersSchema extends z.AnyZodObject ? z.infer : undefined; +}) => Promise; + +export function createLoaderWorkerApiRoute< + TParamsSchema extends z.AnyZodObject | undefined = undefined, + TSearchParamsSchema extends z.AnyZodObject | undefined = undefined, + THeadersSchema extends z.AnyZodObject | undefined = undefined +>( + options: WorkerLoaderRouteBuilderOptions, + handler: WorkerLoaderHandlerFunction +) { + return async function loader({ request, params }: LoaderFunctionArgs) { + const { + params: paramsSchema, + searchParams: searchParamsSchema, + headers: headersSchema, + } = options; + + try { + const service = new WorkerGroupTokenService(); + const authenticationResult = await service.authenticate(request); + + if (!authenticationResult) { + return json({ error: "Invalid or missing worker token" }, { status: 401 }); + } + + let parsedParams: any = undefined; + if (paramsSchema) { + const parsed = paramsSchema.safeParse(params); + if (!parsed.success) { + return json( + { error: "Params Error", details: fromZodError(parsed.error).details }, + { status: 400 } + ); + } + parsedParams = parsed.data; + } + + let parsedSearchParams: any = undefined; + if (searchParamsSchema) { + const searchParams = Object.fromEntries(new URL(request.url).searchParams); + const parsed = searchParamsSchema.safeParse(searchParams); + if (!parsed.success) { + return json( + { error: "Query Error", details: fromZodError(parsed.error).details }, + { status: 400 } + ); + } + parsedSearchParams = parsed.data; + } + + let parsedHeaders: any = undefined; + if (headersSchema) { + const rawHeaders = Object.fromEntries(request.headers); + const headers = headersSchema.safeParse(rawHeaders); + if (!headers.success) { + return json( + { error: "Headers Error", details: fromZodError(headers.error).details }, + { status: 400 } + ); + } + parsedHeaders = headers.data; + } + + const result = await handler({ + params: parsedParams, + searchParams: parsedSearchParams, + authenticatedWorker: authenticationResult, + request, + headers: parsedHeaders, + }); + return result; + } catch (error) { + console.error("Error in API route:", error); + if (error instanceof Response) { + return error; + } + return json({ error: "Internal Server Error" }, { status: 500 }); + } + }; +} + +type WorkerActionRouteBuilderOptions< + TParamsSchema extends z.AnyZodObject | undefined = undefined, + TSearchParamsSchema extends z.AnyZodObject | undefined = undefined, + THeadersSchema extends z.AnyZodObject | undefined = undefined, + TBodySchema extends z.AnyZodObject | undefined = undefined +> = { + params?: TParamsSchema; + searchParams?: TSearchParamsSchema; + headers?: THeadersSchema; + body?: TBodySchema; +}; + +type WorkerActionHandlerFunction< + TParamsSchema extends z.AnyZodObject | undefined, + TSearchParamsSchema extends z.AnyZodObject | undefined, + THeadersSchema extends z.AnyZodObject | undefined = undefined, + TBodySchema extends z.AnyZodObject | undefined = undefined +> = (args: { + params: TParamsSchema extends z.AnyZodObject ? z.infer : undefined; + searchParams: TSearchParamsSchema extends z.AnyZodObject + ? z.infer + : undefined; + authenticatedWorker: AuthenticatedWorkerInstance; + request: Request; + headers: THeadersSchema extends z.AnyZodObject ? z.infer : undefined; + body: TBodySchema extends z.AnyZodObject ? z.infer : undefined; +}) => Promise; + +export function createActionWorkerApiRoute< + TParamsSchema extends z.AnyZodObject | undefined = undefined, + TSearchParamsSchema extends z.AnyZodObject | undefined = undefined, + THeadersSchema extends z.AnyZodObject | undefined = undefined, + TBodySchema extends z.AnyZodObject | undefined = undefined +>( + options: WorkerActionRouteBuilderOptions< + TParamsSchema, + TSearchParamsSchema, + THeadersSchema, + TBodySchema + >, + handler: WorkerActionHandlerFunction< + TParamsSchema, + TSearchParamsSchema, + THeadersSchema, + TBodySchema + > +) { + return async function action({ request, params }: ActionFunctionArgs) { + const { + params: paramsSchema, + searchParams: searchParamsSchema, + body: bodySchema, + headers: headersSchema, + } = options; + + try { + const service = new WorkerGroupTokenService(); + const authenticationResult = await service.authenticate(request); + + if (!authenticationResult) { + return json({ error: "Invalid or missing worker token" }, { status: 401 }); + } + + let parsedParams: any = undefined; + if (paramsSchema) { + const parsed = paramsSchema.safeParse(params); + if (!parsed.success) { + return json( + { error: "Params Error", details: fromZodError(parsed.error).details }, + { status: 400 } + ); + } + parsedParams = parsed.data; + } + + let parsedSearchParams: any = undefined; + if (searchParamsSchema) { + const searchParams = Object.fromEntries(new URL(request.url).searchParams); + const parsed = searchParamsSchema.safeParse(searchParams); + if (!parsed.success) { + return json( + { error: "Query Error", details: fromZodError(parsed.error).details }, + { status: 400 } + ); + } + parsedSearchParams = parsed.data; + } + + let parsedHeaders: any = undefined; + if (headersSchema) { + const rawHeaders = Object.fromEntries(request.headers); + const headers = headersSchema.safeParse(rawHeaders); + if (!headers.success) { + return json( + { error: "Headers Error", details: fromZodError(headers.error).details }, + { status: 400 } + ); + } + parsedHeaders = headers.data; + } + + let parsedBody: any = undefined; + if (bodySchema) { + const body = await request.clone().json(); + const parsed = bodySchema.safeParse(body); + if (!parsed.success) { + return json( + { error: "Body Error", details: fromZodError(parsed.error).details }, + { status: 400 } + ); + } + parsedBody = parsed.data; + } + + const result = await handler({ + params: parsedParams, + searchParams: parsedSearchParams, + authenticatedWorker: authenticationResult, + request, + body: parsedBody, + headers: parsedHeaders, + }); + return result; + } catch (error) { + console.error("Error in API route:", error); + if (error instanceof Response) { + return error; + } + return json({ error: "Internal Server Error" }, { status: 500 }); + } + }; +} diff --git a/apps/webapp/app/services/worker.server.ts b/apps/webapp/app/services/worker.server.ts index c2409cf8c5..3eea668a6e 100644 --- a/apps/webapp/app/services/worker.server.ts +++ b/apps/webapp/app/services/worker.server.ts @@ -56,6 +56,10 @@ import { } from "~/v3/services/cancelDevSessionRuns.server"; import { logger } from "./logger.server"; import { BatchProcessingOptions, BatchTriggerV2Service } from "~/v3/services/batchTriggerV2.server"; +import { + BatchProcessingOptions as BatchProcessingOptionsV3, + BatchTriggerV3Service, +} from "~/v3/services/batchTriggerV3.server"; const workerCatalog = { indexEndpoint: z.object({ @@ -199,6 +203,7 @@ const workerCatalog = { }), "v3.cancelDevSessionRuns": CancelDevSessionRunsServiceOptions, "v3.processBatchTaskRun": BatchProcessingOptions, + "v3.processBatchTaskRunV3": BatchProcessingOptionsV3, }; const executionWorkerCatalog = { @@ -735,6 +740,15 @@ function getWorkerQueue() { handler: async (payload, job) => { const service = new BatchTriggerV2Service(payload.strategy); + await service.processBatchTaskRun(payload); + }, + }, + "v3.processBatchTaskRunV3": { + priority: 0, + maxAttempts: 5, + handler: async (payload, job) => { + const service = new BatchTriggerV3Service(payload.strategy); + await service.processBatchTaskRun(payload); }, }, diff --git a/apps/webapp/app/utils/delays.ts b/apps/webapp/app/utils/delays.ts index 6faa67c677..eaa296e11b 100644 --- a/apps/webapp/app/utils/delays.ts +++ b/apps/webapp/app/utils/delays.ts @@ -1,3 +1,5 @@ +import { parseNaturalLanguageDuration } from "@trigger.dev/core/v3/apps"; + export const calculateDurationInMs = (options: { seconds?: number; minutes?: number; @@ -11,3 +13,30 @@ export const calculateDurationInMs = (options: { (options?.days ?? 0) * 24 * 60 * 60 * 1000 ); }; + +export async function parseDelay(value?: string | Date): Promise { + if (!value) { + return; + } + + if (value instanceof Date) { + return value; + } + + try { + const date = new Date(value); + + // Check if the date is valid + if (isNaN(date.getTime())) { + return parseNaturalLanguageDuration(value); + } + + if (date.getTime() <= Date.now()) { + return; + } + + return date; + } catch (error) { + return parseNaturalLanguageDuration(value); + } +} diff --git a/apps/webapp/app/utils/sse.server.ts b/apps/webapp/app/utils/sse.server.ts index fced1fbaf4..56e7b191af 100644 --- a/apps/webapp/app/utils/sse.server.ts +++ b/apps/webapp/app/utils/sse.server.ts @@ -22,8 +22,8 @@ export function sse({ request, pingInterval = 1000, updateInterval = 348, run }: return new Response("SSE disabled", { status: 200 }); } - let pinger: NodeJS.Timer | undefined = undefined; - let updater: NodeJS.Timer | undefined = undefined; + let pinger: NodeJS.Timeout | undefined = undefined; + let updater: NodeJS.Timeout | undefined = undefined; let timeout: NodeJS.Timeout | undefined = undefined; const abort = () => { diff --git a/apps/webapp/app/utils/string.ts b/apps/webapp/app/utils/string.ts new file mode 100644 index 0000000000..d2dfdbb1d6 --- /dev/null +++ b/apps/webapp/app/utils/string.ts @@ -0,0 +1,3 @@ +export function capitalizeWord(word: string) { + return word.charAt(0).toUpperCase() + word.slice(1).toLowerCase(); +} diff --git a/apps/webapp/app/utils/taskEvent.ts b/apps/webapp/app/utils/taskEvent.ts index 41a1703c25..8b2655559e 100644 --- a/apps/webapp/app/utils/taskEvent.ts +++ b/apps/webapp/app/utils/taskEvent.ts @@ -66,7 +66,6 @@ export function prepareTrace(events: TaskEvent[]): TraceSummary | undefined { id: event.spanId, parentId: event.parentId ?? undefined, runId: event.runId, - idempotencyKey: event.idempotencyKey, data: { message: event.message, style: event.style, @@ -78,8 +77,9 @@ export function prepareTrace(events: TaskEvent[]): TraceSummary | undefined { level: event.level, events: event.events, environmentType: event.environmentType, + isDebug: event.isDebug, }, - }; + } satisfies SpanSummary; spansBySpanId.set(event.spanId, span); diff --git a/apps/webapp/app/v3/authenticatedSocketConnection.server.ts b/apps/webapp/app/v3/authenticatedSocketConnection.server.ts index ce98438784..208deaa2f5 100644 --- a/apps/webapp/app/v3/authenticatedSocketConnection.server.ts +++ b/apps/webapp/app/v3/authenticatedSocketConnection.server.ts @@ -1,4 +1,8 @@ -import { clientWebsocketMessages, serverWebsocketMessages } from "@trigger.dev/core/v3"; +import { + clientWebsocketMessages, + HeartbeatService, + serverWebsocketMessages, +} from "@trigger.dev/core/v3"; import { ZodMessageHandler, ZodMessageSender } from "@trigger.dev/core/v3/zodMessageHandler"; import { Evt } from "evt"; import { randomUUID } from "node:crypto"; @@ -7,7 +11,6 @@ import { WebSocket } from "ws"; import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { DevQueueConsumer } from "./marqs/devQueueConsumer.server"; -import { HeartbeatService } from "./services/heartbeatService.server"; export class AuthenticatedSocketConnection { public id: string; @@ -83,6 +86,7 @@ export class AuthenticatedSocketConnection { ws.ping(); }, + intervalMs: 45_000, }); this._pingService.start(); diff --git a/apps/webapp/app/v3/engineVersion.server.ts b/apps/webapp/app/v3/engineVersion.server.ts new file mode 100644 index 0000000000..26b268fd5c --- /dev/null +++ b/apps/webapp/app/v3/engineVersion.server.ts @@ -0,0 +1,66 @@ +import { RunEngineVersion, RuntimeEnvironmentType } from "@trigger.dev/database"; +import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { findCurrentWorkerDeploymentWithoutTasks } from "./models/workerDeployment.server"; + +export async function determineEngineVersion({ + environment, + version, +}: { + environment: AuthenticatedEnvironment; + version?: RunEngineVersion; +}): Promise { + if (version) { + return version; + } + + // If the project is V1, then none of the background workers are running V2 + if (environment.project.engine === RunEngineVersion.V1) { + return "V1"; + } + + // For now, dev is always V1 + if (environment.type === RuntimeEnvironmentType.DEVELOPMENT) { + return "V1"; + } + + /** + * The project has V2 enabled and this isn't dev + */ + + // Check the current deployment for this environment + const currentDeployment = await findCurrentWorkerDeploymentWithoutTasks(environment.id); + if (currentDeployment?.type === "V1") { + return "V1"; + } + + //todo we need to determine the version using the BackgroundWorker + //- triggerAndWait we can lookup the BackgroundWorker easily, and get the engine. + //- No locked version: lookup the BackgroundWorker via the Deployment/latest dev BW + // const workerWithTasks = workerId + // ? await getWorkerDeploymentFromWorker(prisma, workerId) + // : run.runtimeEnvironment.type === "DEVELOPMENT" + // ? await getMostRecentWorker(prisma, run.runtimeEnvironmentId) + // : await getWorkerFromCurrentlyPromotedDeployment(prisma, run.runtimeEnvironmentId); + + //todo Additional checks + /* + - If the `triggerVersion` is 3.2 or higher AND the project has engine V2, we will use the run engine. + - Add an `engine` column to `Project` in the database. + + Add `engine` to the trigger.config file. It would default to "V1" for now, but you can set it to V2. + + You run `npx trigger.dev@latest deploy` with config v2. + - Create BackgroundWorker with `engine`: `v2`. + - Set the `project` `engine` column to `v2`. + + You run `npx trigger.dev@latest dev` with config v2 + - Create BackgroundWorker with `engine`: `v2`. + - Set the `project` `engine` column to `v2`. + + When triggering + - triggerAndWait we can lookup the BackgroundWorker easily, and get the engine. + - No locked version: lookup the BackgroundWorker via the Deployment/latest dev BW + */ + + return "V2"; +} diff --git a/apps/webapp/app/v3/eventRepository.server.ts b/apps/webapp/app/v3/eventRepository.server.ts index c9c2e8e231..5cf7b5313d 100644 --- a/apps/webapp/app/v3/eventRepository.server.ts +++ b/apps/webapp/app/v3/eventRepository.server.ts @@ -56,6 +56,7 @@ export type TraceAttributes = Partial< | "attemptId" | "isError" | "isCancelled" + | "isDebug" | "runId" | "runIsTest" | "output" @@ -119,6 +120,7 @@ export type QueriedEvent = Prisma.TaskEventGetPayload<{ isError: true; isPartial: true; isCancelled: true; + isDebug: true; level: true; events: true; environmentType: true; @@ -164,6 +166,7 @@ export type SpanSummary = { isError: boolean; isPartial: boolean; isCancelled: boolean; + isDebug: boolean; level: NonNullable; environmentType: CreatableEventEnvironmentType; }; @@ -240,7 +243,7 @@ export class EventRepository { eventId: event.id, }); - await this.insert({ + const completedEvent = { ...omit(event, "id"), isPartial: false, isError: options?.attributes.isError ?? false, @@ -260,7 +263,11 @@ export class EventRepository { : "application/json", payload: event.payload as Attributes, payloadType: event.payloadType, - }); + } satisfies CreatableEvent; + + await this.insert(completedEvent); + + return completedEvent; } async cancelEvent(event: TaskEventRecord, cancelledAt: Date, reason: string) { @@ -397,6 +404,7 @@ export class EventRepository { isError: true, isPartial: true, isCancelled: true, + isDebug: true, level: true, events: true, environmentType: true, @@ -460,6 +468,7 @@ export class EventRepository { isError: event.isError, isPartial: ancestorCancelled ? false : event.isPartial, isCancelled: event.isCancelled === true ? true : event.isPartial && ancestorCancelled, + isDebug: event.isDebug, startTime: getDateFromNanoseconds(event.startTime), level: event.level, events: event.events, @@ -505,6 +514,7 @@ export class EventRepository { isError: true, isPartial: true, isCancelled: true, + isDebug: true, level: true, events: true, environmentType: true, @@ -744,11 +754,13 @@ export class EventRepository { }); } - public async recordEvent(message: string, options: TraceEventOptions) { + public async recordEvent(message: string, options: TraceEventOptions & { duration?: number }) { const propagatedContext = extractContextFromCarrier(options.context ?? {}); const startTime = options.startTime ?? getNowInNanoseconds(); - const duration = options.endTime ? calculateDurationFromStart(startTime, options.endTime) : 100; + const duration = + options.duration ?? + (options.endTime ? calculateDurationFromStart(startTime, options.endTime) : 100); const traceId = propagatedContext?.traceparent?.traceId ?? this.generateTraceId(); const parentId = propagatedContext?.traceparent?.spanId; @@ -772,8 +784,10 @@ export class EventRepository { ...options.attributes.metadata, }; + const isDebug = options.attributes.isDebug; + const style = { - [SemanticInternalAttributes.STYLE_ICON]: "play", + [SemanticInternalAttributes.STYLE_ICON]: isDebug ? "warn" : "play", }; if (!options.attributes.runId) { @@ -788,11 +802,12 @@ export class EventRepository { message: message, serviceName: "api server", serviceNamespace: "trigger.dev", - level: "TRACE", + level: isDebug ? "WARN" : "TRACE", kind: options.kind, status: "OK", startTime, isPartial: false, + isDebug, duration, // convert to nanoseconds environmentId: options.environment.id, environmentType: options.environment.type, diff --git a/apps/webapp/app/v3/featureFlags.server.ts b/apps/webapp/app/v3/featureFlags.server.ts new file mode 100644 index 0000000000..1cc57ed48c --- /dev/null +++ b/apps/webapp/app/v3/featureFlags.server.ts @@ -0,0 +1,54 @@ +import { z } from "zod"; +import { prisma, PrismaClientOrTransaction } from "~/db.server"; + +const FeatureFlagCatalog = { + defaultWorkerInstanceGroupId: z.string(), +}; + +type FeatureFlagKey = keyof typeof FeatureFlagCatalog; + +export type FlagsOptions = { + key: FeatureFlagKey; +}; + +export function makeFlags(_prisma: PrismaClientOrTransaction = prisma) { + return async function flags( + opts: FlagsOptions + ): Promise | undefined> { + const value = await _prisma.featureFlag.findUnique({ + where: { + key: opts.key, + }, + }); + + const parsed = FeatureFlagCatalog[opts.key].safeParse(value?.value); + + if (!parsed.success) { + return; + } + + return parsed.data; + }; +} + +export function makeSetFlags(_prisma: PrismaClientOrTransaction = prisma) { + return async function setFlags( + opts: FlagsOptions & { value: z.infer<(typeof FeatureFlagCatalog)[T]> } + ): Promise { + await _prisma.featureFlag.upsert({ + where: { + key: opts.key, + }, + create: { + key: opts.key, + value: opts.value, + }, + update: { + value: opts.value, + }, + }); + }; +} + +export const flags = makeFlags(); +export const setFlags = makeSetFlags(); diff --git a/apps/webapp/app/v3/handleSocketIo.server.ts b/apps/webapp/app/v3/handleSocketIo.server.ts index e1c97d2e0a..e8aa628a39 100644 --- a/apps/webapp/app/v3/handleSocketIo.server.ts +++ b/apps/webapp/app/v3/handleSocketIo.server.ts @@ -8,7 +8,7 @@ import { SharedQueueToClientMessages, } from "@trigger.dev/core/v3"; import { ZodNamespace } from "@trigger.dev/core/v3/zodNamespace"; -import { Server } from "socket.io"; +import { Namespace, Server, Socket } from "socket.io"; import { env } from "~/env.server"; import { singleton } from "~/utils/singleton"; import { SharedSocketConnection } from "./sharedSocketConnection"; @@ -25,6 +25,8 @@ import { createAdapter } from "@socket.io/redis-adapter"; import { CrashTaskRunService } from "./services/crashTaskRun.server"; import { CreateTaskRunAttemptService } from "./services/createTaskRunAttempt.server"; import { UpdateFatalRunErrorService } from "./services/updateFatalRunError.server"; +import { WorkerGroupTokenService } from "./services/worker/workerGroupTokenService.server"; +import type { WorkerClientToServerEvents, WorkerServerToClientEvents } from "@trigger.dev/worker"; export const socketIo = singleton("socketIo", initalizeIoServer); @@ -38,12 +40,14 @@ function initalizeIoServer() { const coordinatorNamespace = createCoordinatorNamespace(io); const providerNamespace = createProviderNamespace(io); const sharedQueueConsumerNamespace = createSharedQueueConsumerNamespace(io); + const workerNamespace = createWorkerNamespace(io); return { io, coordinatorNamespace, providerNamespace, sharedQueueConsumerNamespace, + workerNamespace, }; } @@ -367,3 +371,152 @@ function createSharedQueueConsumerNamespace(io: Server) { return sharedQueue.namespace; } + +function headersFromHandshake(handshake: Socket["handshake"]) { + const headers = new Headers(); + + for (const [key, value] of Object.entries(handshake.headers)) { + if (typeof value !== "string") continue; + headers.append(key, value); + } + + return headers; +} + +function createWorkerNamespace(io: Server) { + const worker: Namespace = + io.of("/worker"); + + worker.use(async (socket, next) => { + try { + const headers = headersFromHandshake(socket.handshake); + + logger.debug("Worker authentication", { + socketId: socket.id, + headers: Object.fromEntries(headers), + }); + + const request = new Request("https://example.com", { + headers, + }); + + const tokenService = new WorkerGroupTokenService(); + const authenticatedInstance = await tokenService.authenticate(request); + + if (!authenticatedInstance) { + throw new Error("unauthorized"); + } + + next(); + } catch (error) { + logger.error("Worker authentication failed", { + error: error instanceof Error ? error.message : error, + }); + + socket.disconnect(true); + } + }); + + worker.on("connection", async (socket) => { + logger.debug("worker connected", { socketId: socket.id }); + + const rooms = new Set(); + + const interval = setInterval(() => { + logger.debug("Rooms for socket", { + socketId: socket.id, + rooms: Array.from(rooms), + }); + }, 5000); + + socket.on("disconnect", (reason, description) => { + logger.debug("worker disconnected", { + socketId: socket.id, + reason, + description, + }); + clearInterval(interval); + }); + + socket.on("disconnecting", (reason, description) => { + logger.debug("worker disconnecting", { + socketId: socket.id, + reason, + description, + }); + clearInterval(interval); + }); + + socket.on("error", (error) => { + logger.error("worker error", { + socketId: socket.id, + error: JSON.parse(JSON.stringify(error)), + }); + clearInterval(interval); + }); + + socket.on("run:subscribe", async ({ version, runFriendlyIds }) => { + logger.debug("run:subscribe", { version, runFriendlyIds }); + + const settledResult = await Promise.allSettled( + runFriendlyIds.map((friendlyId) => { + const room = roomFromFriendlyRunId(friendlyId); + + logger.debug("Joining room", { room }); + + socket.join(room); + rooms.add(room); + }) + ); + + for (const result of settledResult) { + if (result.status === "rejected") { + logger.error("Error joining room", { + runFriendlyIds, + error: result.reason instanceof Error ? result.reason.message : result.reason, + }); + } + } + + logger.debug("Rooms for socket after subscribe", { + socketId: socket.id, + rooms: Array.from(rooms), + }); + }); + + socket.on("run:unsubscribe", async ({ version, runFriendlyIds }) => { + logger.debug("run:unsubscribe", { version, runFriendlyIds }); + + const settledResult = await Promise.allSettled( + runFriendlyIds.map((friendlyId) => { + const room = roomFromFriendlyRunId(friendlyId); + + logger.debug("Leaving room", { room }); + + socket.leave(room); + rooms.delete(room); + }) + ); + + for (const result of settledResult) { + if (result.status === "rejected") { + logger.error("Error leaving room", { + runFriendlyIds, + error: result.reason instanceof Error ? result.reason.message : result.reason, + }); + } + } + + logger.debug("Rooms for socket after unsubscribe", { + socketId: socket.id, + rooms: Array.from(rooms), + }); + }); + }); + + return worker; +} + +export function roomFromFriendlyRunId(id: string) { + return `room:${id}`; +} diff --git a/apps/webapp/app/v3/machinePresets.server.ts b/apps/webapp/app/v3/machinePresets.server.ts index 120a235c54..612dc16258 100644 --- a/apps/webapp/app/v3/machinePresets.server.ts +++ b/apps/webapp/app/v3/machinePresets.server.ts @@ -41,3 +41,15 @@ function derivePresetNameFromValues(cpu: number, memory: number): MachinePresetN return defaultMachine; } + +export function allMachines(): Record { + return Object.fromEntries( + Object.entries(machines).map(([name, preset]) => [ + name, + { + name: name as MachinePresetName, + ...preset, + }, + ]) + ); +} diff --git a/apps/webapp/app/v3/marqs/devQueueConsumer.server.ts b/apps/webapp/app/v3/marqs/devQueueConsumer.server.ts index d629225a9c..ac9d4d531a 100644 --- a/apps/webapp/app/v3/marqs/devQueueConsumer.server.ts +++ b/apps/webapp/app/v3/marqs/devQueueConsumer.server.ts @@ -24,7 +24,7 @@ import { tracer, } from "../tracer.server"; import { DevSubscriber, devPubSub } from "./devPubSub.server"; -import { getMaxDuration } from "../utils/maxDuration"; +import { getMaxDuration } from "@trigger.dev/core/v3/apps"; const MessageBody = z.discriminatedUnion("type", [ z.object({ diff --git a/apps/webapp/app/v3/marqs/index.server.ts b/apps/webapp/app/v3/marqs/index.server.ts index 6839f7761b..e2ef37ea30 100644 --- a/apps/webapp/app/v3/marqs/index.server.ts +++ b/apps/webapp/app/v3/marqs/index.server.ts @@ -33,6 +33,7 @@ import { } from "./types"; import { V3VisibilityTimeout } from "./v3VisibilityTimeout.server"; import { concurrencyTracker } from "../services/taskRunConcurrencyTracker.server"; +export { sanitizeQueueName } from "@trigger.dev/core/v3/apps"; const KEY_PREFIX = "marqs:"; @@ -1660,7 +1661,7 @@ local currentConcurrency = tonumber(redis.call('SCARD', currentConcurrencyKey) o local concurrencyLimit = redis.call('GET', concurrencyLimitKey) -- Return current capacity and concurrency limits for the queue, env, org -return { currentConcurrency, concurrencyLimit, currentEnvConcurrency, envConcurrencyLimit, currentOrgConcurrency, orgConcurrencyLimit } +return { currentConcurrency, concurrencyLimit, currentEnvConcurrency, envConcurrencyLimit, currentOrgConcurrency, orgConcurrencyLimit } `, }); @@ -1689,7 +1690,7 @@ local currentConcurrency = tonumber(redis.call('SCARD', currentConcurrencyKey) o local concurrencyLimit = redis.call('GET', concurrencyLimitKey) -- Return current capacity and concurrency limits for the queue, env, org -return { currentConcurrency, concurrencyLimit, currentEnvConcurrency, envConcurrencyLimit, currentOrgConcurrency, orgConcurrencyLimit } +return { currentConcurrency, concurrencyLimit, currentEnvConcurrency, envConcurrencyLimit, currentOrgConcurrency, orgConcurrencyLimit } `, }); @@ -1908,8 +1909,3 @@ function getMarQSClient() { } } } - -// Only allow alphanumeric characters, underscores, hyphens, and slashes (and only the first 128 characters) -export function sanitizeQueueName(queueName: string) { - return queueName.replace(/[^a-zA-Z0-9_\-\/]/g, "").substring(0, 128); -} diff --git a/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts b/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts index c45ef65f59..9b419e4b09 100644 --- a/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts +++ b/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts @@ -44,13 +44,13 @@ import { generateJWTTokenForEnvironment } from "~/services/apiAuth.server"; import { EnvironmentVariable } from "../environmentVariables/repository"; import { machinePresetFromConfig } from "../machinePresets.server"; import { env } from "~/env.server"; +import { getMaxDuration } from "@trigger.dev/core/v3/apps"; import { FINAL_ATTEMPT_STATUSES, FINAL_RUN_STATUSES, isFinalAttemptStatus, isFinalRunStatus, } from "../taskStatus"; -import { getMaxDuration } from "../utils/maxDuration"; const WithTraceContext = z.object({ traceparent: z.string().optional(), diff --git a/apps/webapp/app/v3/models/workerDeployment.server.ts b/apps/webapp/app/v3/models/workerDeployment.server.ts index 49595dab7e..fa98dba818 100644 --- a/apps/webapp/app/v3/models/workerDeployment.server.ts +++ b/apps/webapp/app/v3/models/workerDeployment.server.ts @@ -1,6 +1,9 @@ import type { Prettify } from "@trigger.dev/core"; -import { BackgroundWorker } from "@trigger.dev/database"; -import { CURRENT_DEPLOYMENT_LABEL } from "~/consts"; +import { BackgroundWorker, WorkerDeployment } from "@trigger.dev/database"; +import { + CURRENT_DEPLOYMENT_LABEL, + CURRENT_UNMANAGED_DEPLOYMENT_LABEL, +} from "@trigger.dev/core/v3/apps"; import { Prisma, prisma } from "~/db.server"; import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; @@ -19,13 +22,14 @@ type WorkerDeploymentWithWorkerTasks = Prisma.WorkerDeploymentGetPayload<{ }>; export async function findCurrentWorkerDeployment( - environmentId: string + environmentId: string, + label = CURRENT_DEPLOYMENT_LABEL ): Promise { const promotion = await prisma.workerDeploymentPromotion.findUnique({ where: { environmentId_label: { environmentId, - label: CURRENT_DEPLOYMENT_LABEL, + label, }, }, include: { @@ -44,8 +48,34 @@ export async function findCurrentWorkerDeployment( return promotion?.deployment; } +export async function findCurrentWorkerDeploymentWithoutTasks( + environmentId: string, + label = CURRENT_DEPLOYMENT_LABEL +): Promise { + const promotion = await prisma.workerDeploymentPromotion.findUnique({ + where: { + environmentId_label: { + environmentId, + label, + }, + }, + include: { + deployment: true, + }, + }); + + return promotion?.deployment; +} + +export async function findCurrentUnmanagedWorkerDeployment( + environmentId: string +): Promise { + return await findCurrentWorkerDeployment(environmentId, CURRENT_UNMANAGED_DEPLOYMENT_LABEL); +} + export async function findCurrentWorkerFromEnvironment( - environment: Pick + environment: Pick, + label = CURRENT_DEPLOYMENT_LABEL ): Promise { if (environment.type === "DEVELOPMENT") { const latestDevWorker = await prisma.backgroundWorker.findFirst({ @@ -58,11 +88,21 @@ export async function findCurrentWorkerFromEnvironment( }); return latestDevWorker; } else { - const deployment = await findCurrentWorkerDeployment(environment.id); + const deployment = await findCurrentWorkerDeployment(environment.id, label); return deployment?.worker ?? null; } } +export async function findCurrentUnmanagedWorkerFromEnvironment( + environment: Pick +): Promise { + if (environment.type === "DEVELOPMENT") { + return null; + } + + return await findCurrentWorkerFromEnvironment(environment, CURRENT_UNMANAGED_DEPLOYMENT_LABEL); +} + export async function getWorkerDeploymentFromWorker( workerId: string ): Promise { diff --git a/apps/webapp/app/v3/registryProxy.server.ts b/apps/webapp/app/v3/registryProxy.server.ts index b06ee89304..df38d6e6ac 100644 --- a/apps/webapp/app/v3/registryProxy.server.ts +++ b/apps/webapp/app/v3/registryProxy.server.ts @@ -13,6 +13,7 @@ import { mkdtemp } from "fs/promises"; import { createReadStream, createWriteStream } from "node:fs"; import { pipeline } from "node:stream/promises"; import { unlinkSync } from "fs"; +import { parseDockerImageReference, rebuildDockerImageReference } from "@trigger.dev/core/v3"; const TokenResponseBody = z.object({ token: z.string(), @@ -461,70 +462,3 @@ async function streamRequestBodyToTempFile(request: IncomingMessage): Promise 1) { - parts.digest = atSplit[1]; - imageReference = atSplit[0]; - } - - // Splitting by ':' to separate the tag (if exists) and to ensure it's not part of a port - let colonSplit = imageReference.split(":"); - if (colonSplit.length > 2 || (colonSplit.length === 2 && !colonSplit[1].includes("/"))) { - // It's a tag if there's no '/' in the second part (after colon), or there are more than 2 parts (implying a port number in registry) - parts.tag = colonSplit.pop(); // The last part is the tag - imageReference = colonSplit.join(":"); // Join back in case it was a port number - } - - // Check for registry - let slashIndex = imageReference.indexOf("/"); - if (slashIndex !== -1) { - let potentialRegistry = imageReference.substring(0, slashIndex); - // Validate if the first part is a valid hostname-like string (registry), otherwise treat the entire string as the repo - if ( - potentialRegistry.includes(".") || - potentialRegistry === "localhost" || - potentialRegistry.includes(":") - ) { - parts.registry = potentialRegistry; - parts.repo = imageReference.substring(slashIndex + 1); - } else { - parts.repo = imageReference; // No valid registry found, treat as repo - } - } else { - parts.repo = imageReference; // Only repo is present - } - - return parts; -} - -function rebuildDockerImageReference(parts: DockerImageParts): string { - let imageReference = ""; - - if (parts.registry) { - imageReference += `${parts.registry}/`; - } - - imageReference += parts.repo; // Repo is now guaranteed to be defined - - if (parts.tag) { - imageReference += `:${parts.tag}`; - } - - if (parts.digest) { - imageReference += `@${parts.digest}`; - } - - return imageReference; -} diff --git a/apps/webapp/app/v3/runEngine.server.ts b/apps/webapp/app/v3/runEngine.server.ts new file mode 100644 index 0000000000..cdf1a9fa72 --- /dev/null +++ b/apps/webapp/app/v3/runEngine.server.ts @@ -0,0 +1,41 @@ +import { RunEngine } from "@internal/run-engine"; +import { prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { tracer } from "./tracer.server"; +import { singleton } from "~/utils/singleton"; +import { defaultMachine, machines } from "@trigger.dev/platform/v3"; +import { allMachines } from "./machinePresets.server"; + +export const engine = singleton("RunEngine", createRunEngine); + +export type { RunEngine }; + +function createRunEngine() { + const engine = new RunEngine({ + prisma, + redis: { + port: env.VALKEY_PORT ?? undefined, + host: env.VALKEY_HOST ?? undefined, + username: env.VALKEY_USERNAME ?? undefined, + password: env.VALKEY_PASSWORD ?? undefined, + enableAutoPipelining: true, + ...(env.VALKEY_TLS_DISABLED === "true" ? {} : { tls: {} }), + }, + worker: { + workers: env.RUN_ENGINE_WORKER_COUNT, + tasksPerWorker: env.RUN_ENGINE_TASKS_PER_WORKER, + pollIntervalMs: env.RUN_ENGINE_WORKER_POLL_INTERVAL, + }, + machines: { + defaultMachine, + machines: allMachines(), + baseCostInCents: env.CENTS_PER_RUN, + }, + queue: { + defaultEnvConcurrency: env.DEFAULT_ENV_EXECUTION_CONCURRENCY_LIMIT, + }, + tracer, + }); + + return engine; +} diff --git a/apps/webapp/app/v3/runEngineHandlers.server.ts b/apps/webapp/app/v3/runEngineHandlers.server.ts new file mode 100644 index 0000000000..952f72847c --- /dev/null +++ b/apps/webapp/app/v3/runEngineHandlers.server.ts @@ -0,0 +1,303 @@ +import { prisma } from "~/db.server"; +import { createExceptionPropertiesFromError, eventRepository } from "./eventRepository.server"; +import { createJsonErrorObject, sanitizeError } from "@trigger.dev/core/v3"; +import { logger } from "~/services/logger.server"; +import { safeJsonParse } from "~/utils/json"; +import type { Attributes } from "@opentelemetry/api"; +import { reportInvocationUsage } from "~/services/platform.v3.server"; +import { roomFromFriendlyRunId, socketIo } from "./handleSocketIo.server"; +import { engine } from "./runEngine.server"; +import { PerformTaskRunAlertsService } from "./services/alerts/performTaskRunAlerts.server"; +import { RunId } from "@trigger.dev/core/v3/apps"; + +export function registerRunEngineEventBusHandlers() { + engine.eventBus.on("runSucceeded", async ({ time, run }) => { + try { + const completedEvent = await eventRepository.completeEvent(run.spanId, { + endTime: time, + attributes: { + isError: false, + output: + run.outputType === "application/store" || run.outputType === "text/plain" + ? run.output + : run.output + ? (safeJsonParse(run.output) as Attributes) + : undefined, + outputType: run.outputType, + }, + }); + + if (!completedEvent) { + logger.error("[runSucceeded] Failed to complete event for unknown reason", { + runId: run.id, + spanId: run.spanId, + }); + return; + } + } catch (error) { + logger.error("[runSucceeded] Failed to complete event", { + error: error instanceof Error ? error.message : error, + runId: run.id, + spanId: run.spanId, + }); + } + }); + + // Handle alerts + engine.eventBus.on("runFailed", async ({ time, run }) => { + try { + await PerformTaskRunAlertsService.enqueue(run.id, prisma); + } catch (error) { + logger.error("[runFailed] Failed to enqueue alerts", { + error: error instanceof Error ? error.message : error, + runId: run.id, + spanId: run.spanId, + }); + } + }); + + // Handle events + engine.eventBus.on("runFailed", async ({ time, run }) => { + try { + const sanitizedError = sanitizeError(run.error); + const exception = createExceptionPropertiesFromError(sanitizedError); + + const completedEvent = await eventRepository.completeEvent(run.spanId, { + endTime: time, + attributes: { + isError: true, + }, + events: [ + { + name: "exception", + time, + properties: { + exception, + }, + }, + ], + }); + + if (!completedEvent) { + logger.error("[runFailed] Failed to complete event for unknown reason", { + runId: run.id, + spanId: run.spanId, + }); + return; + } + + const inProgressEvents = await eventRepository.queryIncompleteEvents({ + runId: completedEvent?.runId, + }); + + await Promise.all( + inProgressEvents.map((event) => { + try { + const completedEvent = eventRepository.completeEvent(event.spanId, { + endTime: time, + attributes: { + isError: true, + }, + events: [ + { + name: "exception", + time, + properties: { + exception, + }, + }, + ], + }); + + if (!completedEvent) { + logger.error("[runFailed] Failed to complete in-progress event for unknown reason", { + runId: run.id, + spanId: run.spanId, + eventId: event.id, + }); + return; + } + } catch (error) { + logger.error("[runFailed] Failed to complete in-progress event", { + error: error instanceof Error ? error.message : error, + runId: run.id, + spanId: run.spanId, + eventId: event.id, + }); + } + }) + ); + } catch (error) { + logger.error("[runFailed] Failed to complete event", { + error: error instanceof Error ? error.message : error, + runId: run.id, + spanId: run.spanId, + }); + } + }); + + engine.eventBus.on("runExpired", async ({ time, run }) => { + try { + const completedEvent = await eventRepository.completeEvent(run.spanId, { + endTime: time, + attributes: { + isError: true, + }, + events: [ + { + name: "exception", + time, + properties: { + exception: { + message: `Run expired because the TTL (${run.ttl}) was reached`, + }, + }, + }, + ], + }); + + if (!completedEvent) { + logger.error("[runFailed] Failed to complete event for unknown reason", { + runId: run.id, + spanId: run.spanId, + }); + return; + } + } catch (error) { + logger.error("[runExpired] Failed to complete event", { + error: error instanceof Error ? error.message : error, + runId: run.id, + spanId: run.spanId, + }); + } + }); + + engine.eventBus.on("runCancelled", async ({ time, run }) => { + try { + const inProgressEvents = await eventRepository.queryIncompleteEvents({ + runId: run.friendlyId, + }); + + await Promise.all( + inProgressEvents.map((event) => { + const error = createJsonErrorObject(run.error); + return eventRepository.cancelEvent(event, time, error.message); + }) + ); + } catch (error) { + logger.error("[runCancelled] Failed to cancel event", { + error: error instanceof Error ? error.message : error, + runId: run.id, + spanId: run.spanId, + }); + } + }); + + engine.eventBus.on("runRetryScheduled", async ({ time, run, environment, retryAt }) => { + try { + await eventRepository.recordEvent(`Retry #${run.attemptNumber} delay`, { + taskSlug: run.taskIdentifier, + environment, + attributes: { + properties: { + retryAt: retryAt.toISOString(), + }, + runId: run.friendlyId, + style: { + icon: "schedule-attempt", + }, + queueName: run.queue, + }, + context: run.traceContext as Record, + spanIdSeed: `retry-${run.attemptNumber + 1}`, + endTime: retryAt, + }); + } catch (error) { + logger.error("[runRetryScheduled] Failed to record retry event", { + error: error instanceof Error ? error.message : error, + runId: run.id, + spanId: run.spanId, + }); + } + }); + + engine.eventBus.on("runAttemptStarted", async ({ time, run, organization }) => { + try { + if (run.attemptNumber === 1 && run.baseCostInCents > 0) { + await reportInvocationUsage(organization.id, run.baseCostInCents, { runId: run.id }); + } + } catch (error) { + logger.error("[runAttemptStarted] Failed to report invocation usage", { + error: error instanceof Error ? error.message : error, + runId: run.id, + orgId: organization.id, + }); + } + }); + + engine.eventBus.on("executionSnapshotCreated", async ({ time, run, snapshot }) => { + try { + const foundRun = await prisma.taskRun.findUnique({ + where: { + id: run.id, + }, + include: { + runtimeEnvironment: { + include: { + project: true, + organization: true, + }, + }, + }, + }); + + if (!foundRun) { + logger.error("Failed to find run", { runId: run.id }); + return; + } + + await eventRepository.recordEvent( + `[ExecutionSnapshot] ${snapshot.executionStatus} - ${snapshot.description}`, + { + environment: foundRun.runtimeEnvironment, + taskSlug: foundRun.taskIdentifier, + context: foundRun.traceContext as Record, + attributes: { + runId: foundRun.friendlyId, + isDebug: true, + properties: { + snapshotId: snapshot.id, + snapshotDescription: snapshot.description, + snapshotStatus: snapshot.executionStatus, + }, + }, + duration: 0, + startTime: BigInt(time.getTime() * 1_000_000), + } + ); + } catch (error) { + logger.error("[executionSnapshotCreated] Failed to record event", { + error: error instanceof Error ? error.message : error, + runId: run.id, + }); + } + }); + + engine.eventBus.on("workerNotification", async ({ time, run }) => { + logger.debug("[workerNotification] Notifying worker", { time, runId: run.id }); + + try { + const runFriendlyId = RunId.toFriendlyId(run.id); + const room = roomFromFriendlyRunId(runFriendlyId); + + socketIo.workerNamespace + .to(room) + .emit("run:notify", { version: "1", run: { friendlyId: runFriendlyId } }); + } catch (error) { + logger.error("[workerNotification] Failed to notify worker", { + error: error instanceof Error ? error.message : error, + runId: run.id, + }); + } + }); +} diff --git a/apps/webapp/app/v3/runQueue.server.ts b/apps/webapp/app/v3/runQueue.server.ts new file mode 100644 index 0000000000..7198456d39 --- /dev/null +++ b/apps/webapp/app/v3/runQueue.server.ts @@ -0,0 +1,36 @@ +import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { marqs } from "./marqs/index.server"; +import { engine } from "./runEngine.server"; + +//This allows us to update MARQS and the RunQueue + +/** Updates MARQS and the RunQueue limits */ +export async function updateEnvConcurrencyLimits(environment: AuthenticatedEnvironment) { + await Promise.allSettled([ + marqs?.updateEnvConcurrencyLimits(environment), + engine.runQueue.updateEnvConcurrencyLimits(environment), + ]); +} + +/** Updates MARQS and the RunQueue limits for a queue */ +export async function updateQueueConcurrencyLimits( + environment: AuthenticatedEnvironment, + queueName: string, + concurrency: number +) { + await Promise.allSettled([ + marqs?.updateQueueConcurrencyLimits(environment, queueName, concurrency), + engine.runQueue.updateQueueConcurrencyLimits(environment, queueName, concurrency), + ]); +} + +/** Removes MARQS and the RunQueue limits for a queue */ +export async function removeQueueConcurrencyLimits( + environment: AuthenticatedEnvironment, + queueName: string +) { + await Promise.allSettled([ + marqs?.removeQueueConcurrencyLimits(environment, queueName), + engine.runQueue.removeQueueConcurrencyLimits(environment, queueName), + ]); +} diff --git a/apps/webapp/app/v3/services/baseService.server.ts b/apps/webapp/app/v3/services/baseService.server.ts index 4e7c79d46e..7686f41b6f 100644 --- a/apps/webapp/app/v3/services/baseService.server.ts +++ b/apps/webapp/app/v3/services/baseService.server.ts @@ -2,6 +2,7 @@ import { Span, SpanKind } from "@opentelemetry/api"; import { PrismaClientOrTransaction, prisma } from "~/db.server"; import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { attributesFromAuthenticatedEnv, tracer } from "../tracer.server"; +import { engine, RunEngine } from "../runEngine.server"; export abstract class BaseService { constructor(protected readonly _prisma: PrismaClientOrTransaction = prisma) {} @@ -37,6 +38,20 @@ export abstract class BaseService { } } +export type WithRunEngineOptions = T & { + prisma?: PrismaClientOrTransaction; + engine?: RunEngine; +}; + +export class WithRunEngine extends BaseService { + protected readonly _engine: RunEngine; + + constructor(opts: { prisma?: PrismaClientOrTransaction; engine?: RunEngine } = {}) { + super(opts.prisma); + this._engine = opts.engine ?? engine; + } +} + export class ServiceValidationError extends Error { constructor(message: string, public status?: number) { super(message); diff --git a/apps/webapp/app/v3/services/batchTriggerV2.server.ts b/apps/webapp/app/v3/services/batchTriggerV2.server.ts index 02a0cb768d..8c7bbf411f 100644 --- a/apps/webapp/app/v3/services/batchTriggerV2.server.ts +++ b/apps/webapp/app/v3/services/batchTriggerV2.server.ts @@ -49,6 +49,9 @@ export type BatchTriggerTaskServiceOptions = { oneTimeUseToken?: string; }; +/** + * Larger batches, used in Run Engine v1 + */ export class BatchTriggerV2Service extends BaseService { private _batchProcessingStrategy: BatchProcessingStrategy; @@ -709,18 +712,18 @@ export class BatchTriggerV2Service extends BaseService { | { status: "ERROR"; error: string; workingIndex: number } > { // Grab the next PROCESSING_BATCH_SIZE runIds - const runIds = batch.runIds.slice(currentIndex, currentIndex + batchSize); + const runFriendlyIds = batch.runIds.slice(currentIndex, currentIndex + batchSize); logger.debug("[BatchTriggerV2][processBatchTaskRun] Processing batch items", { batchId: batch.friendlyId, currentIndex, - runIds, + runIds: runFriendlyIds, runCount: batch.runCount, }); // Combine the "window" between currentIndex and currentIndex + PROCESSING_BATCH_SIZE with the runId and the item in the payload which is an array - const itemsToProcess = runIds.map((runId, index) => ({ - runId, + const itemsToProcess = runFriendlyIds.map((runFriendlyId, index) => ({ + runFriendlyId, item: items[index + currentIndex], })); @@ -757,13 +760,13 @@ export class BatchTriggerV2Service extends BaseService { async #processBatchTaskRunItem( batch: BatchTaskRun, environment: AuthenticatedEnvironment, - task: { runId: string; item: BatchTriggerTaskV2RequestBody["items"][number] }, + task: { runFriendlyId: string; item: BatchTriggerTaskV2RequestBody["items"][number] }, currentIndex: number, options?: BatchTriggerTaskServiceOptions ) { logger.debug("[BatchTriggerV2][processBatchTaskRunItem] Processing item", { batchId: batch.friendlyId, - runId: task.runId, + runId: task.runFriendlyId, currentIndex, }); @@ -786,12 +789,13 @@ export class BatchTriggerV2Service extends BaseService { spanParentAsLink: options?.spanParentAsLink, batchId: batch.friendlyId, skipChecks: true, - runId: task.runId, - } + runFriendlyId: task.runFriendlyId, + }, + "V1" ); if (!run) { - throw new Error(`Failed to trigger run ${task.runId} for batch ${batch.friendlyId}`); + throw new Error(`Failed to trigger run ${task.runFriendlyId} for batch ${batch.friendlyId}`); } await this._prisma.batchTaskRunItem.create({ diff --git a/apps/webapp/app/v3/services/batchTriggerV3.server.ts b/apps/webapp/app/v3/services/batchTriggerV3.server.ts new file mode 100644 index 0000000000..66c259f83a --- /dev/null +++ b/apps/webapp/app/v3/services/batchTriggerV3.server.ts @@ -0,0 +1,914 @@ +import { + BatchTriggerTaskV2RequestBody, + BatchTriggerTaskV2Response, + IOPacket, + packetRequiresOffloading, + parsePacket, +} from "@trigger.dev/core/v3"; +import { BatchId, RunId } from "@trigger.dev/core/v3/apps"; +import { BatchTaskRun, Prisma } from "@trigger.dev/database"; +import { z } from "zod"; +import { $transaction, prisma, PrismaClientOrTransaction } from "~/db.server"; +import { env } from "~/env.server"; +import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { logger } from "~/services/logger.server"; +import { getEntitlement } from "~/services/platform.v3.server"; +import { workerQueue } from "~/services/worker.server"; +import { downloadPacketFromObjectStore, uploadPacketToObjectStore } from "../r2.server"; +import { startActiveSpan } from "../tracer.server"; +import { ServiceValidationError, WithRunEngine } from "./baseService.server"; +import { OutOfEntitlementError, TriggerTaskService } from "./triggerTask.server"; +import { guardQueueSizeLimitsForEnv } from "./triggerTaskV2.server"; + +const PROCESSING_BATCH_SIZE = 50; +const ASYNC_BATCH_PROCESS_SIZE_THRESHOLD = 20; +const MAX_ATTEMPTS = 10; + +export const BatchProcessingStrategy = z.enum(["sequential", "parallel"]); +export type BatchProcessingStrategy = z.infer; + +export const BatchProcessingOptions = z.object({ + batchId: z.string(), + processingId: z.string(), + range: z.object({ start: z.number().int(), count: z.number().int() }), + attemptCount: z.number().int(), + strategy: BatchProcessingStrategy, + parentRunId: z.string().optional(), + resumeParentOnCompletion: z.boolean().optional(), +}); + +export type BatchProcessingOptions = z.infer; + +export type BatchTriggerTaskServiceOptions = { + idempotencyKey?: string; + idempotencyKeyExpiresAt?: Date; + triggerVersion?: string; + traceContext?: Record; + spanParentAsLink?: boolean; + oneTimeUseToken?: string; +}; + +/** + * Larger batches, used in Run Engine v2 + */ +export class BatchTriggerV3Service extends WithRunEngine { + private _batchProcessingStrategy: BatchProcessingStrategy; + + constructor( + batchProcessingStrategy?: BatchProcessingStrategy, + protected readonly _prisma: PrismaClientOrTransaction = prisma + ) { + super({ prisma }); + + this._batchProcessingStrategy = batchProcessingStrategy ?? "parallel"; + } + + public async call( + environment: AuthenticatedEnvironment, + body: BatchTriggerTaskV2RequestBody, + options: BatchTriggerTaskServiceOptions = {} + ): Promise { + try { + return await this.traceWithEnv( + "call()", + environment, + async (span) => { + const existingBatch = options.idempotencyKey + ? await this._prisma.batchTaskRun.findUnique({ + where: { + runtimeEnvironmentId_idempotencyKey: { + runtimeEnvironmentId: environment.id, + idempotencyKey: options.idempotencyKey, + }, + }, + }) + : undefined; + + if (existingBatch) { + if ( + existingBatch.idempotencyKeyExpiresAt && + existingBatch.idempotencyKeyExpiresAt < new Date() + ) { + logger.debug("[BatchTriggerV3][call] Idempotency key has expired", { + idempotencyKey: options.idempotencyKey, + batch: { + id: existingBatch.id, + friendlyId: existingBatch.friendlyId, + runCount: existingBatch.runCount, + idempotencyKeyExpiresAt: existingBatch.idempotencyKeyExpiresAt, + idempotencyKey: existingBatch.idempotencyKey, + }, + }); + + // Update the existing batch to remove the idempotency key + await this._prisma.batchTaskRun.update({ + where: { id: existingBatch.id }, + data: { idempotencyKey: null }, + }); + + // Don't return, just continue with the batch trigger + } else { + span.setAttribute("batchId", existingBatch.friendlyId); + + return this.#respondWithExistingBatch( + existingBatch, + environment, + body.resumeParentOnCompletion ? body.parentRunId : undefined + ); + } + } + + const { id, friendlyId } = BatchId.generate(); + + span.setAttribute("batchId", friendlyId); + + if (environment.type !== "DEVELOPMENT") { + const result = await getEntitlement(environment.organizationId); + if (result && result.hasAccess === false) { + throw new OutOfEntitlementError(); + } + } + + const idempotencyKeys = body.items.map((i) => i.options?.idempotencyKey).filter(Boolean); + + const cachedRuns = + idempotencyKeys.length > 0 + ? await this._prisma.taskRun.findMany({ + where: { + runtimeEnvironmentId: environment.id, + idempotencyKey: { + in: body.items.map((i) => i.options?.idempotencyKey).filter(Boolean), + }, + }, + select: { + friendlyId: true, + idempotencyKey: true, + idempotencyKeyExpiresAt: true, + }, + }) + : []; + + if (cachedRuns.length) { + logger.debug("[BatchTriggerV3][call] Found cached runs", { + cachedRuns, + batchId: friendlyId, + }); + } + + // Now we need to create an array of all the run IDs, in order + // If we have a cached run, that isn't expired, we should use that run ID + // If we have a cached run, that is expired, we should generate a new run ID and save that cached run ID to a set of expired run IDs + // If we don't have a cached run, we should generate a new run ID + const expiredRunIds = new Set(); + let cachedRunCount = 0; + + const runs = body.items.map((item) => { + const cachedRun = cachedRuns.find( + (r) => r.idempotencyKey === item.options?.idempotencyKey + ); + + const runId = RunId.generate(); + + if (cachedRun) { + if ( + cachedRun.idempotencyKeyExpiresAt && + cachedRun.idempotencyKeyExpiresAt < new Date() + ) { + expiredRunIds.add(cachedRun.friendlyId); + + return { + id: runId.friendlyId, + isCached: false, + idempotencyKey: item.options?.idempotencyKey ?? undefined, + taskIdentifier: item.task, + }; + } + + cachedRunCount++; + + return { + id: cachedRun.friendlyId, + isCached: true, + idempotencyKey: item.options?.idempotencyKey ?? undefined, + taskIdentifier: item.task, + }; + } + + return { + id: runId.friendlyId, + isCached: false, + idempotencyKey: item.options?.idempotencyKey ?? undefined, + taskIdentifier: item.task, + }; + }); + + //block the parent with any existing children + if (body.resumeParentOnCompletion && body.parentRunId) { + const existingChildFriendlyIds = runs.flatMap((r) => (r.isCached ? [r.id] : [])); + + if (existingChildFriendlyIds.length > 0) { + await this.#blockParentRun({ + parentRunId: body.parentRunId, + childFriendlyIds: existingChildFriendlyIds, + environment, + }); + } + } + + // Calculate how many new runs we need to create + const newRunCount = body.items.length - cachedRunCount; + + if (newRunCount === 0) { + logger.debug("[BatchTriggerV3][call] All runs are cached", { + batchId: friendlyId, + }); + + await this._prisma.batchTaskRun.create({ + data: { + friendlyId, + runtimeEnvironmentId: environment.id, + idempotencyKey: options.idempotencyKey, + idempotencyKeyExpiresAt: options.idempotencyKeyExpiresAt, + runCount: body.items.length, + runIds: runs.map((r) => r.id), + //todo is this correct? Surely some of the runs could still be in progress? + status: "COMPLETED", + batchVersion: "v2", + oneTimeUseToken: options.oneTimeUseToken, + }, + }); + + return { + id: friendlyId, + isCached: false, + idempotencyKey: options.idempotencyKey ?? undefined, + runs, + }; + } + + const queueSizeGuard = await guardQueueSizeLimitsForEnv( + this._engine, + environment, + newRunCount + ); + + logger.debug("Queue size guard result", { + newRunCount, + queueSizeGuard, + environment: { + id: environment.id, + type: environment.type, + organization: environment.organization, + project: environment.project, + }, + }); + + if (!queueSizeGuard.isWithinLimits) { + throw new ServiceValidationError( + `Cannot trigger ${newRunCount} tasks as the queue size limit for this environment has been reached. The maximum size is ${queueSizeGuard.maximumSize}` + ); + } + + // Expire the cached runs that are no longer valid + if (expiredRunIds.size) { + logger.debug("Expiring cached runs", { + expiredRunIds: Array.from(expiredRunIds), + batchId: friendlyId, + }); + + // TODO: is there a limit to the number of items we can update in a single query? + await this._prisma.taskRun.updateMany({ + where: { friendlyId: { in: Array.from(expiredRunIds) } }, + data: { idempotencyKey: null }, + }); + } + + // Upload to object store + const payloadPacket = await this.#handlePayloadPacket( + body.items, + `batch/${friendlyId}`, + environment + ); + + const batch = await this.#createAndProcessBatchTaskRun( + friendlyId, + runs, + payloadPacket, + newRunCount, + environment, + body, + options + ); + + if (!batch) { + throw new Error("Failed to create batch"); + } + + return { + id: batch.friendlyId, + isCached: false, + idempotencyKey: batch.idempotencyKey ?? undefined, + runs, + }; + } + ); + } catch (error) { + // Detect a prisma transaction Unique constraint violation + if (error instanceof Prisma.PrismaClientKnownRequestError) { + logger.debug("BatchTriggerV3: Prisma transaction error", { + code: error.code, + message: error.message, + meta: error.meta, + }); + + if (error.code === "P2002") { + const target = error.meta?.target; + + if ( + Array.isArray(target) && + target.length > 0 && + typeof target[0] === "string" && + target[0].includes("oneTimeUseToken") + ) { + throw new ServiceValidationError( + "Cannot batch trigger with a one-time use token as it has already been used." + ); + } else { + throw new ServiceValidationError( + "Cannot batch trigger as it has already been triggered with the same idempotency key." + ); + } + } + } + + throw error; + } + } + + async #createAndProcessBatchTaskRun( + batchId: string, + runs: Array<{ + id: string; + isCached: boolean; + idempotencyKey: string | undefined; + taskIdentifier: string; + }>, + payloadPacket: IOPacket, + newRunCount: number, + environment: AuthenticatedEnvironment, + body: BatchTriggerTaskV2RequestBody, + options: BatchTriggerTaskServiceOptions = {} + ) { + if (newRunCount <= ASYNC_BATCH_PROCESS_SIZE_THRESHOLD) { + const batch = await this._prisma.batchTaskRun.create({ + data: { + friendlyId: batchId, + runtimeEnvironmentId: environment.id, + idempotencyKey: options.idempotencyKey, + idempotencyKeyExpiresAt: options.idempotencyKeyExpiresAt, + runCount: newRunCount, + runIds: runs.map((r) => r.id), + payload: payloadPacket.data, + payloadType: payloadPacket.dataType, + options, + batchVersion: "v2", + oneTimeUseToken: options.oneTimeUseToken, + }, + }); + + const result = await this.#processBatchTaskRunItems({ + batch, + environment, + currentIndex: 0, + batchSize: PROCESSING_BATCH_SIZE, + items: body.items, + options, + parentRunId: body.parentRunId, + resumeParentOnCompletion: body.resumeParentOnCompletion, + }); + + switch (result.status) { + case "COMPLETE": { + logger.debug("[BatchTriggerV3][call] Batch inline processing complete", { + batchId: batch.friendlyId, + currentIndex: 0, + }); + + return batch; + } + case "INCOMPLETE": { + logger.debug("[BatchTriggerV3][call] Batch inline processing incomplete", { + batchId: batch.friendlyId, + currentIndex: result.workingIndex, + }); + + // If processing inline does not finish for some reason, enqueue processing the rest of the batch + await this.#enqueueBatchTaskRun({ + batchId: batch.id, + processingId: "0", + range: { + start: result.workingIndex, + count: PROCESSING_BATCH_SIZE, + }, + attemptCount: 0, + strategy: "sequential", + parentRunId: body.parentRunId, + resumeParentOnCompletion: body.resumeParentOnCompletion, + }); + + return batch; + } + case "ERROR": { + logger.error("[BatchTriggerV3][call] Batch inline processing error", { + batchId: batch.friendlyId, + currentIndex: result.workingIndex, + error: result.error, + }); + + await this.#enqueueBatchTaskRun({ + batchId: batch.id, + processingId: "0", + range: { + start: result.workingIndex, + count: PROCESSING_BATCH_SIZE, + }, + attemptCount: 0, + strategy: "sequential", + parentRunId: body.parentRunId, + resumeParentOnCompletion: body.resumeParentOnCompletion, + }); + + return batch; + } + } + } else { + return await $transaction(this._prisma, async (tx) => { + const batch = await tx.batchTaskRun.create({ + data: { + friendlyId: batchId, + runtimeEnvironmentId: environment.id, + idempotencyKey: options.idempotencyKey, + idempotencyKeyExpiresAt: options.idempotencyKeyExpiresAt, + runCount: body.items.length, + runIds: runs.map((r) => r.id), + payload: payloadPacket.data, + payloadType: payloadPacket.dataType, + options, + batchVersion: "v2", + oneTimeUseToken: options.oneTimeUseToken, + }, + }); + + switch (this._batchProcessingStrategy) { + case "sequential": { + await this.#enqueueBatchTaskRun({ + batchId: batch.id, + processingId: batchId, + range: { start: 0, count: PROCESSING_BATCH_SIZE }, + attemptCount: 0, + strategy: this._batchProcessingStrategy, + parentRunId: body.parentRunId, + resumeParentOnCompletion: body.resumeParentOnCompletion, + }); + + break; + } + case "parallel": { + const ranges = Array.from({ + length: Math.ceil(newRunCount / PROCESSING_BATCH_SIZE), + }).map((_, index) => ({ + start: index * PROCESSING_BATCH_SIZE, + count: PROCESSING_BATCH_SIZE, + })); + + await Promise.all( + ranges.map((range, index) => + this.#enqueueBatchTaskRun( + { + batchId: batch.id, + processingId: `${index}`, + range, + attemptCount: 0, + strategy: this._batchProcessingStrategy, + parentRunId: body.parentRunId, + resumeParentOnCompletion: body.resumeParentOnCompletion, + }, + tx + ) + ) + ); + + break; + } + } + + return batch; + }); + } + } + + async #respondWithExistingBatch( + batch: BatchTaskRun, + environment: AuthenticatedEnvironment, + blockParentRunId: string | undefined + ): Promise { + // Resolve the payload + const payloadPacket = await downloadPacketFromObjectStore( + { + data: batch.payload ?? undefined, + dataType: batch.payloadType, + }, + environment + ); + + const payload = await parsePacket(payloadPacket).then( + (p) => p as BatchTriggerTaskV2RequestBody["items"] + ); + + const runs = batch.runIds.map((id, index) => { + const item = payload[index]; + + return { + id, + taskIdentifier: item.task, + isCached: true, + idempotencyKey: item.options?.idempotencyKey ?? undefined, + }; + }); + + //block the parent with all of the children + if (blockParentRunId) { + await this.#blockParentRun({ + parentRunId: blockParentRunId, + childFriendlyIds: batch.runIds, + environment, + }); + } + + return { + id: batch.friendlyId, + idempotencyKey: batch.idempotencyKey ?? undefined, + isCached: true, + runs, + }; + } + + async processBatchTaskRun(options: BatchProcessingOptions) { + logger.debug("[BatchTriggerV3][processBatchTaskRun] Processing batch", { + options, + }); + + const $attemptCount = options.attemptCount + 1; + + // Add early return if max attempts reached + if ($attemptCount > MAX_ATTEMPTS) { + logger.error("[BatchTriggerV3][processBatchTaskRun] Max attempts reached", { + options, + attemptCount: $attemptCount, + }); + // You might want to update the batch status to failed here + return; + } + + const batch = await this._prisma.batchTaskRun.findFirst({ + where: { id: options.batchId }, + include: { + runtimeEnvironment: { + include: { + project: true, + organization: true, + }, + }, + }, + }); + + if (!batch) { + return; + } + + // Check to make sure the currentIndex is not greater than the runCount + if (options.range.start >= batch.runCount) { + logger.debug("[BatchTriggerV3][processBatchTaskRun] currentIndex is greater than runCount", { + options, + batchId: batch.friendlyId, + runCount: batch.runCount, + attemptCount: $attemptCount, + }); + + return; + } + + // Resolve the payload + const payloadPacket = await downloadPacketFromObjectStore( + { + data: batch.payload ?? undefined, + dataType: batch.payloadType, + }, + batch.runtimeEnvironment + ); + + const payload = await parsePacket(payloadPacket); + + if (!payload) { + logger.debug("[BatchTriggerV3][processBatchTaskRun] Failed to parse payload", { + options, + batchId: batch.friendlyId, + attemptCount: $attemptCount, + }); + + throw new Error("Failed to parse payload"); + } + + // Skip zod parsing + const $payload = payload as BatchTriggerTaskV2RequestBody["items"]; + const $options = batch.options as BatchTriggerTaskServiceOptions; + + const result = await this.#processBatchTaskRunItems({ + batch, + environment: batch.runtimeEnvironment, + currentIndex: options.range.start, + batchSize: options.range.count, + items: $payload, + options: $options, + }); + + switch (result.status) { + case "COMPLETE": { + logger.debug("[BatchTriggerV3][processBatchTaskRun] Batch processing complete", { + options, + batchId: batch.friendlyId, + attemptCount: $attemptCount, + }); + + return; + } + case "INCOMPLETE": { + logger.debug("[BatchTriggerV3][processBatchTaskRun] Batch processing incomplete", { + batchId: batch.friendlyId, + currentIndex: result.workingIndex, + attemptCount: $attemptCount, + }); + + // Only enqueue the next batch task run if the strategy is sequential + // if the strategy is parallel, we will already have enqueued the next batch task run + if (options.strategy === "sequential") { + await this.#enqueueBatchTaskRun({ + batchId: batch.id, + processingId: options.processingId, + range: { + start: result.workingIndex, + count: options.range.count, + }, + attemptCount: 0, + strategy: options.strategy, + parentRunId: options.parentRunId, + resumeParentOnCompletion: options.resumeParentOnCompletion, + }); + } + + return; + } + case "ERROR": { + logger.error("[BatchTriggerV3][processBatchTaskRun] Batch processing error", { + batchId: batch.friendlyId, + currentIndex: result.workingIndex, + error: result.error, + attemptCount: $attemptCount, + }); + + // if the strategy is sequential, we will requeue processing with a count of the PROCESSING_BATCH_SIZE + // if the strategy is parallel, we will requeue processing with a range starting at the workingIndex and a count that is the remainder of this "slice" of the batch + if (options.strategy === "sequential") { + await this.#enqueueBatchTaskRun({ + batchId: batch.id, + processingId: options.processingId, + range: { + start: result.workingIndex, + count: options.range.count, // This will be the same as the original count + }, + attemptCount: $attemptCount, + strategy: options.strategy, + parentRunId: options.parentRunId, + resumeParentOnCompletion: options.resumeParentOnCompletion, + }); + } else { + await this.#enqueueBatchTaskRun({ + batchId: batch.id, + processingId: options.processingId, + range: { + start: result.workingIndex, + // This will be the remainder of the slice + // for example if the original range was 0-50 and the workingIndex is 25, the new range will be 25-25 + // if the original range was 51-100 and the workingIndex is 75, the new range will be 75-25 + count: options.range.count - result.workingIndex - options.range.start, + }, + attemptCount: $attemptCount, + strategy: options.strategy, + parentRunId: options.parentRunId, + resumeParentOnCompletion: options.resumeParentOnCompletion, + }); + } + + return; + } + } + } + + async #processBatchTaskRunItems({ + batch, + environment, + currentIndex, + batchSize, + items, + options, + parentRunId, + resumeParentOnCompletion, + }: { + batch: BatchTaskRun; + environment: AuthenticatedEnvironment; + currentIndex: number; + batchSize: number; + items: BatchTriggerTaskV2RequestBody["items"]; + options?: BatchTriggerTaskServiceOptions; + parentRunId?: string | undefined; + resumeParentOnCompletion?: boolean | undefined; + }): Promise< + | { status: "COMPLETE" } + | { status: "INCOMPLETE"; workingIndex: number } + | { status: "ERROR"; error: string; workingIndex: number } + > { + // Grab the next PROCESSING_BATCH_SIZE runIds + const runFriendlyIds = batch.runIds.slice(currentIndex, currentIndex + batchSize); + + logger.debug("[BatchTriggerV3][processBatchTaskRun] Processing batch items", { + batchId: batch.friendlyId, + currentIndex, + runIds: runFriendlyIds, + runCount: batch.runCount, + }); + + // Combine the "window" between currentIndex and currentIndex + PROCESSING_BATCH_SIZE with the runId and the item in the payload which is an array + const itemsToProcess = runFriendlyIds.map((runFriendlyId, index) => ({ + runFriendlyId, + item: items[index + currentIndex], + })); + + let workingIndex = currentIndex; + + for (const item of itemsToProcess) { + try { + await this.#processBatchTaskRunItem({ + batch, + environment, + task: item, + currentIndex: workingIndex, + options, + parentRunId, + resumeParentOnCompletion, + }); + + workingIndex++; + } catch (error) { + logger.error("[BatchTriggerV3][processBatchTaskRun] Failed to process item", { + batchId: batch.friendlyId, + currentIndex: workingIndex, + error, + }); + + return { + status: "ERROR", + error: error instanceof Error ? error.message : String(error), + workingIndex, + }; + } + } + + // if there are more items to process, requeue the batch + if (workingIndex < batch.runCount) { + return { status: "INCOMPLETE", workingIndex }; + } + + return { status: "COMPLETE" }; + } + + async #processBatchTaskRunItem({ + batch, + environment, + task, + currentIndex, + options, + parentRunId, + resumeParentOnCompletion, + }: { + batch: BatchTaskRun; + environment: AuthenticatedEnvironment; + task: { runFriendlyId: string; item: BatchTriggerTaskV2RequestBody["items"][number] }; + currentIndex: number; + options?: BatchTriggerTaskServiceOptions; + parentRunId: string | undefined; + resumeParentOnCompletion: boolean | undefined; + }) { + logger.debug("[BatchTriggerV3][processBatchTaskRunItem] Processing item", { + batchId: batch.friendlyId, + runId: task.runFriendlyId, + currentIndex, + }); + + const triggerTaskService = new TriggerTaskService(); + + await triggerTaskService.call( + task.item.task, + environment, + { + ...task.item, + options: { + ...task.item.options, + parentRunId, + resumeParentOnCompletion, + }, + }, + { + triggerVersion: options?.triggerVersion, + traceContext: options?.traceContext, + spanParentAsLink: options?.spanParentAsLink, + batchId: batch.friendlyId, + skipChecks: true, + runFriendlyId: task.runFriendlyId, + }, + "V2" + ); + } + + async #enqueueBatchTaskRun(options: BatchProcessingOptions, tx?: PrismaClientOrTransaction) { + await workerQueue.enqueue("v3.processBatchTaskRunV3", options, { + tx, + jobKey: `BatchTriggerV3Service.process:${options.batchId}:${options.processingId}`, + }); + } + + async #handlePayloadPacket( + payload: any, + pathPrefix: string, + environment: AuthenticatedEnvironment + ) { + return await startActiveSpan("handlePayloadPacket()", async (span) => { + const packet = { data: JSON.stringify(payload), dataType: "application/json" }; + + if (!packet.data) { + return packet; + } + + const { needsOffloading } = packetRequiresOffloading( + packet, + env.TASK_PAYLOAD_OFFLOAD_THRESHOLD + ); + + if (!needsOffloading) { + return packet; + } + + const filename = `${pathPrefix}/payload.json`; + + await uploadPacketToObjectStore(filename, packet.data, packet.dataType, environment); + + return { + data: filename, + dataType: "application/store", + }; + }); + } + + async #blockParentRun({ + parentRunId, + childFriendlyIds, + environment, + }: { + parentRunId: string; + childFriendlyIds: string[]; + environment: AuthenticatedEnvironment; + }) { + const runsWithAssociatedWaitpoints = await this._prisma.taskRun.findMany({ + where: { + id: { + in: childFriendlyIds.map((r) => RunId.fromFriendlyId(r)), + }, + }, + select: { + associatedWaitpoint: { + select: { + id: true, + }, + }, + }, + }); + + await this._engine.blockRunWithWaitpoint({ + runId: RunId.fromFriendlyId(parentRunId), + waitpointId: runsWithAssociatedWaitpoints.flatMap((r) => + r.associatedWaitpoint ? [r.associatedWaitpoint.id] : [] + ), + environmentId: environment.id, + projectId: environment.projectId, + }); + } +} diff --git a/apps/webapp/app/v3/services/cancelTaskRun.server.ts b/apps/webapp/app/v3/services/cancelTaskRun.server.ts index a0f37ab23b..32387994fc 100644 --- a/apps/webapp/app/v3/services/cancelTaskRun.server.ts +++ b/apps/webapp/app/v3/services/cancelTaskRun.server.ts @@ -1,27 +1,7 @@ -import { type Prisma, type TaskRun } from "@trigger.dev/database"; -import assertNever from "assert-never"; -import { logger } from "~/services/logger.server"; -import { eventRepository } from "../eventRepository.server"; -import { socketIo } from "../handleSocketIo.server"; -import { devPubSub } from "../marqs/devPubSub.server"; -import { CANCELLABLE_ATTEMPT_STATUSES, isCancellableRunStatus } from "../taskStatus"; +import { RunEngineVersion, type TaskRun } from "@trigger.dev/database"; import { BaseService } from "./baseService.server"; -import { CancelAttemptService } from "./cancelAttempt.server"; -import { CancelTaskAttemptDependenciesService } from "./cancelTaskAttemptDependencies.server"; -import { FinalizeTaskRunService } from "./finalizeTaskRun.server"; - -type ExtendedTaskRun = Prisma.TaskRunGetPayload<{ - include: { - runtimeEnvironment: true; - lockedToVersion: true; - }; -}>; - -type ExtendedTaskRunAttempt = Prisma.TaskRunAttemptGetPayload<{ - include: { - backgroundWorker: true; - }; -}>; +import { CancelTaskRunServiceV1 } from "./cancelTaskRunV1.server"; +import { engine } from "../runEngine.server"; export type CancelTaskRunServiceOptions = { reason?: string; @@ -29,158 +9,43 @@ export type CancelTaskRunServiceOptions = { cancelledAt?: Date; }; -export class CancelTaskRunService extends BaseService { - public async call(taskRun: TaskRun, options?: CancelTaskRunServiceOptions) { - const opts = { - reason: "Task run was cancelled by user", - cancelAttempts: true, - cancelledAt: new Date(), - ...options, - }; +type CancelTaskRunServiceResult = { + id: string; +}; - // Make sure the task run is in a cancellable state - if (!isCancellableRunStatus(taskRun.status)) { - logger.error("Task run is not in a cancellable state", { - runId: taskRun.id, - status: taskRun.status, - }); - return; +export class CancelTaskRunService extends BaseService { + public async call( + taskRun: TaskRun, + options?: CancelTaskRunServiceOptions + ): Promise { + if (taskRun.engine === RunEngineVersion.V1) { + return await this.callV1(taskRun, options); + } else { + return await this.callV2(taskRun, options); } + } - const finalizeService = new FinalizeTaskRunService(); - const cancelledTaskRun = await finalizeService.call({ - id: taskRun.id, - status: "CANCELED", - completedAt: opts.cancelledAt, - include: { - attempts: { - where: { - status: { - in: CANCELLABLE_ATTEMPT_STATUSES, - }, - }, - include: { - backgroundWorker: true, - dependencies: { - include: { - taskRun: true, - }, - }, - batchTaskRunItems: { - include: { - taskRun: true, - }, - }, - }, - }, - runtimeEnvironment: true, - lockedToVersion: true, - }, - attemptStatus: "CANCELED", - error: { - type: "STRING_ERROR", - raw: opts.reason, - }, - }); - - const inProgressEvents = await eventRepository.queryIncompleteEvents({ - runId: taskRun.friendlyId, - }); + private async callV1( + taskRun: TaskRun, + options?: CancelTaskRunServiceOptions + ): Promise { + const service = new CancelTaskRunServiceV1(this._prisma); + return await service.call(taskRun, options); + } - logger.debug("Cancelling in-progress events", { - inProgressEvents: inProgressEvents.map((event) => event.id), + private async callV2( + taskRun: TaskRun, + options?: CancelTaskRunServiceOptions + ): Promise { + const result = await engine.cancelRun({ + runId: taskRun.id, + completedAt: options?.cancelledAt, + reason: options?.reason, + tx: this._prisma, }); - await Promise.all( - inProgressEvents.map((event) => { - return eventRepository.cancelEvent(event, opts.cancelledAt, opts.reason); - }) - ); - - // Cancel any in progress attempts - if (opts.cancelAttempts) { - await this.#cancelPotentiallyRunningAttempts(cancelledTaskRun, cancelledTaskRun.attempts); - await this.#cancelRemainingRunWorkers(cancelledTaskRun); - } - return { - id: cancelledTaskRun.id, + id: result.run.id, }; } - - async #cancelPotentiallyRunningAttempts( - run: ExtendedTaskRun, - attempts: ExtendedTaskRunAttempt[] - ) { - for (const attempt of attempts) { - await CancelTaskAttemptDependenciesService.enqueue(attempt.id, this._prisma); - - if (run.runtimeEnvironment.type === "DEVELOPMENT") { - // Signal the task run attempt to stop - await devPubSub.publish( - `backgroundWorker:${attempt.backgroundWorkerId}:${attempt.id}`, - "CANCEL_ATTEMPT", - { - attemptId: attempt.friendlyId, - backgroundWorkerId: attempt.backgroundWorker.friendlyId, - taskRunId: run.friendlyId, - } - ); - } else { - switch (attempt.status) { - case "EXECUTING": { - // We need to send a cancel message to the coordinator - socketIo.coordinatorNamespace.emit("REQUEST_ATTEMPT_CANCELLATION", { - version: "v1", - attemptId: attempt.id, - attemptFriendlyId: attempt.friendlyId, - }); - - break; - } - case "PENDING": - case "PAUSED": { - logger.debug("Cancelling pending or paused attempt", { - attempt, - }); - - const service = new CancelAttemptService(); - - await service.call( - attempt.friendlyId, - run.id, - new Date(), - "Task run was cancelled by user" - ); - - break; - } - case "CANCELED": - case "COMPLETED": - case "FAILED": { - // Do nothing - break; - } - default: { - assertNever(attempt.status); - } - } - } - } - } - - async #cancelRemainingRunWorkers(run: ExtendedTaskRun) { - if (run.runtimeEnvironment.type === "DEVELOPMENT") { - // Nothing to do - return; - } - - // Broadcast cancel message to all coordinators - socketIo.coordinatorNamespace.emit("REQUEST_RUN_CANCELLATION", { - version: "v1", - runId: run.id, - // Give the attempts some time to exit gracefully. If the runs supports lazy attempts, it also supports exit delays. - delayInMs: run.lockedToVersion?.supportsLazyAttempts ? 5_000 : undefined, - }); - } } diff --git a/apps/webapp/app/v3/services/cancelTaskRunV1.server.ts b/apps/webapp/app/v3/services/cancelTaskRunV1.server.ts new file mode 100644 index 0000000000..f25a0da86c --- /dev/null +++ b/apps/webapp/app/v3/services/cancelTaskRunV1.server.ts @@ -0,0 +1,186 @@ +import { type Prisma, type TaskRun } from "@trigger.dev/database"; +import assertNever from "assert-never"; +import { logger } from "~/services/logger.server"; +import { eventRepository } from "../eventRepository.server"; +import { socketIo } from "../handleSocketIo.server"; +import { devPubSub } from "../marqs/devPubSub.server"; +import { CANCELLABLE_ATTEMPT_STATUSES, isCancellableRunStatus } from "../taskStatus"; +import { BaseService } from "./baseService.server"; +import { CancelAttemptService } from "./cancelAttempt.server"; +import { CancelTaskAttemptDependenciesService } from "./cancelTaskAttemptDependencies.server"; +import { FinalizeTaskRunService } from "./finalizeTaskRun.server"; + +type ExtendedTaskRun = Prisma.TaskRunGetPayload<{ + include: { + runtimeEnvironment: true; + lockedToVersion: true; + }; +}>; + +type ExtendedTaskRunAttempt = Prisma.TaskRunAttemptGetPayload<{ + include: { + backgroundWorker: true; + }; +}>; + +export type CancelTaskRunServiceOptions = { + reason?: string; + cancelAttempts?: boolean; + cancelledAt?: Date; +}; + +export class CancelTaskRunServiceV1 extends BaseService { + public async call(taskRun: TaskRun, options?: CancelTaskRunServiceOptions) { + const opts = { + reason: "Task run was cancelled by user", + cancelAttempts: true, + cancelledAt: new Date(), + ...options, + }; + + // Make sure the task run is in a cancellable state + if (!isCancellableRunStatus(taskRun.status)) { + logger.error("Task run is not in a cancellable state", { + runId: taskRun.id, + status: taskRun.status, + }); + return; + } + + const finalizeService = new FinalizeTaskRunService(); + const cancelledTaskRun = await finalizeService.call({ + id: taskRun.id, + status: "CANCELED", + completedAt: opts.cancelledAt, + include: { + attempts: { + where: { + status: { + in: CANCELLABLE_ATTEMPT_STATUSES, + }, + }, + include: { + backgroundWorker: true, + dependencies: { + include: { + taskRun: true, + }, + }, + batchTaskRunItems: { + include: { + taskRun: true, + }, + }, + }, + }, + runtimeEnvironment: true, + lockedToVersion: true, + }, + attemptStatus: "CANCELED", + error: { + type: "STRING_ERROR", + raw: opts.reason, + }, + }); + + const inProgressEvents = await eventRepository.queryIncompleteEvents({ + runId: taskRun.friendlyId, + }); + + logger.debug("Cancelling in-progress events", { + inProgressEvents: inProgressEvents.map((event) => event.id), + }); + + await Promise.all( + inProgressEvents.map((event) => { + return eventRepository.cancelEvent(event, opts.cancelledAt, opts.reason); + }) + ); + + // Cancel any in progress attempts + if (opts.cancelAttempts) { + await this.#cancelPotentiallyRunningAttempts(cancelledTaskRun, cancelledTaskRun.attempts); + await this.#cancelRemainingRunWorkers(cancelledTaskRun); + } + + return { + id: cancelledTaskRun.id, + }; + } + + async #cancelPotentiallyRunningAttempts( + run: ExtendedTaskRun, + attempts: ExtendedTaskRunAttempt[] + ) { + for (const attempt of attempts) { + await CancelTaskAttemptDependenciesService.enqueue(attempt.id, this._prisma); + + if (run.runtimeEnvironment.type === "DEVELOPMENT") { + // Signal the task run attempt to stop + await devPubSub.publish( + `backgroundWorker:${attempt.backgroundWorkerId}:${attempt.id}`, + "CANCEL_ATTEMPT", + { + attemptId: attempt.friendlyId, + backgroundWorkerId: attempt.backgroundWorker.friendlyId, + taskRunId: run.friendlyId, + } + ); + } else { + switch (attempt.status) { + case "EXECUTING": { + // We need to send a cancel message to the coordinator + socketIo.coordinatorNamespace.emit("REQUEST_ATTEMPT_CANCELLATION", { + version: "v1", + attemptId: attempt.id, + attemptFriendlyId: attempt.friendlyId, + }); + + break; + } + case "PENDING": + case "PAUSED": { + logger.debug("Cancelling pending or paused attempt", { + attempt, + }); + + const service = new CancelAttemptService(); + + await service.call( + attempt.friendlyId, + run.id, + new Date(), + "Task run was cancelled by user" + ); + + break; + } + case "CANCELED": + case "COMPLETED": + case "FAILED": { + // Do nothing + break; + } + default: { + assertNever(attempt.status); + } + } + } + } + } + + async #cancelRemainingRunWorkers(run: ExtendedTaskRun) { + if (run.runtimeEnvironment.type === "DEVELOPMENT") { + // Nothing to do + return; + } + + // Broadcast cancel message to all coordinators + socketIo.coordinatorNamespace.emit("REQUEST_RUN_CANCELLATION", { + version: "v1", + runId: run.id, + // Give the attempts some time to exit gracefully. If the runs supports lazy attempts, it also supports exit delays. + delayInMs: run.lockedToVersion?.supportsLazyAttempts ? 5_000 : undefined, + }); + } +} diff --git a/apps/webapp/app/v3/services/completeAttempt.server.ts b/apps/webapp/app/v3/services/completeAttempt.server.ts index b044a4d291..56adaaa5fd 100644 --- a/apps/webapp/app/v3/services/completeAttempt.server.ts +++ b/apps/webapp/app/v3/services/completeAttempt.server.ts @@ -12,7 +12,7 @@ import { shouldRetryError, taskRunErrorEnhancer, } from "@trigger.dev/core/v3"; -import { $transaction, PrismaClientOrTransaction } from "~/db.server"; +import { PrismaClientOrTransaction } from "~/db.server"; import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { safeJsonParse } from "~/utils/json"; diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index c4fc3a2f83..0627fc83f7 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -16,6 +16,12 @@ import { RegisterNextTaskScheduleInstanceService } from "./registerNextTaskSched import cronstrue from "cronstrue"; import { CheckScheduleService } from "./checkSchedule.server"; import { clampMaxDuration } from "../utils/maxDuration"; +import { + removeQueueConcurrencyLimits, + updateEnvConcurrencyLimits, + updateQueueConcurrencyLimits, +} from "../runQueue.server"; +import { BackgroundWorkerId } from "@trigger.dev/core/v3/apps"; export class CreateBackgroundWorkerService extends BaseService { public async call( @@ -63,7 +69,7 @@ export class CreateBackgroundWorkerService extends BaseService { const backgroundWorker = await this._prisma.backgroundWorker.create({ data: { - friendlyId: generateFriendlyId("worker"), + ...BackgroundWorkerId.generate(), version: nextVersion, runtimeEnvironmentId: environment.id, projectId: project.id, @@ -109,7 +115,7 @@ export class CreateBackgroundWorkerService extends BaseService { } ); - await marqs?.updateEnvConcurrencyLimits(environment); + await updateEnvConcurrencyLimits(environment); } catch (err) { logger.error( "Error publishing WORKER_CREATED event or updating global concurrency limits", @@ -211,11 +217,7 @@ export async function createBackgroundTasks( concurrencyLimit, taskidentifier: task.id, }); - await marqs?.updateQueueConcurrencyLimits( - environment, - taskQueue.name, - taskQueue.concurrencyLimit - ); + await updateQueueConcurrencyLimits(environment, taskQueue.name, taskQueue.concurrencyLimit); } else { logger.debug("CreateBackgroundWorkerService: removing concurrency limit", { workerId: worker.id, @@ -226,7 +228,7 @@ export async function createBackgroundTasks( concurrencyLimit, taskidentifier: task.id, }); - await marqs?.removeQueueConcurrencyLimits(environment, taskQueue.name); + await removeQueueConcurrencyLimits(environment, taskQueue.name); } } catch (error) { if (error instanceof Prisma.PrismaClientKnownRequestError) { diff --git a/apps/webapp/app/v3/services/createCheckpoint.server.ts b/apps/webapp/app/v3/services/createCheckpoint.server.ts index 7290424248..f8028fffff 100644 --- a/apps/webapp/app/v3/services/createCheckpoint.server.ts +++ b/apps/webapp/app/v3/services/createCheckpoint.server.ts @@ -3,12 +3,12 @@ import type { InferSocketMessageSchema } from "@trigger.dev/core/v3/zodSocket"; import type { Checkpoint, CheckpointRestoreEvent } from "@trigger.dev/database"; import { logger } from "~/services/logger.server"; import { marqs } from "~/v3/marqs/index.server"; -import { generateFriendlyId } from "../friendlyIdentifiers"; import { isFreezableAttemptStatus, isFreezableRunStatus } from "../taskStatus"; import { BaseService } from "./baseService.server"; import { CreateCheckpointRestoreEventService } from "./createCheckpointRestoreEvent.server"; import { ResumeBatchRunService } from "./resumeBatchRun.server"; import { ResumeDependentParentsService } from "./resumeDependentParents.server"; +import { CheckpointId } from "@trigger.dev/core/v3/apps"; export class CreateCheckpointService extends BaseService { public async call( @@ -98,7 +98,7 @@ export class CreateCheckpointService extends BaseService { const checkpoint = await this._prisma.checkpoint.create({ data: { - friendlyId: generateFriendlyId("checkpoint"), + ...CheckpointId.generate(), runtimeEnvironmentId: attempt.taskRun.runtimeEnvironmentId, projectId: attempt.taskRun.projectId, attemptId: attempt.id, diff --git a/apps/webapp/app/v3/services/createDeployedBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createDeployedBackgroundWorker.server.ts index 85c652a38a..1840a9c1e3 100644 --- a/apps/webapp/app/v3/services/createDeployedBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createDeployedBackgroundWorker.server.ts @@ -1,17 +1,17 @@ import { CreateBackgroundWorkerRequestBody } from "@trigger.dev/core/v3"; import type { BackgroundWorker } from "@trigger.dev/database"; +import { CURRENT_DEPLOYMENT_LABEL } from "@trigger.dev/core/v3/apps"; import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; -import { generateFriendlyId } from "../friendlyIdentifiers"; +import { logger } from "~/services/logger.server"; +import { socketIo } from "../handleSocketIo.server"; +import { updateEnvConcurrencyLimits } from "../runQueue.server"; +import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server"; import { BaseService } from "./baseService.server"; import { createBackgroundTasks, syncDeclarativeSchedules } from "./createBackgroundWorker.server"; -import { CURRENT_DEPLOYMENT_LABEL } from "~/consts"; -import { projectPubSub } from "./projectPubSub.server"; -import { marqs } from "~/v3/marqs/index.server"; -import { logger } from "~/services/logger.server"; import { ExecuteTasksWaitingForDeployService } from "./executeTasksWaitingForDeploy"; -import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server"; +import { projectPubSub } from "./projectPubSub.server"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; -import { socketIo } from "../handleSocketIo.server"; +import { BackgroundWorkerId } from "@trigger.dev/core/v3/apps"; export class CreateDeployedBackgroundWorkerService extends BaseService { public async call( @@ -39,7 +39,7 @@ export class CreateDeployedBackgroundWorkerService extends BaseService { const backgroundWorker = await this._prisma.backgroundWorker.create({ data: { - friendlyId: generateFriendlyId("worker"), + ...BackgroundWorkerId.generate(), version: deployment.version, runtimeEnvironmentId: environment.id, projectId: environment.projectId, @@ -128,7 +128,7 @@ export class CreateDeployedBackgroundWorkerService extends BaseService { type: "deployed", } ); - await marqs?.updateEnvConcurrencyLimits(environment); + await updateEnvConcurrencyLimits(environment); } catch (err) { logger.error("Failed to publish WORKER_CREATED event", { err }); } diff --git a/apps/webapp/app/v3/services/createDeploymentBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createDeploymentBackgroundWorker.server.ts index cea395510c..67e10d9b0f 100644 --- a/apps/webapp/app/v3/services/createDeploymentBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createDeploymentBackgroundWorker.server.ts @@ -9,7 +9,7 @@ import { syncDeclarativeSchedules, } from "./createBackgroundWorker.server"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; -import { logger } from "~/services/logger.server"; +import { BackgroundWorkerId } from "@trigger.dev/core/v3/apps"; export class CreateDeploymentBackgroundWorkerService extends BaseService { public async call( @@ -36,7 +36,7 @@ export class CreateDeploymentBackgroundWorkerService extends BaseService { const backgroundWorker = await this._prisma.backgroundWorker.create({ data: { - friendlyId: generateFriendlyId("worker"), + ...BackgroundWorkerId.generate(), version: deployment.version, runtimeEnvironmentId: environment.id, projectId: environment.projectId, @@ -97,7 +97,7 @@ export class CreateDeploymentBackgroundWorkerService extends BaseService { data: { status: "DEPLOYING", workerId: backgroundWorker.id, - deployedAt: new Date(), + builtAt: new Date(), }, }); diff --git a/apps/webapp/app/v3/services/finalizeDeployment.server.ts b/apps/webapp/app/v3/services/finalizeDeployment.server.ts index c610f91225..baeca927b2 100644 --- a/apps/webapp/app/v3/services/finalizeDeployment.server.ts +++ b/apps/webapp/app/v3/services/finalizeDeployment.server.ts @@ -1,10 +1,10 @@ import { FinalizeDeploymentRequestBody } from "@trigger.dev/core/v3/schemas"; -import { CURRENT_DEPLOYMENT_LABEL } from "~/consts"; +import { CURRENT_DEPLOYMENT_LABEL } from "@trigger.dev/core/v3/apps"; import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { socketIo } from "../handleSocketIo.server"; -import { marqs } from "../marqs/index.server"; import { registryProxy } from "../registryProxy.server"; +import { updateEnvConcurrencyLimits } from "../runQueue.server"; import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server"; import { BaseService, ServiceValidationError } from "./baseService.server"; import { ExecuteTasksWaitingForDeployService } from "./executeTasksWaitingForDeploy"; @@ -95,7 +95,7 @@ export class FinalizeDeploymentService extends BaseService { } ); - await marqs?.updateEnvConcurrencyLimits(authenticatedEnv); + await updateEnvConcurrencyLimits(authenticatedEnv); } catch (err) { logger.error("Failed to publish WORKER_CREATED event", { err }); } diff --git a/apps/webapp/app/v3/services/initializeDeployment.server.ts b/apps/webapp/app/v3/services/initializeDeployment.server.ts index 12be3ee783..c5a375ba90 100644 --- a/apps/webapp/app/v3/services/initializeDeployment.server.ts +++ b/apps/webapp/app/v3/services/initializeDeployment.server.ts @@ -4,9 +4,11 @@ import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { generateFriendlyId } from "../friendlyIdentifiers"; import { createRemoteImageBuild } from "../remoteImageBuilder.server"; import { calculateNextBuildVersion } from "../utils/calculateNextBuildVersion"; -import { BaseService } from "./baseService.server"; +import { BaseService, ServiceValidationError } from "./baseService.server"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; import { env } from "~/env.server"; +import { WorkerDeploymentType } from "@trigger.dev/database"; +import { logger } from "~/services/logger.server"; const nanoid = customAlphabet("1234567890abcdefghijklmnopqrstuvwxyz", 8); @@ -16,6 +18,10 @@ export class InitializeDeploymentService extends BaseService { payload: InitializeDeploymentRequestBody ) { return this.traceWithEnv("call", environment, async (span) => { + if (payload.type !== "V1" && environment.project.engine !== "V2") { + throw new ServiceValidationError("Only V1 deployments are supported for this project"); + } + const latestDeployment = await this._prisma.workerDeployment.findFirst({ where: { environmentId: environment.id, @@ -46,6 +52,36 @@ export class InitializeDeploymentService extends BaseService { }) : undefined; + const sharedImageTag = `${payload.namespace ?? env.DEPLOY_REGISTRY_NAMESPACE}/${ + environment.project.externalRef + }:${nextVersion}.${environment.slug}`; + + const unmanagedImageParts = []; + + if (payload.registryHost) { + unmanagedImageParts.push(payload.registryHost); + } + if (payload.namespace) { + unmanagedImageParts.push(payload.namespace); + } + unmanagedImageParts.push( + `${environment.project.externalRef}:${nextVersion}.${environment.slug}` + ); + + const unmanagedImageTag = unmanagedImageParts.join("/"); + + const isManaged = payload.type === WorkerDeploymentType.MANAGED; + + logger.debug("Creating deployment", { + environmentId: environment.id, + projectId: environment.projectId, + version: nextVersion, + triggeredById: triggeredBy?.id, + type: payload.type, + imageTag: isManaged ? sharedImageTag : unmanagedImageTag, + imageReference: isManaged ? undefined : unmanagedImageTag, + }); + const deployment = await this._prisma.workerDeployment.create({ data: { friendlyId: generateFriendlyId("deployment"), @@ -57,6 +93,8 @@ export class InitializeDeploymentService extends BaseService { projectId: environment.projectId, externalBuildData, triggeredById: triggeredBy?.id, + type: payload.type, + imageReference: isManaged ? undefined : unmanagedImageTag, }, }); @@ -67,11 +105,10 @@ export class InitializeDeploymentService extends BaseService { new Date(Date.now() + env.DEPLOY_TIMEOUT_MS) ); - const imageTag = `${payload.namespace ?? env.DEPLOY_REGISTRY_NAMESPACE}/${ - environment.project.externalRef - }:${deployment.version}.${environment.slug}`; - - return { deployment, imageTag }; + return { + deployment, + imageTag: isManaged ? sharedImageTag : unmanagedImageTag, + }; }); } } diff --git a/apps/webapp/app/v3/services/replayTaskRun.server.ts b/apps/webapp/app/v3/services/replayTaskRun.server.ts index 601bb8a075..96415a270d 100644 --- a/apps/webapp/app/v3/services/replayTaskRun.server.ts +++ b/apps/webapp/app/v3/services/replayTaskRun.server.ts @@ -119,7 +119,9 @@ export class ReplayTaskRunService extends BaseService { return; } - logger.error("Failed to replay a run", { error: error }); + logger.error("Failed to replay a run", { + error: error instanceof Error ? error.message : error, + }); return; } diff --git a/apps/webapp/app/v3/services/rescheduleTaskRun.server.ts b/apps/webapp/app/v3/services/rescheduleTaskRun.server.ts index 4d9461d06b..e764a6c459 100644 --- a/apps/webapp/app/v3/services/rescheduleTaskRun.server.ts +++ b/apps/webapp/app/v3/services/rescheduleTaskRun.server.ts @@ -1,9 +1,9 @@ import { TaskRun } from "@trigger.dev/database"; import { BaseService, ServiceValidationError } from "./baseService.server"; import { RescheduleRunRequestBody } from "@trigger.dev/core/v3"; -import { parseDelay } from "./triggerTask.server"; import { $transaction } from "~/db.server"; import { workerQueue } from "~/services/worker.server"; +import { parseDelay } from "~/utils/delays"; export class RescheduleTaskRunService extends BaseService { public async call(taskRun: TaskRun, body: RescheduleRunRequestBody) { diff --git a/apps/webapp/app/v3/services/rollbackDeployment.server.ts b/apps/webapp/app/v3/services/rollbackDeployment.server.ts index 24f25e69cd..128797cb06 100644 --- a/apps/webapp/app/v3/services/rollbackDeployment.server.ts +++ b/apps/webapp/app/v3/services/rollbackDeployment.server.ts @@ -1,7 +1,7 @@ import { logger } from "~/services/logger.server"; import { BaseService } from "./baseService.server"; -import { WorkerDeployment } from "@trigger.dev/database"; -import { CURRENT_DEPLOYMENT_LABEL } from "~/consts"; +import { WorkerDeployment, WorkerInstanceGroupType } from "@trigger.dev/database"; +import { CURRENT_DEPLOYMENT_LABEL } from "@trigger.dev/core/v3/apps"; import { ExecuteTasksWaitingForDeployService } from "./executeTasksWaitingForDeploy"; export class RollbackDeploymentService extends BaseService { @@ -11,6 +11,14 @@ export class RollbackDeploymentService extends BaseService { return; } + if (deployment.type !== WorkerInstanceGroupType.MANAGED) { + logger.error("Can only roll back managed deployments", { + id: deployment.id, + type: deployment.type, + }); + return; + } + const promotion = await this._prisma.workerDeploymentPromotion.findFirst({ where: { deploymentId: deployment.id, diff --git a/apps/webapp/app/v3/services/triggerTask.server.ts b/apps/webapp/app/v3/services/triggerTask.server.ts index b3eb7b55dc..55ed259b8d 100644 --- a/apps/webapp/app/v3/services/triggerTask.server.ts +++ b/apps/webapp/app/v3/services/triggerTask.server.ts @@ -1,32 +1,10 @@ -import { - IOPacket, - QueueOptions, - SemanticInternalAttributes, - TriggerTaskRequestBody, - packetRequiresOffloading, -} from "@trigger.dev/core/v3"; -import { env } from "~/env.server"; +import { TriggerTaskRequestBody } from "@trigger.dev/core/v3"; import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; -import { autoIncrementCounter } from "~/services/autoIncrementCounter.server"; -import { workerQueue } from "~/services/worker.server"; -import { marqs, sanitizeQueueName } from "~/v3/marqs/index.server"; -import { eventRepository } from "../eventRepository.server"; -import { generateFriendlyId } from "../friendlyIdentifiers"; -import { uploadPacketToObjectStore } from "../r2.server"; -import { startActiveSpan } from "../tracer.server"; -import { getEntitlement } from "~/services/platform.v3.server"; -import { BaseService, ServiceValidationError } from "./baseService.server"; -import { logger } from "~/services/logger.server"; -import { isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus"; -import { createTag, MAX_TAGS_PER_RUN } from "~/models/taskRunTag.server"; -import { findCurrentWorkerFromEnvironment } from "../models/workerDeployment.server"; -import { handleMetadataPacket } from "~/utils/packets"; -import { parseNaturalLanguageDuration } from "@trigger.dev/core/v3/apps"; -import { ExpireEnqueuedRunService } from "./expireEnqueuedRun.server"; -import { guardQueueSizeLimitsForEnv } from "../queueSizeLimits.server"; -import { clampMaxDuration } from "../utils/maxDuration"; -import { resolveIdempotencyKeyTTL } from "~/utils/idempotencyKeys.server"; -import { Prisma } from "@trigger.dev/database"; +import { WithRunEngine } from "./baseService.server"; +import { RunEngineVersion, RuntimeEnvironmentType } from "@trigger.dev/database"; +import { TriggerTaskServiceV1 } from "./triggerTaskV1.server"; +import { TriggerTaskServiceV2 } from "./triggerTaskV2.server"; +import { determineEngineVersion } from "../engineVersion.server"; export type TriggerTaskServiceOptions = { idempotencyKey?: string; @@ -37,7 +15,7 @@ export type TriggerTaskServiceOptions = { parentAsLinkType?: "replay" | "trigger"; batchId?: string; customIcon?: string; - runId?: string; + runFriendlyId?: string; skipChecks?: boolean; oneTimeUseToken?: string; }; @@ -48,734 +26,55 @@ export class OutOfEntitlementError extends Error { } } -export class TriggerTaskService extends BaseService { +export class TriggerTaskService extends WithRunEngine { public async call( taskId: string, environment: AuthenticatedEnvironment, body: TriggerTaskRequestBody, - options: TriggerTaskServiceOptions = {} + options: TriggerTaskServiceOptions = {}, + version?: RunEngineVersion ) { return await this.traceWithEnv("call()", environment, async (span) => { span.setAttribute("taskId", taskId); - // TODO: Add idempotency key expiring here - const idempotencyKey = options.idempotencyKey ?? body.options?.idempotencyKey; - const idempotencyKeyExpiresAt = - options.idempotencyKeyExpiresAt ?? - resolveIdempotencyKeyTTL(body.options?.idempotencyKeyTTL) ?? - new Date(Date.now() + 24 * 60 * 60 * 1000 * 30); // 30 days - - const delayUntil = await parseDelay(body.options?.delay); - - const ttl = - typeof body.options?.ttl === "number" - ? stringifyDuration(body.options?.ttl) - : body.options?.ttl ?? (environment.type === "DEVELOPMENT" ? "10m" : undefined); - - const existingRun = idempotencyKey - ? await this._prisma.taskRun.findUnique({ - where: { - runtimeEnvironmentId_taskIdentifier_idempotencyKey: { - runtimeEnvironmentId: environment.id, - idempotencyKey, - taskIdentifier: taskId, - }, - }, - }) - : undefined; - - if (existingRun) { - if ( - existingRun.idempotencyKeyExpiresAt && - existingRun.idempotencyKeyExpiresAt < new Date() - ) { - logger.debug("[TriggerTaskService][call] Idempotency key has expired", { - idempotencyKey: options.idempotencyKey, - run: existingRun, - }); - - // Update the existing batch to remove the idempotency key - await this._prisma.taskRun.update({ - where: { id: existingRun.id }, - data: { idempotencyKey: null }, - }); - } else { - span.setAttribute("runId", existingRun.friendlyId); - - return existingRun; - } - } - - if (environment.type !== "DEVELOPMENT" && !options.skipChecks) { - const result = await getEntitlement(environment.organizationId); - if (result && result.hasAccess === false) { - throw new OutOfEntitlementError(); - } - } - - if (!options.skipChecks) { - const queueSizeGuard = await guardQueueSizeLimitsForEnv(environment, marqs); - - logger.debug("Queue size guard result", { - queueSizeGuard, - environment: { - id: environment.id, - type: environment.type, - organization: environment.organization, - project: environment.project, - }, - }); + const v = await determineEngineVersion({ environment, version }); - if (!queueSizeGuard.isWithinLimits) { - throw new ServiceValidationError( - `Cannot trigger ${taskId} as the queue size limit for this environment has been reached. The maximum size is ${queueSizeGuard.maximumSize}` - ); + switch (v) { + case "V1": { + return await this.callV1(taskId, environment, body, options); } - } - - if ( - body.options?.tags && - typeof body.options.tags !== "string" && - body.options.tags.length > MAX_TAGS_PER_RUN - ) { - throw new ServiceValidationError( - `Runs can only have ${MAX_TAGS_PER_RUN} tags, you're trying to set ${body.options.tags.length}.` - ); - } - - const runFriendlyId = options?.runId ?? generateFriendlyId("run"); - - const payloadPacket = await this.#handlePayloadPacket( - body.payload, - body.options?.payloadType ?? "application/json", - runFriendlyId, - environment - ); - - const metadataPacket = body.options?.metadata - ? handleMetadataPacket( - body.options?.metadata, - body.options?.metadataType ?? "application/json" - ) - : undefined; - - const dependentAttempt = body.options?.dependentAttempt - ? await this._prisma.taskRunAttempt.findUnique({ - where: { friendlyId: body.options.dependentAttempt }, - include: { - taskRun: { - select: { - id: true, - status: true, - taskIdentifier: true, - rootTaskRunId: true, - depth: true, - }, - }, - }, - }) - : undefined; - - if ( - dependentAttempt && - (isFinalAttemptStatus(dependentAttempt.status) || - isFinalRunStatus(dependentAttempt.taskRun.status)) - ) { - logger.debug("Dependent attempt or run is in a terminal state", { - dependentAttempt: dependentAttempt, - }); - - if (isFinalAttemptStatus(dependentAttempt.status)) { - throw new ServiceValidationError( - `Cannot trigger ${taskId} as the parent attempt has a status of ${dependentAttempt.status}` - ); - } else { - throw new ServiceValidationError( - `Cannot trigger ${taskId} as the parent run has a status of ${dependentAttempt.taskRun.status}` - ); - } - } - - const parentAttempt = body.options?.parentAttempt - ? await this._prisma.taskRunAttempt.findUnique({ - where: { friendlyId: body.options.parentAttempt }, - include: { - taskRun: { - select: { - id: true, - status: true, - taskIdentifier: true, - rootTaskRunId: true, - depth: true, - }, - }, - }, - }) - : undefined; - - const dependentBatchRun = body.options?.dependentBatch - ? await this._prisma.batchTaskRun.findUnique({ - where: { friendlyId: body.options.dependentBatch }, - include: { - dependentTaskAttempt: { - include: { - taskRun: { - select: { - id: true, - status: true, - taskIdentifier: true, - rootTaskRunId: true, - depth: true, - }, - }, - }, - }, - }, - }) - : undefined; - - if ( - dependentBatchRun && - dependentBatchRun.dependentTaskAttempt && - (isFinalAttemptStatus(dependentBatchRun.dependentTaskAttempt.status) || - isFinalRunStatus(dependentBatchRun.dependentTaskAttempt.taskRun.status)) - ) { - logger.debug("Dependent batch run task attempt or run has been canceled", { - dependentBatchRunId: dependentBatchRun.id, - status: dependentBatchRun.status, - attempt: dependentBatchRun.dependentTaskAttempt, - }); - - if (isFinalAttemptStatus(dependentBatchRun.dependentTaskAttempt.status)) { - throw new ServiceValidationError( - `Cannot trigger ${taskId} as the parent attempt has a status of ${dependentBatchRun.dependentTaskAttempt.status}` - ); - } else { - throw new ServiceValidationError( - `Cannot trigger ${taskId} as the parent run has a status of ${dependentBatchRun.dependentTaskAttempt.taskRun.status}` - ); - } - } - - const parentBatchRun = body.options?.parentBatch - ? await this._prisma.batchTaskRun.findUnique({ - where: { friendlyId: body.options.parentBatch }, - include: { - dependentTaskAttempt: { - include: { - taskRun: { - select: { - id: true, - status: true, - taskIdentifier: true, - rootTaskRunId: true, - }, - }, - }, - }, - }, - }) - : undefined; - - try { - return await eventRepository.traceEvent( - taskId, - { - context: options.traceContext, - spanParentAsLink: options.spanParentAsLink, - parentAsLinkType: options.parentAsLinkType, - kind: "SERVER", - environment, - taskSlug: taskId, - attributes: { - properties: { - [SemanticInternalAttributes.SHOW_ACTIONS]: true, - }, - style: { - icon: options.customIcon ?? "task", - }, - runIsTest: body.options?.test ?? false, - batchId: options.batchId, - idempotencyKey, - }, - incomplete: true, - immediate: true, - }, - async (event, traceContext, traceparent) => { - const run = await autoIncrementCounter.incrementInTransaction( - `v3-run:${environment.id}:${taskId}`, - async (num, tx) => { - const lockedToBackgroundWorker = body.options?.lockToVersion - ? await tx.backgroundWorker.findUnique({ - where: { - projectId_runtimeEnvironmentId_version: { - projectId: environment.projectId, - runtimeEnvironmentId: environment.id, - version: body.options?.lockToVersion, - }, - }, - }) - : undefined; - - let queueName = sanitizeQueueName( - await this.#getQueueName(taskId, environment, body.options?.queue?.name) - ); - - // Check that the queuename is not an empty string - if (!queueName) { - queueName = sanitizeQueueName(`task/${taskId}`); - } - - event.setAttribute("queueName", queueName); - span.setAttribute("queueName", queueName); - - //upsert tags - let tagIds: string[] = []; - const bodyTags = - typeof body.options?.tags === "string" ? [body.options.tags] : body.options?.tags; - if (bodyTags && bodyTags.length > 0) { - for (const tag of bodyTags) { - const tagRecord = await createTag({ - tag, - projectId: environment.projectId, - }); - if (tagRecord) { - tagIds.push(tagRecord.id); - } - } - } - - const depth = dependentAttempt - ? dependentAttempt.taskRun.depth + 1 - : parentAttempt - ? parentAttempt.taskRun.depth + 1 - : dependentBatchRun?.dependentTaskAttempt - ? dependentBatchRun.dependentTaskAttempt.taskRun.depth + 1 - : 0; - - const taskRun = await tx.taskRun.create({ - data: { - status: delayUntil ? "DELAYED" : "PENDING", - number: num, - friendlyId: runFriendlyId, - runtimeEnvironmentId: environment.id, - projectId: environment.projectId, - idempotencyKey, - idempotencyKeyExpiresAt: idempotencyKey ? idempotencyKeyExpiresAt : undefined, - taskIdentifier: taskId, - payload: payloadPacket.data ?? "", - payloadType: payloadPacket.dataType, - context: body.context, - traceContext: traceContext, - traceId: event.traceId, - spanId: event.spanId, - parentSpanId: - options.parentAsLinkType === "replay" ? undefined : traceparent?.spanId, - lockedToVersionId: lockedToBackgroundWorker?.id, - taskVersion: lockedToBackgroundWorker?.version, - sdkVersion: lockedToBackgroundWorker?.sdkVersion, - cliVersion: lockedToBackgroundWorker?.cliVersion, - concurrencyKey: body.options?.concurrencyKey, - queue: queueName, - isTest: body.options?.test ?? false, - delayUntil, - queuedAt: delayUntil ? undefined : new Date(), - maxAttempts: body.options?.maxAttempts, - ttl, - tags: - tagIds.length === 0 - ? undefined - : { - connect: tagIds.map((id) => ({ id })), - }, - parentTaskRunId: - dependentAttempt?.taskRun.id ?? - parentAttempt?.taskRun.id ?? - dependentBatchRun?.dependentTaskAttempt?.taskRun.id, - parentTaskRunAttemptId: - dependentAttempt?.id ?? - parentAttempt?.id ?? - dependentBatchRun?.dependentTaskAttempt?.id, - rootTaskRunId: - dependentAttempt?.taskRun.rootTaskRunId ?? - dependentAttempt?.taskRun.id ?? - parentAttempt?.taskRun.rootTaskRunId ?? - parentAttempt?.taskRun.id ?? - dependentBatchRun?.dependentTaskAttempt?.taskRun.rootTaskRunId ?? - dependentBatchRun?.dependentTaskAttempt?.taskRun.id, - batchId: dependentBatchRun?.id ?? parentBatchRun?.id, - resumeParentOnCompletion: !!(dependentAttempt ?? dependentBatchRun), - depth, - metadata: metadataPacket?.data, - metadataType: metadataPacket?.dataType, - seedMetadata: metadataPacket?.data, - seedMetadataType: metadataPacket?.dataType, - maxDurationInSeconds: body.options?.maxDuration - ? clampMaxDuration(body.options.maxDuration) - : undefined, - runTags: bodyTags, - oneTimeUseToken: options.oneTimeUseToken, - }, - }); - - event.setAttribute("runId", taskRun.friendlyId); - span.setAttribute("runId", taskRun.friendlyId); - - if (dependentAttempt) { - await tx.taskRunDependency.create({ - data: { - taskRunId: taskRun.id, - dependentAttemptId: dependentAttempt.id, - }, - }); - } else if (dependentBatchRun) { - await tx.taskRunDependency.create({ - data: { - taskRunId: taskRun.id, - dependentBatchRunId: dependentBatchRun.id, - }, - }); - } - - if (body.options?.queue) { - const concurrencyLimit = - typeof body.options.queue?.concurrencyLimit === "number" - ? Math.max( - Math.min( - body.options.queue.concurrencyLimit, - environment.maximumConcurrencyLimit, - environment.organization.maximumConcurrencyLimit - ), - 0 - ) - : null; - - let taskQueue = await tx.taskQueue.findFirst({ - where: { - runtimeEnvironmentId: environment.id, - name: queueName, - }, - }); - - const existingConcurrencyLimit = - typeof taskQueue?.concurrencyLimit === "number" - ? taskQueue.concurrencyLimit - : undefined; - - if (taskQueue) { - if (existingConcurrencyLimit !== concurrencyLimit) { - taskQueue = await tx.taskQueue.update({ - where: { - id: taskQueue.id, - }, - data: { - concurrencyLimit: - typeof concurrencyLimit === "number" ? concurrencyLimit : null, - }, - }); - - if (typeof taskQueue.concurrencyLimit === "number") { - logger.debug("TriggerTaskService: updating concurrency limit", { - runId: taskRun.id, - friendlyId: taskRun.friendlyId, - taskQueue, - orgId: environment.organizationId, - projectId: environment.projectId, - existingConcurrencyLimit, - concurrencyLimit, - queueOptions: body.options?.queue, - }); - await marqs?.updateQueueConcurrencyLimits( - environment, - taskQueue.name, - taskQueue.concurrencyLimit - ); - } else { - logger.debug("TriggerTaskService: removing concurrency limit", { - runId: taskRun.id, - friendlyId: taskRun.friendlyId, - taskQueue, - orgId: environment.organizationId, - projectId: environment.projectId, - existingConcurrencyLimit, - concurrencyLimit, - queueOptions: body.options?.queue, - }); - await marqs?.removeQueueConcurrencyLimits(environment, taskQueue.name); - } - } - } else { - const queueId = generateFriendlyId("queue"); - - taskQueue = await tx.taskQueue.create({ - data: { - friendlyId: queueId, - name: queueName, - concurrencyLimit, - runtimeEnvironmentId: environment.id, - projectId: environment.projectId, - type: "NAMED", - }, - }); - - if (typeof taskQueue.concurrencyLimit === "number") { - await marqs?.updateQueueConcurrencyLimits( - environment, - taskQueue.name, - taskQueue.concurrencyLimit - ); - } - } - } - - if (taskRun.delayUntil) { - await workerQueue.enqueue( - "v3.enqueueDelayedRun", - { runId: taskRun.id }, - { tx, runAt: delayUntil, jobKey: `v3.enqueueDelayedRun.${taskRun.id}` } - ); - } - - if (!taskRun.delayUntil && taskRun.ttl) { - const expireAt = parseNaturalLanguageDuration(taskRun.ttl); - - if (expireAt) { - await ExpireEnqueuedRunService.enqueue(taskRun.id, expireAt, tx); - } - } - - return taskRun; - }, - async (_, tx) => { - const counter = await tx.taskRunNumberCounter.findUnique({ - where: { - taskIdentifier_environmentId: { - taskIdentifier: taskId, - environmentId: environment.id, - }, - }, - select: { lastNumber: true }, - }); - - return counter?.lastNumber; - }, - this._prisma - ); - - //release the concurrency for the env and org, if part of a (batch)triggerAndWait - if (dependentAttempt) { - const isSameTask = dependentAttempt.taskRun.taskIdentifier === taskId; - await marqs?.releaseConcurrency(dependentAttempt.taskRun.id, isSameTask); - } - if (dependentBatchRun?.dependentTaskAttempt) { - const isSameTask = - dependentBatchRun.dependentTaskAttempt.taskRun.taskIdentifier === taskId; - await marqs?.releaseConcurrency( - dependentBatchRun.dependentTaskAttempt.taskRun.id, - isSameTask - ); - } - - if (!run) { - return; - } - - // We need to enqueue the task run into the appropriate queue. This is done after the tx completes to prevent a race condition where the task run hasn't been created yet by the time we dequeue. - if (run.status === "PENDING") { - await marqs?.enqueueMessage( - environment, - run.queue, - run.id, - { - type: "EXECUTE", - taskIdentifier: taskId, - projectId: environment.projectId, - environmentId: environment.id, - environmentType: environment.type, - }, - body.options?.concurrencyKey - ); - } - - return run; - } - ); - } catch (error) { - // Detect a prisma transaction Unique constraint violation - if (error instanceof Prisma.PrismaClientKnownRequestError) { - logger.debug("TriggerTask: Prisma transaction error", { - code: error.code, - message: error.message, - meta: error.meta, - }); - - if (error.code === "P2002") { - const target = error.meta?.target; - - if ( - Array.isArray(target) && - target.length > 0 && - typeof target[0] === "string" && - target[0].includes("oneTimeUseToken") - ) { - throw new ServiceValidationError( - `Cannot trigger ${taskId} with a one-time use token as it has already been used.` - ); - } else { - throw new ServiceValidationError( - `Cannot trigger ${taskId} as it has already been triggered with the same idempotency key.` - ); - } - } + case "V2": { + return await this.callV2(taskId, environment, body, options); } - - throw error; } }); } - async #getQueueName(taskId: string, environment: AuthenticatedEnvironment, queueName?: string) { - if (queueName) { - return queueName; - } - - const defaultQueueName = `task/${taskId}`; - - const worker = await findCurrentWorkerFromEnvironment(environment); - - if (!worker) { - logger.debug("Failed to get queue name: No worker found", { - taskId, - environmentId: environment.id, - }); - - return defaultQueueName; - } - - const task = await this._prisma.backgroundWorkerTask.findUnique({ - where: { - workerId_slug: { - workerId: worker.id, - slug: taskId, - }, - }, - }); - - if (!task) { - console.log("Failed to get queue name: No task found", { - taskId, - environmentId: environment.id, - }); - - return defaultQueueName; - } - - const queueConfig = QueueOptions.optional().nullable().safeParse(task.queueConfig); - - if (!queueConfig.success) { - console.log("Failed to get queue name: Invalid queue config", { - taskId, - environmentId: environment.id, - queueConfig: task.queueConfig, - }); - - return defaultQueueName; - } - - return queueConfig.data?.name ?? defaultQueueName; + private async callV1( + taskId: string, + environment: AuthenticatedEnvironment, + body: TriggerTaskRequestBody, + options: TriggerTaskServiceOptions = {} + ) { + const service = new TriggerTaskServiceV1(this._prisma); + return await service.call(taskId, environment, body, options); } - async #handlePayloadPacket( - payload: any, - payloadType: string, - pathPrefix: string, - environment: AuthenticatedEnvironment + private async callV2( + taskId: string, + environment: AuthenticatedEnvironment, + body: TriggerTaskRequestBody, + options: TriggerTaskServiceOptions = {} ) { - return await startActiveSpan("handlePayloadPacket()", async (span) => { - const packet = this.#createPayloadPacket(payload, payloadType); - - if (!packet.data) { - return packet; - } - - const { needsOffloading, size } = packetRequiresOffloading( - packet, - env.TASK_PAYLOAD_OFFLOAD_THRESHOLD - ); - - if (!needsOffloading) { - return packet; - } - - const filename = `${pathPrefix}/payload.json`; - - await uploadPacketToObjectStore(filename, packet.data, packet.dataType, environment); - - return { - data: filename, - dataType: "application/store", - }; + const service = new TriggerTaskServiceV2({ + prisma: this._prisma, + engine: this._engine, + }); + return await service.call({ + taskId, + environment, + body, + options, }); } - - #createPayloadPacket(payload: any, payloadType: string): IOPacket { - if (payloadType === "application/json") { - return { data: JSON.stringify(payload), dataType: "application/json" }; - } - - if (typeof payload === "string") { - return { data: payload, dataType: payloadType }; - } - - return { dataType: payloadType }; - } -} - -export async function parseDelay(value?: string | Date): Promise { - if (!value) { - return; - } - - if (value instanceof Date) { - return value; - } - - try { - const date = new Date(value); - - // Check if the date is valid - if (isNaN(date.getTime())) { - return parseNaturalLanguageDuration(value); - } - - if (date.getTime() <= Date.now()) { - return; - } - - return date; - } catch (error) { - return parseNaturalLanguageDuration(value); - } -} - -function stringifyDuration(seconds: number): string | undefined { - if (seconds <= 0) { - return; - } - - const units = { - w: Math.floor(seconds / 604800), - d: Math.floor((seconds % 604800) / 86400), - h: Math.floor((seconds % 86400) / 3600), - m: Math.floor((seconds % 3600) / 60), - s: Math.floor(seconds % 60), - }; - - // Filter the units having non-zero values and join them - const result: string = Object.entries(units) - .filter(([unit, val]) => val != 0) - .map(([unit, val]) => `${val}${unit}`) - .join(""); - - return result; } diff --git a/apps/webapp/app/v3/services/triggerTaskV1.server.ts b/apps/webapp/app/v3/services/triggerTaskV1.server.ts new file mode 100644 index 0000000000..51c4e4cffc --- /dev/null +++ b/apps/webapp/app/v3/services/triggerTaskV1.server.ts @@ -0,0 +1,716 @@ +import { + IOPacket, + QueueOptions, + SemanticInternalAttributes, + TriggerTaskRequestBody, + packetRequiresOffloading, +} from "@trigger.dev/core/v3"; +import { env } from "~/env.server"; +import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { autoIncrementCounter } from "~/services/autoIncrementCounter.server"; +import { workerQueue } from "~/services/worker.server"; +import { marqs, sanitizeQueueName } from "~/v3/marqs/index.server"; +import { eventRepository } from "../eventRepository.server"; +import { generateFriendlyId } from "../friendlyIdentifiers"; +import { uploadPacketToObjectStore } from "../r2.server"; +import { startActiveSpan } from "../tracer.server"; +import { getEntitlement } from "~/services/platform.v3.server"; +import { BaseService, ServiceValidationError } from "./baseService.server"; +import { logger } from "~/services/logger.server"; +import { isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus"; +import { createTag, MAX_TAGS_PER_RUN } from "~/models/taskRunTag.server"; +import { findCurrentWorkerFromEnvironment } from "../models/workerDeployment.server"; +import { handleMetadataPacket } from "~/utils/packets"; +import { parseNaturalLanguageDuration, stringifyDuration } from "@trigger.dev/core/v3/apps"; +import { ExpireEnqueuedRunService } from "./expireEnqueuedRun.server"; +import { guardQueueSizeLimitsForEnv } from "../queueSizeLimits.server"; +import { clampMaxDuration } from "../utils/maxDuration"; +import { resolveIdempotencyKeyTTL } from "~/utils/idempotencyKeys.server"; +import { Prisma } from "@trigger.dev/database"; +import { parseDelay } from "~/utils/delays"; +import { OutOfEntitlementError, TriggerTaskServiceOptions } from "./triggerTask.server"; +import { removeQueueConcurrencyLimits, updateQueueConcurrencyLimits } from "../runQueue.server"; + +/** @deprecated Use TriggerTaskService in `triggerTask.server.ts` instead. */ +export class TriggerTaskServiceV1 extends BaseService { + public async call( + taskId: string, + environment: AuthenticatedEnvironment, + body: TriggerTaskRequestBody, + options: TriggerTaskServiceOptions = {} + ) { + return await this.traceWithEnv("call()", environment, async (span) => { + span.setAttribute("taskId", taskId); + + // TODO: Add idempotency key expiring here + const idempotencyKey = options.idempotencyKey ?? body.options?.idempotencyKey; + const idempotencyKeyExpiresAt = + options.idempotencyKeyExpiresAt ?? + resolveIdempotencyKeyTTL(body.options?.idempotencyKeyTTL) ?? + new Date(Date.now() + 24 * 60 * 60 * 1000 * 30); // 30 days + + const delayUntil = await parseDelay(body.options?.delay); + + const ttl = + typeof body.options?.ttl === "number" + ? stringifyDuration(body.options?.ttl) + : body.options?.ttl ?? (environment.type === "DEVELOPMENT" ? "10m" : undefined); + + const existingRun = idempotencyKey + ? await this._prisma.taskRun.findUnique({ + where: { + runtimeEnvironmentId_taskIdentifier_idempotencyKey: { + runtimeEnvironmentId: environment.id, + idempotencyKey, + taskIdentifier: taskId, + }, + }, + }) + : undefined; + + if (existingRun) { + if ( + existingRun.idempotencyKeyExpiresAt && + existingRun.idempotencyKeyExpiresAt < new Date() + ) { + logger.debug("[TriggerTaskService][call] Idempotency key has expired", { + idempotencyKey: options.idempotencyKey, + run: existingRun, + }); + + // Update the existing batch to remove the idempotency key + await this._prisma.taskRun.update({ + where: { id: existingRun.id }, + data: { idempotencyKey: null }, + }); + } else { + span.setAttribute("runId", existingRun.friendlyId); + + return existingRun; + } + } + + if (environment.type !== "DEVELOPMENT" && !options.skipChecks) { + const result = await getEntitlement(environment.organizationId); + if (result && result.hasAccess === false) { + throw new OutOfEntitlementError(); + } + } + + if (!options.skipChecks) { + const queueSizeGuard = await guardQueueSizeLimitsForEnv(environment, marqs); + + logger.debug("Queue size guard result", { + queueSizeGuard, + environment: { + id: environment.id, + type: environment.type, + organization: environment.organization, + project: environment.project, + }, + }); + + if (!queueSizeGuard.isWithinLimits) { + throw new ServiceValidationError( + `Cannot trigger ${taskId} as the queue size limit for this environment has been reached. The maximum size is ${queueSizeGuard.maximumSize}` + ); + } + } + + if ( + body.options?.tags && + typeof body.options.tags !== "string" && + body.options.tags.length > MAX_TAGS_PER_RUN + ) { + throw new ServiceValidationError( + `Runs can only have ${MAX_TAGS_PER_RUN} tags, you're trying to set ${body.options.tags.length}.` + ); + } + + const runFriendlyId = options?.runFriendlyId ?? generateFriendlyId("run"); + + const payloadPacket = await this.#handlePayloadPacket( + body.payload, + body.options?.payloadType ?? "application/json", + runFriendlyId, + environment + ); + + const metadataPacket = body.options?.metadata + ? handleMetadataPacket( + body.options?.metadata, + body.options?.metadataType ?? "application/json" + ) + : undefined; + + const dependentAttempt = body.options?.dependentAttempt + ? await this._prisma.taskRunAttempt.findUnique({ + where: { friendlyId: body.options.dependentAttempt }, + include: { + taskRun: { + select: { + id: true, + status: true, + taskIdentifier: true, + rootTaskRunId: true, + depth: true, + }, + }, + }, + }) + : undefined; + + if ( + dependentAttempt && + (isFinalAttemptStatus(dependentAttempt.status) || + isFinalRunStatus(dependentAttempt.taskRun.status)) + ) { + logger.debug("Dependent attempt or run is in a terminal state", { + dependentAttempt: dependentAttempt, + }); + + if (isFinalAttemptStatus(dependentAttempt.status)) { + throw new ServiceValidationError( + `Cannot trigger ${taskId} as the parent attempt has a status of ${dependentAttempt.status}` + ); + } else { + throw new ServiceValidationError( + `Cannot trigger ${taskId} as the parent run has a status of ${dependentAttempt.taskRun.status}` + ); + } + } + + const parentAttempt = body.options?.parentAttempt + ? await this._prisma.taskRunAttempt.findUnique({ + where: { friendlyId: body.options.parentAttempt }, + include: { + taskRun: { + select: { + id: true, + status: true, + taskIdentifier: true, + rootTaskRunId: true, + depth: true, + }, + }, + }, + }) + : undefined; + + const dependentBatchRun = body.options?.dependentBatch + ? await this._prisma.batchTaskRun.findUnique({ + where: { friendlyId: body.options.dependentBatch }, + include: { + dependentTaskAttempt: { + include: { + taskRun: { + select: { + id: true, + status: true, + taskIdentifier: true, + rootTaskRunId: true, + depth: true, + }, + }, + }, + }, + }, + }) + : undefined; + + if ( + dependentBatchRun && + dependentBatchRun.dependentTaskAttempt && + (isFinalAttemptStatus(dependentBatchRun.dependentTaskAttempt.status) || + isFinalRunStatus(dependentBatchRun.dependentTaskAttempt.taskRun.status)) + ) { + logger.debug("Dependent batch run task attempt or run has been canceled", { + dependentBatchRunId: dependentBatchRun.id, + status: dependentBatchRun.status, + attempt: dependentBatchRun.dependentTaskAttempt, + }); + + if (isFinalAttemptStatus(dependentBatchRun.dependentTaskAttempt.status)) { + throw new ServiceValidationError( + `Cannot trigger ${taskId} as the parent attempt has a status of ${dependentBatchRun.dependentTaskAttempt.status}` + ); + } else { + throw new ServiceValidationError( + `Cannot trigger ${taskId} as the parent run has a status of ${dependentBatchRun.dependentTaskAttempt.taskRun.status}` + ); + } + } + + const parentBatchRun = body.options?.parentBatch + ? await this._prisma.batchTaskRun.findUnique({ + where: { friendlyId: body.options.parentBatch }, + include: { + dependentTaskAttempt: { + include: { + taskRun: { + select: { + id: true, + status: true, + taskIdentifier: true, + rootTaskRunId: true, + }, + }, + }, + }, + }, + }) + : undefined; + + try { + return await eventRepository.traceEvent( + taskId, + { + context: options.traceContext, + spanParentAsLink: options.spanParentAsLink, + parentAsLinkType: options.parentAsLinkType, + kind: "SERVER", + environment, + taskSlug: taskId, + attributes: { + properties: { + [SemanticInternalAttributes.SHOW_ACTIONS]: true, + }, + style: { + icon: options.customIcon ?? "task", + }, + runIsTest: body.options?.test ?? false, + batchId: options.batchId, + idempotencyKey, + }, + incomplete: true, + immediate: true, + }, + async (event, traceContext, traceparent) => { + const run = await autoIncrementCounter.incrementInTransaction( + `v3-run:${environment.id}:${taskId}`, + async (num, tx) => { + const lockedToBackgroundWorker = body.options?.lockToVersion + ? await tx.backgroundWorker.findUnique({ + where: { + projectId_runtimeEnvironmentId_version: { + projectId: environment.projectId, + runtimeEnvironmentId: environment.id, + version: body.options?.lockToVersion, + }, + }, + }) + : undefined; + + let queueName = sanitizeQueueName( + await this.#getQueueName(taskId, environment, body.options?.queue?.name) + ); + + // Check that the queuename is not an empty string + if (!queueName) { + queueName = sanitizeQueueName(`task/${taskId}`); + } + + event.setAttribute("queueName", queueName); + span.setAttribute("queueName", queueName); + + //upsert tags + let tagIds: string[] = []; + const bodyTags = + typeof body.options?.tags === "string" ? [body.options.tags] : body.options?.tags; + if (bodyTags && bodyTags.length > 0) { + for (const tag of bodyTags) { + const tagRecord = await createTag({ + tag, + projectId: environment.projectId, + }); + if (tagRecord) { + tagIds.push(tagRecord.id); + } + } + } + + const depth = dependentAttempt + ? dependentAttempt.taskRun.depth + 1 + : parentAttempt + ? parentAttempt.taskRun.depth + 1 + : dependentBatchRun?.dependentTaskAttempt + ? dependentBatchRun.dependentTaskAttempt.taskRun.depth + 1 + : 0; + + const taskRun = await tx.taskRun.create({ + data: { + status: delayUntil ? "DELAYED" : "PENDING", + number: num, + friendlyId: runFriendlyId, + runtimeEnvironmentId: environment.id, + projectId: environment.projectId, + idempotencyKey, + idempotencyKeyExpiresAt: idempotencyKey ? idempotencyKeyExpiresAt : undefined, + taskIdentifier: taskId, + payload: payloadPacket.data ?? "", + payloadType: payloadPacket.dataType, + context: body.context, + traceContext: traceContext, + traceId: event.traceId, + spanId: event.spanId, + parentSpanId: + options.parentAsLinkType === "replay" ? undefined : traceparent?.spanId, + lockedToVersionId: lockedToBackgroundWorker?.id, + taskVersion: lockedToBackgroundWorker?.version, + sdkVersion: lockedToBackgroundWorker?.sdkVersion, + cliVersion: lockedToBackgroundWorker?.cliVersion, + concurrencyKey: body.options?.concurrencyKey, + queue: queueName, + isTest: body.options?.test ?? false, + delayUntil, + queuedAt: delayUntil ? undefined : new Date(), + maxAttempts: body.options?.maxAttempts, + ttl, + tags: + tagIds.length === 0 + ? undefined + : { + connect: tagIds.map((id) => ({ id })), + }, + parentTaskRunId: + dependentAttempt?.taskRun.id ?? + parentAttempt?.taskRun.id ?? + dependentBatchRun?.dependentTaskAttempt?.taskRun.id, + parentTaskRunAttemptId: + dependentAttempt?.id ?? + parentAttempt?.id ?? + dependentBatchRun?.dependentTaskAttempt?.id, + rootTaskRunId: + dependentAttempt?.taskRun.rootTaskRunId ?? + dependentAttempt?.taskRun.id ?? + parentAttempt?.taskRun.rootTaskRunId ?? + parentAttempt?.taskRun.id ?? + dependentBatchRun?.dependentTaskAttempt?.taskRun.rootTaskRunId ?? + dependentBatchRun?.dependentTaskAttempt?.taskRun.id, + batchId: dependentBatchRun?.id ?? parentBatchRun?.id, + resumeParentOnCompletion: !!(dependentAttempt ?? dependentBatchRun), + depth, + metadata: metadataPacket?.data, + metadataType: metadataPacket?.dataType, + seedMetadata: metadataPacket?.data, + seedMetadataType: metadataPacket?.dataType, + maxDurationInSeconds: body.options?.maxDuration + ? clampMaxDuration(body.options.maxDuration) + : undefined, + runTags: bodyTags, + oneTimeUseToken: options.oneTimeUseToken, + }, + }); + + event.setAttribute("runId", taskRun.friendlyId); + span.setAttribute("runId", taskRun.friendlyId); + + if (dependentAttempt) { + await tx.taskRunDependency.create({ + data: { + taskRunId: taskRun.id, + dependentAttemptId: dependentAttempt.id, + }, + }); + } else if (dependentBatchRun) { + await tx.taskRunDependency.create({ + data: { + taskRunId: taskRun.id, + dependentBatchRunId: dependentBatchRun.id, + }, + }); + } + + if (body.options?.queue) { + const concurrencyLimit = + typeof body.options.queue?.concurrencyLimit === "number" + ? Math.max( + Math.min( + body.options.queue.concurrencyLimit, + environment.maximumConcurrencyLimit, + environment.organization.maximumConcurrencyLimit + ), + 0 + ) + : null; + + let taskQueue = await tx.taskQueue.findFirst({ + where: { + runtimeEnvironmentId: environment.id, + name: queueName, + }, + }); + + const existingConcurrencyLimit = + typeof taskQueue?.concurrencyLimit === "number" + ? taskQueue.concurrencyLimit + : undefined; + + if (taskQueue) { + if (existingConcurrencyLimit !== concurrencyLimit) { + taskQueue = await tx.taskQueue.update({ + where: { + id: taskQueue.id, + }, + data: { + concurrencyLimit: + typeof concurrencyLimit === "number" ? concurrencyLimit : null, + }, + }); + + if (typeof taskQueue.concurrencyLimit === "number") { + logger.debug("TriggerTaskService: updating concurrency limit", { + runId: taskRun.id, + friendlyId: taskRun.friendlyId, + taskQueue, + orgId: environment.organizationId, + projectId: environment.projectId, + existingConcurrencyLimit, + concurrencyLimit, + queueOptions: body.options?.queue, + }); + await updateQueueConcurrencyLimits( + environment, + taskQueue.name, + taskQueue.concurrencyLimit + ); + } else { + logger.debug("TriggerTaskService: removing concurrency limit", { + runId: taskRun.id, + friendlyId: taskRun.friendlyId, + taskQueue, + orgId: environment.organizationId, + projectId: environment.projectId, + existingConcurrencyLimit, + concurrencyLimit, + queueOptions: body.options?.queue, + }); + await removeQueueConcurrencyLimits(environment, taskQueue.name); + } + } + } else { + const queueId = generateFriendlyId("queue"); + + taskQueue = await tx.taskQueue.create({ + data: { + friendlyId: queueId, + name: queueName, + concurrencyLimit, + runtimeEnvironmentId: environment.id, + projectId: environment.projectId, + type: "NAMED", + }, + }); + + if (typeof taskQueue.concurrencyLimit === "number") { + await marqs?.updateQueueConcurrencyLimits( + environment, + taskQueue.name, + taskQueue.concurrencyLimit + ); + } + } + } + + if (taskRun.delayUntil) { + await workerQueue.enqueue( + "v3.enqueueDelayedRun", + { runId: taskRun.id }, + { tx, runAt: delayUntil, jobKey: `v3.enqueueDelayedRun.${taskRun.id}` } + ); + } + + if (!taskRun.delayUntil && taskRun.ttl) { + const expireAt = parseNaturalLanguageDuration(taskRun.ttl); + + if (expireAt) { + await ExpireEnqueuedRunService.enqueue(taskRun.id, expireAt, tx); + } + } + + return taskRun; + }, + async (_, tx) => { + const counter = await tx.taskRunNumberCounter.findUnique({ + where: { + taskIdentifier_environmentId: { + taskIdentifier: taskId, + environmentId: environment.id, + }, + }, + select: { lastNumber: true }, + }); + + return counter?.lastNumber; + }, + this._prisma + ); + + //release the concurrency for the env and org, if part of a (batch)triggerAndWait + if (dependentAttempt) { + const isSameTask = dependentAttempt.taskRun.taskIdentifier === taskId; + await marqs?.releaseConcurrency(dependentAttempt.taskRun.id, isSameTask); + } + if (dependentBatchRun?.dependentTaskAttempt) { + const isSameTask = + dependentBatchRun.dependentTaskAttempt.taskRun.taskIdentifier === taskId; + await marqs?.releaseConcurrency( + dependentBatchRun.dependentTaskAttempt.taskRun.id, + isSameTask + ); + } + + if (!run) { + return; + } + + // We need to enqueue the task run into the appropriate queue. This is done after the tx completes to prevent a race condition where the task run hasn't been created yet by the time we dequeue. + if (run.status === "PENDING") { + await marqs?.enqueueMessage( + environment, + run.queue, + run.id, + { + type: "EXECUTE", + taskIdentifier: taskId, + projectId: environment.projectId, + environmentId: environment.id, + environmentType: environment.type, + }, + body.options?.concurrencyKey + ); + } + + return run; + } + ); + } catch (error) { + // Detect a prisma transaction Unique constraint violation + if (error instanceof Prisma.PrismaClientKnownRequestError) { + logger.debug("TriggerTask: Prisma transaction error", { + code: error.code, + message: error.message, + meta: error.meta, + }); + + if (error.code === "P2002") { + const target = error.meta?.target; + + if ( + Array.isArray(target) && + target.length > 0 && + typeof target[0] === "string" && + target[0].includes("oneTimeUseToken") + ) { + throw new ServiceValidationError( + `Cannot trigger ${taskId} with a one-time use token as it has already been used.` + ); + } else { + throw new ServiceValidationError( + `Cannot trigger ${taskId} as it has already been triggered with the same idempotency key.` + ); + } + } + } + + throw error; + } + }); + } + + async #getQueueName(taskId: string, environment: AuthenticatedEnvironment, queueName?: string) { + if (queueName) { + return queueName; + } + + const defaultQueueName = `task/${taskId}`; + + const worker = await findCurrentWorkerFromEnvironment(environment); + + if (!worker) { + logger.debug("Failed to get queue name: No worker found", { + taskId, + environmentId: environment.id, + }); + + return defaultQueueName; + } + + const task = await this._prisma.backgroundWorkerTask.findUnique({ + where: { + workerId_slug: { + workerId: worker.id, + slug: taskId, + }, + }, + }); + + if (!task) { + console.log("Failed to get queue name: No task found", { + taskId, + environmentId: environment.id, + }); + + return defaultQueueName; + } + + const queueConfig = QueueOptions.optional().nullable().safeParse(task.queueConfig); + + if (!queueConfig.success) { + console.log("Failed to get queue name: Invalid queue config", { + taskId, + environmentId: environment.id, + queueConfig: task.queueConfig, + }); + + return defaultQueueName; + } + + return queueConfig.data?.name ?? defaultQueueName; + } + + async #handlePayloadPacket( + payload: any, + payloadType: string, + pathPrefix: string, + environment: AuthenticatedEnvironment + ) { + return await startActiveSpan("handlePayloadPacket()", async (span) => { + const packet = this.#createPayloadPacket(payload, payloadType); + + if (!packet.data) { + return packet; + } + + const { needsOffloading, size } = packetRequiresOffloading( + packet, + env.TASK_PAYLOAD_OFFLOAD_THRESHOLD + ); + + if (!needsOffloading) { + return packet; + } + + const filename = `${pathPrefix}/payload.json`; + + await uploadPacketToObjectStore(filename, packet.data, packet.dataType, environment); + + return { + data: filename, + dataType: "application/store", + }; + }); + } + + #createPayloadPacket(payload: any, payloadType: string): IOPacket { + if (payloadType === "application/json") { + return { data: JSON.stringify(payload), dataType: "application/json" }; + } + + if (typeof payload === "string") { + return { data: payload, dataType: payloadType }; + } + + return { dataType: payloadType }; + } +} diff --git a/apps/webapp/app/v3/services/triggerTaskV2.server.ts b/apps/webapp/app/v3/services/triggerTaskV2.server.ts new file mode 100644 index 0000000000..ad1dd097d0 --- /dev/null +++ b/apps/webapp/app/v3/services/triggerTaskV2.server.ts @@ -0,0 +1,495 @@ +import { + IOPacket, + QueueOptions, + SemanticInternalAttributes, + TriggerTaskRequestBody, + packetRequiresOffloading, +} from "@trigger.dev/core/v3"; +import { env } from "~/env.server"; +import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { autoIncrementCounter } from "~/services/autoIncrementCounter.server"; +import { sanitizeQueueName } from "~/v3/marqs/index.server"; +import { eventRepository } from "../eventRepository.server"; +import { uploadPacketToObjectStore } from "../r2.server"; +import { startActiveSpan } from "../tracer.server"; +import { getEntitlement } from "~/services/platform.v3.server"; +import { ServiceValidationError, WithRunEngine } from "./baseService.server"; +import { logger } from "~/services/logger.server"; +import { isFinalRunStatus } from "../taskStatus"; +import { createTag, MAX_TAGS_PER_RUN } from "~/models/taskRunTag.server"; +import { findCurrentWorkerFromEnvironment } from "../models/workerDeployment.server"; +import { handleMetadataPacket } from "~/utils/packets"; +import { WorkerGroupService } from "./worker/workerGroupService.server"; +import { parseDelay } from "~/utils/delays"; +import { RunId, stringifyDuration } from "@trigger.dev/core/v3/apps"; +import { OutOfEntitlementError, TriggerTaskServiceOptions } from "./triggerTask.server"; +import { Prisma } from "@trigger.dev/database"; +import { resolveIdempotencyKeyTTL } from "~/utils/idempotencyKeys.server"; +import { clampMaxDuration } from "../utils/maxDuration"; +import { RunEngine } from "@internal/run-engine"; + +/** @deprecated Use TriggerTaskService in `triggerTask.server.ts` instead. */ +export class TriggerTaskServiceV2 extends WithRunEngine { + public async call({ + taskId, + environment, + body, + options = {}, + }: { + taskId: string; + environment: AuthenticatedEnvironment; + body: TriggerTaskRequestBody; + options?: TriggerTaskServiceOptions; + }) { + return await this.traceWithEnv("call()", environment, async (span) => { + span.setAttribute("taskId", taskId); + + const idempotencyKey = options.idempotencyKey ?? body.options?.idempotencyKey; + const idempotencyKeyExpiresAt = + options.idempotencyKeyExpiresAt ?? + resolveIdempotencyKeyTTL(body.options?.idempotencyKeyTTL) ?? + new Date(Date.now() + 24 * 60 * 60 * 1000 * 30); // 30 days + + const delayUntil = await parseDelay(body.options?.delay); + + const ttl = + typeof body.options?.ttl === "number" + ? stringifyDuration(body.options?.ttl) + : body.options?.ttl ?? (environment.type === "DEVELOPMENT" ? "10m" : undefined); + + const existingRun = idempotencyKey + ? await this._prisma.taskRun.findFirst({ + where: { + runtimeEnvironmentId: environment.id, + idempotencyKey, + taskIdentifier: taskId, + }, + include: { + associatedWaitpoint: true, + }, + }) + : undefined; + + if (existingRun) { + span.setAttribute("runId", existingRun.friendlyId); + + if ( + existingRun.idempotencyKeyExpiresAt && + existingRun.idempotencyKeyExpiresAt < new Date() + ) { + logger.debug("[TriggerTaskService][call] Idempotency key has expired", { + idempotencyKey: options.idempotencyKey, + run: existingRun, + }); + + // Update the existing run to remove the idempotency key + await this._prisma.taskRun.update({ + where: { id: existingRun.id }, + data: { idempotencyKey: null }, + }); + } else { + //We're using `andWait` so we need to block the parent run with a waitpoint + if ( + existingRun.associatedWaitpoint && + body.options?.resumeParentOnCompletion && + body.options?.parentRunId + ) { + await this._engine.blockRunWithWaitpoint({ + runId: RunId.fromFriendlyId(body.options.parentRunId), + waitpointId: existingRun.associatedWaitpoint.id, + environmentId: environment.id, + projectId: environment.projectId, + tx: this._prisma, + }); + } + + return existingRun; + } + } + + if (environment.type !== "DEVELOPMENT") { + const result = await getEntitlement(environment.organizationId); + if (result && result.hasAccess === false) { + throw new OutOfEntitlementError(); + } + } + + if (!options.skipChecks) { + const queueSizeGuard = await guardQueueSizeLimitsForEnv(this._engine, environment); + + logger.debug("Queue size guard result", { + queueSizeGuard, + environment: { + id: environment.id, + type: environment.type, + organization: environment.organization, + project: environment.project, + }, + }); + + if (!queueSizeGuard.isWithinLimits) { + throw new ServiceValidationError( + `Cannot trigger ${taskId} as the queue size limit for this environment has been reached. The maximum size is ${queueSizeGuard.maximumSize}` + ); + } + } + + if ( + body.options?.tags && + typeof body.options.tags !== "string" && + body.options.tags.length > MAX_TAGS_PER_RUN + ) { + throw new ServiceValidationError( + `Runs can only have ${MAX_TAGS_PER_RUN} tags, you're trying to set ${body.options.tags.length}.` + ); + } + + const runFriendlyId = options?.runFriendlyId ?? RunId.generate().friendlyId; + + const payloadPacket = await this.#handlePayloadPacket( + body.payload, + body.options?.payloadType ?? "application/json", + runFriendlyId, + environment + ); + + const metadataPacket = body.options?.metadata + ? handleMetadataPacket( + body.options?.metadata, + body.options?.metadataType ?? "application/json" + ) + : undefined; + + //todo we will pass in the `parentRun` and `resumeParentOnCompletion` + const parentRun = body.options?.parentRunId + ? await this._prisma.taskRun.findFirst({ + where: { id: RunId.fromFriendlyId(body.options.parentRunId) }, + }) + : undefined; + + if (parentRun && isFinalRunStatus(parentRun.status)) { + logger.debug("Parent run is in a terminal state", { + parentRun, + }); + + throw new ServiceValidationError( + `Cannot trigger ${taskId} as the parent run has a status of ${parentRun.status}` + ); + } + + try { + return await eventRepository.traceEvent( + taskId, + { + context: options.traceContext, + spanParentAsLink: options.spanParentAsLink, + parentAsLinkType: options.parentAsLinkType, + kind: "SERVER", + environment, + taskSlug: taskId, + attributes: { + properties: { + [SemanticInternalAttributes.SHOW_ACTIONS]: true, + }, + style: { + icon: options.customIcon ?? "task", + }, + runIsTest: body.options?.test ?? false, + batchId: options.batchId, + idempotencyKey, + }, + incomplete: true, + immediate: true, + }, + async (event, traceContext, traceparent) => { + const run = await autoIncrementCounter.incrementInTransaction( + `v3-run:${environment.id}:${taskId}`, + async (num, tx) => { + const lockedToBackgroundWorker = body.options?.lockToVersion + ? await tx.backgroundWorker.findUnique({ + where: { + projectId_runtimeEnvironmentId_version: { + projectId: environment.projectId, + runtimeEnvironmentId: environment.id, + version: body.options?.lockToVersion, + }, + }, + }) + : undefined; + + let queueName = sanitizeQueueName( + await this.#getQueueName(taskId, environment, body.options?.queue?.name) + ); + + // Check that the queuename is not an empty string + if (!queueName) { + queueName = sanitizeQueueName(`task/${taskId}`); + } + + event.setAttribute("queueName", queueName); + span.setAttribute("queueName", queueName); + + //upsert tags + let tags: { id: string; name: string }[] = []; + const bodyTags = + typeof body.options?.tags === "string" ? [body.options.tags] : body.options?.tags; + if (bodyTags && bodyTags.length > 0) { + for (const tag of bodyTags) { + const tagRecord = await createTag({ + tag, + projectId: environment.projectId, + }); + if (tagRecord) { + tags.push(tagRecord); + } + } + } + + const depth = parentRun ? parentRun.depth + 1 : 0; + + event.setAttribute("runId", runFriendlyId); + span.setAttribute("runId", runFriendlyId); + + const workerGroupService = new WorkerGroupService({ + prisma: this._prisma, + engine: this._engine, + }); + const workerGroup = await workerGroupService.getDefaultWorkerGroupForProject({ + projectId: environment.projectId, + }); + + if (!workerGroup) { + logger.error("Default worker group not found", { + projectId: environment.projectId, + }); + + return; + } + + const taskRun = await this._engine.trigger( + { + number: num, + friendlyId: runFriendlyId, + environment: environment, + idempotencyKey, + idempotencyKeyExpiresAt: idempotencyKey ? idempotencyKeyExpiresAt : undefined, + taskIdentifier: taskId, + payload: payloadPacket.data ?? "", + payloadType: payloadPacket.dataType, + context: body.context, + traceContext: traceContext, + traceId: event.traceId, + spanId: event.spanId, + parentSpanId: + options.parentAsLinkType === "replay" ? undefined : traceparent?.spanId, + lockedToVersionId: lockedToBackgroundWorker?.id, + taskVersion: lockedToBackgroundWorker?.version, + sdkVersion: lockedToBackgroundWorker?.sdkVersion, + cliVersion: lockedToBackgroundWorker?.cliVersion, + concurrencyKey: body.options?.concurrencyKey, + queueName, + queue: body.options?.queue, + masterQueue: workerGroup.masterQueue, + isTest: body.options?.test ?? false, + delayUntil, + queuedAt: delayUntil ? undefined : new Date(), + maxAttempts: body.options?.maxAttempts, + ttl, + tags, + oneTimeUseToken: options.oneTimeUseToken, + parentTaskRunId: parentRun?.id, + rootTaskRunId: parentRun?.rootTaskRunId ?? parentRun?.id, + batchId: body.options?.parentBatch ?? undefined, + resumeParentOnCompletion: body.options?.resumeParentOnCompletion, + depth, + metadata: metadataPacket?.data, + metadataType: metadataPacket?.dataType, + seedMetadata: metadataPacket?.data, + seedMetadataType: metadataPacket?.dataType, + maxDurationInSeconds: body.options?.maxDuration + ? clampMaxDuration(body.options.maxDuration) + : undefined, + }, + this._prisma + ); + + return taskRun; + }, + async (_, tx) => { + const counter = await tx.taskRunNumberCounter.findUnique({ + where: { + taskIdentifier_environmentId: { + taskIdentifier: taskId, + environmentId: environment.id, + }, + }, + select: { lastNumber: true }, + }); + + return counter?.lastNumber; + }, + this._prisma + ); + + return run; + } + ); + } catch (error) { + // Detect a prisma transaction Unique constraint violation + if (error instanceof Prisma.PrismaClientKnownRequestError) { + logger.debug("TriggerTask: Prisma transaction error", { + code: error.code, + message: error.message, + meta: error.meta, + }); + + if (error.code === "P2002") { + const target = error.meta?.target; + + if ( + Array.isArray(target) && + target.length > 0 && + typeof target[0] === "string" && + target[0].includes("oneTimeUseToken") + ) { + throw new ServiceValidationError( + `Cannot trigger ${taskId} with a one-time use token as it has already been used.` + ); + } else { + throw new ServiceValidationError( + `Cannot trigger ${taskId} as it has already been triggered with the same idempotency key.` + ); + } + } + } + + throw error; + } + }); + } + + async #getQueueName(taskId: string, environment: AuthenticatedEnvironment, queueName?: string) { + if (queueName) { + return queueName; + } + + const defaultQueueName = `task/${taskId}`; + + const worker = await findCurrentWorkerFromEnvironment(environment); + + if (!worker) { + logger.debug("Failed to get queue name: No worker found", { + taskId, + environmentId: environment.id, + }); + + return defaultQueueName; + } + + const task = await this._prisma.backgroundWorkerTask.findUnique({ + where: { + workerId_slug: { + workerId: worker.id, + slug: taskId, + }, + }, + }); + + if (!task) { + console.log("Failed to get queue name: No task found", { + taskId, + environmentId: environment.id, + }); + + return defaultQueueName; + } + + const queueConfig = QueueOptions.optional().nullable().safeParse(task.queueConfig); + + if (!queueConfig.success) { + console.log("Failed to get queue name: Invalid queue config", { + taskId, + environmentId: environment.id, + queueConfig: task.queueConfig, + }); + + return defaultQueueName; + } + + return queueConfig.data?.name ?? defaultQueueName; + } + + async #handlePayloadPacket( + payload: any, + payloadType: string, + pathPrefix: string, + environment: AuthenticatedEnvironment + ) { + return await startActiveSpan("handlePayloadPacket()", async (span) => { + const packet = this.#createPayloadPacket(payload, payloadType); + + if (!packet.data) { + return packet; + } + + const { needsOffloading, size } = packetRequiresOffloading( + packet, + env.TASK_PAYLOAD_OFFLOAD_THRESHOLD + ); + + if (!needsOffloading) { + return packet; + } + + const filename = `${pathPrefix}/payload.json`; + + await uploadPacketToObjectStore(filename, packet.data, packet.dataType, environment); + + return { + data: filename, + dataType: "application/store", + }; + }); + } + + #createPayloadPacket(payload: any, payloadType: string): IOPacket { + if (payloadType === "application/json") { + return { data: JSON.stringify(payload), dataType: "application/json" }; + } + + if (typeof payload === "string") { + return { data: payload, dataType: payloadType }; + } + + return { dataType: payloadType }; + } +} + +function getMaximumSizeForEnvironment(environment: AuthenticatedEnvironment): number | undefined { + if (environment.type === "DEVELOPMENT") { + return environment.organization.maximumDevQueueSize ?? env.MAXIMUM_DEV_QUEUE_SIZE; + } else { + return environment.organization.maximumDeployedQueueSize ?? env.MAXIMUM_DEPLOYED_QUEUE_SIZE; + } +} + +export async function guardQueueSizeLimitsForEnv( + engine: RunEngine, + environment: AuthenticatedEnvironment, + itemsToAdd: number = 1 +) { + const maximumSize = getMaximumSizeForEnvironment(environment); + + if (typeof maximumSize === "undefined") { + return { isWithinLimits: true }; + } + + const queueSize = await engine.lengthOfEnvQueue(environment); + const projectedSize = queueSize + itemsToAdd; + + return { + isWithinLimits: projectedSize <= maximumSize, + maximumSize, + queueSize, + }; +} diff --git a/apps/webapp/app/v3/services/worker/workerGroupService.server.ts b/apps/webapp/app/v3/services/worker/workerGroupService.server.ts new file mode 100644 index 0000000000..affdabd319 --- /dev/null +++ b/apps/webapp/app/v3/services/worker/workerGroupService.server.ts @@ -0,0 +1,261 @@ +import { WorkerInstanceGroup, WorkerInstanceGroupType } from "@trigger.dev/database"; +import { WithRunEngine } from "../baseService.server"; +import { WorkerGroupTokenService } from "./workerGroupTokenService.server"; +import { logger } from "~/services/logger.server"; +import { makeFlags, makeSetFlags } from "~/v3/featureFlags.server"; + +export class WorkerGroupService extends WithRunEngine { + private readonly defaultNamePrefix = "worker_group"; + + async createWorkerGroup({ + projectId, + organizationId, + name, + description, + }: { + projectId?: string; + organizationId?: string; + name?: string; + description?: string; + }) { + name = name ?? (await this.generateWorkerName({ projectId })); + + const tokenService = new WorkerGroupTokenService({ + prisma: this._prisma, + engine: this._engine, + }); + const token = await tokenService.createToken(); + + const workerGroup = await this._prisma.workerInstanceGroup.create({ + data: { + projectId, + organizationId, + type: projectId ? WorkerInstanceGroupType.UNMANAGED : WorkerInstanceGroupType.MANAGED, + masterQueue: this.generateMasterQueueName({ projectId, name }), + tokenId: token.id, + description, + name, + }, + }); + + if (workerGroup.type === WorkerInstanceGroupType.MANAGED) { + const managedCount = await this._prisma.workerInstanceGroup.count({ + where: { + type: WorkerInstanceGroupType.MANAGED, + }, + }); + + if (managedCount === 1) { + const setFlag = makeSetFlags(this._prisma); + await setFlag({ + key: "defaultWorkerInstanceGroupId", + value: workerGroup.id, + }); + } + } + + return { + workerGroup, + token, + }; + } + + /** + This updates a single worker group. + The name should never be updated. This would mean changing the masterQueue name which can have unexpected consequences. + */ + async updateWorkerGroup({ + projectId, + workerGroupId, + description, + }: { + projectId: string; + workerGroupId: string; + description?: string; + }) { + const workerGroup = await this._prisma.workerInstanceGroup.findUnique({ + where: { + id: workerGroupId, + projectId, + }, + }); + + if (!workerGroup) { + logger.error("[WorkerGroupService] No worker group found for update", { + workerGroupId, + description, + }); + return; + } + + await this._prisma.workerInstanceGroup.update({ + where: { + id: workerGroup.id, + }, + data: { + description, + }, + }); + } + + /** + This lists worker groups. + Without a project ID, only shared worker groups will be returned. + With a project ID, in addition to all shared worker groups, ones associated with the project will also be returned. + */ + async listWorkerGroups({ projectId, listHidden }: { projectId?: string; listHidden?: boolean }) { + const workerGroups = await this._prisma.workerInstanceGroup.findMany({ + where: { + OR: [ + { + type: WorkerInstanceGroupType.MANAGED, + }, + { + projectId, + }, + ], + AND: listHidden ? [] : [{ hidden: false }], + }, + }); + + return workerGroups; + } + + async deleteWorkerGroup({ + projectId, + workerGroupId, + }: { + projectId: string; + workerGroupId: string; + }) { + const workerGroup = await this._prisma.workerInstanceGroup.findUnique({ + where: { + id: workerGroupId, + }, + }); + + if (!workerGroup) { + logger.error("[WorkerGroupService] WorkerGroup not found for deletion", { + workerGroupId, + projectId, + }); + return; + } + + if (workerGroup.projectId !== projectId) { + logger.error("[WorkerGroupService] WorkerGroup does not belong to project", { + workerGroupId, + projectId, + }); + return; + } + + await this._prisma.workerInstanceGroup.delete({ + where: { + id: workerGroupId, + }, + }); + } + + async getGlobalDefaultWorkerGroup() { + const flags = makeFlags(this._prisma); + + const defaultWorkerInstanceGroupId = await flags({ + key: "defaultWorkerInstanceGroupId", + }); + + if (!defaultWorkerInstanceGroupId) { + logger.error("[WorkerGroupService] Default worker group not found in feature flags"); + return; + } + + const workerGroup = await this._prisma.workerInstanceGroup.findUnique({ + where: { + id: defaultWorkerInstanceGroupId, + }, + }); + + if (!workerGroup) { + logger.error("[WorkerGroupService] Default worker group not found", { + defaultWorkerInstanceGroupId, + }); + return; + } + + return workerGroup; + } + + async getDefaultWorkerGroupForProject({ + projectId, + }: { + projectId: string; + }): Promise { + const project = await this._prisma.project.findUnique({ + where: { + id: projectId, + }, + include: { + defaultWorkerGroup: true, + }, + }); + + if (!project) { + logger.error("[WorkerGroupService] Project not found", { projectId }); + return; + } + + if (project.defaultWorkerGroup) { + return project.defaultWorkerGroup; + } + + return await this.getGlobalDefaultWorkerGroup(); + } + + async setDefaultWorkerGroupForProject({ + projectId, + workerGroupId, + }: { + projectId: string; + workerGroupId: string; + }) { + const workerGroup = await this._prisma.workerInstanceGroup.findUnique({ + where: { + id: workerGroupId, + }, + }); + + if (!workerGroup) { + logger.error("[WorkerGroupService] WorkerGroup not found", { + workerGroupId, + }); + return; + } + + await this._prisma.project.update({ + where: { + id: projectId, + }, + data: { + defaultWorkerGroupId: workerGroupId, + }, + }); + } + + private async generateWorkerName({ projectId }: { projectId?: string }) { + const workerGroups = await this._prisma.workerInstanceGroup.count({ + where: { + projectId: projectId ?? null, + }, + }); + + return `${this.defaultNamePrefix}_${workerGroups + 1}`; + } + + private generateMasterQueueName({ projectId, name }: { projectId?: string; name: string }) { + if (!projectId) { + return name; + } + + return `${projectId}-${name}`; + } +} diff --git a/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts b/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts new file mode 100644 index 0000000000..54bef5c7b5 --- /dev/null +++ b/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts @@ -0,0 +1,779 @@ +import { customAlphabet } from "nanoid"; +import { WithRunEngine, WithRunEngineOptions } from "../baseService.server"; +import { createHash, timingSafeEqual } from "crypto"; +import { logger } from "~/services/logger.server"; +import { + Prisma, + RuntimeEnvironment, + WorkerInstanceGroup, + WorkerInstanceGroupType, +} from "@trigger.dev/database"; +import { z } from "zod"; +import { HEADER_NAME } from "@trigger.dev/worker"; +import { + TaskRunExecutionResult, + DequeuedMessage, + CompleteRunAttemptResult, + StartRunAttemptResult, + ExecutionResult, + MachinePreset, + WaitForDurationResult, +} from "@trigger.dev/core/v3"; +import { env } from "~/env.server"; +import { $transaction } from "~/db.server"; +import { resolveVariablesForEnvironment } from "~/v3/environmentVariables/environmentVariablesRepository.server"; +import { generateJWTTokenForEnvironment } from "~/services/apiAuth.server"; +import { CURRENT_UNMANAGED_DEPLOYMENT_LABEL, fromFriendlyId } from "@trigger.dev/core/v3/apps"; +import { machinePresetFromName } from "~/v3/machinePresets.server"; +import { defaultMachine } from "@trigger.dev/platform/v3"; + +export class WorkerGroupTokenService extends WithRunEngine { + private readonly tokenPrefix = "tr_wgt_"; + private readonly tokenLength = 40; + private readonly tokenChars = "1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + private readonly tokenGenerator = customAlphabet(this.tokenChars, this.tokenLength); + + async createToken() { + const rawToken = await this.generateToken(); + + const workerGroupToken = await this._prisma.workerGroupToken.create({ + data: { + tokenHash: rawToken.hash, + }, + }); + + return { + id: workerGroupToken.id, + tokenHash: workerGroupToken.tokenHash, + plaintext: rawToken.plaintext, + }; + } + + async findWorkerGroup({ token }: { token: string }) { + const tokenHash = await this.hashToken(token); + + const workerGroup = await this._prisma.workerInstanceGroup.findFirst({ + where: { + token: { + tokenHash, + }, + }, + }); + + if (!workerGroup) { + logger.warn("[WorkerGroupTokenService] No matching worker group found", { token }); + return null; + } + + return workerGroup; + } + + async rotateToken({ workerGroupId }: { workerGroupId: string }) { + const workerGroup = await this._prisma.workerInstanceGroup.findUnique({ + where: { + id: workerGroupId, + }, + }); + + if (!workerGroup) { + logger.error("[WorkerGroupTokenService] WorkerGroup not found", { workerGroupId }); + return; + } + + const rawToken = await this.generateToken(); + + const workerGroupToken = await this._prisma.workerGroupToken.update({ + where: { + id: workerGroup.tokenId, + }, + data: { + tokenHash: rawToken.hash, + }, + }); + + if (!workerGroupToken) { + logger.error("[WorkerGroupTokenService] WorkerGroupToken not found", { workerGroupId }); + return; + } + + return { + id: workerGroupToken.id, + tokenHash: workerGroupToken.tokenHash, + plaintext: rawToken.plaintext, + }; + } + + private async hashToken(token: string) { + return createHash("sha256").update(token).digest("hex"); + } + + private async generateToken() { + const plaintext = `${this.tokenPrefix}${this.tokenGenerator()}`; + const hash = await this.hashToken(plaintext); + + return { + plaintext, + hash, + }; + } + + async authenticate(request: Request): Promise { + const token = request.headers.get("Authorization")?.replace("Bearer ", "").trim(); + + if (!token) { + logger.error("[WorkerGroupTokenService] Token not found in request", { + headers: this.sanitizeHeaders(request), + }); + return; + } + + if (!token.startsWith(this.tokenPrefix)) { + logger.error("[WorkerGroupTokenService] Token does not start with expected prefix", { + token, + prefix: this.tokenPrefix, + }); + return; + } + + const instanceName = request.headers.get(HEADER_NAME.WORKER_INSTANCE_NAME); + + if (!instanceName) { + logger.error("[WorkerGroupTokenService] Instance name not found in request", { + headers: this.sanitizeHeaders(request), + }); + return; + } + + const workerGroup = await this.findWorkerGroup({ token }); + + if (!workerGroup) { + logger.warn("[WorkerGroupTokenService] Worker group not found", { token }); + return; + } + + if (workerGroup.type === WorkerInstanceGroupType.MANAGED) { + const managedWorkerSecret = request.headers.get(HEADER_NAME.WORKER_MANAGED_SECRET); + + if (!managedWorkerSecret) { + logger.error("[WorkerGroupTokenService] Managed secret not found in request", { + headers: this.sanitizeHeaders(request), + }); + return; + } + + const encoder = new TextEncoder(); + + const a = encoder.encode(managedWorkerSecret); + const b = encoder.encode(env.MANAGED_WORKER_SECRET); + + if (a.byteLength !== b.byteLength) { + logger.error("[WorkerGroupTokenService] Managed secret length mismatch", { + managedWorkerSecret, + headers: this.sanitizeHeaders(request), + }); + return; + } + + if (!timingSafeEqual(a, b)) { + logger.error("[WorkerGroupTokenService] Managed secret mismatch", { + managedWorkerSecret, + headers: this.sanitizeHeaders(request), + }); + return; + } + } + + const workerInstance = await this.getOrCreateWorkerInstance({ + workerGroup, + instanceName, + deploymentId: request.headers.get(HEADER_NAME.WORKER_DEPLOYMENT_ID) ?? undefined, + }); + + if (!workerInstance) { + logger.error("[WorkerGroupTokenService] Unable to get or create worker instance", { + workerGroup, + instanceName, + }); + return; + } + + if (workerGroup.type === WorkerInstanceGroupType.MANAGED) { + return new AuthenticatedWorkerInstance({ + prisma: this._prisma, + engine: this._engine, + type: WorkerInstanceGroupType.MANAGED, + name: workerGroup.name, + workerGroupId: workerGroup.id, + workerInstanceId: workerInstance.id, + masterQueue: workerGroup.masterQueue, + environment: null, + }); + } + + if (!workerInstance.environment) { + logger.error( + "[WorkerGroupTokenService] Unmanaged worker instance not linked to environment", + { workerGroup, workerInstance } + ); + return; + } + + if (!workerInstance.deployment) { + logger.error("[WorkerGroupTokenService] Unmanaged worker instance not linked to deployment", { + workerGroup, + workerInstance, + }); + return; + } + + if (!workerInstance.deployment.workerId) { + logger.error( + "[WorkerGroupTokenService] Unmanaged worker instance deployment not linked to background worker", + { workerGroup, workerInstance } + ); + return; + } + + return new AuthenticatedWorkerInstance({ + prisma: this._prisma, + engine: this._engine, + type: WorkerInstanceGroupType.UNMANAGED, + name: workerGroup.name, + workerGroupId: workerGroup.id, + workerInstanceId: workerInstance.id, + masterQueue: workerGroup.masterQueue, + environmentId: workerInstance.environment.id, + deploymentId: workerInstance.deployment.id, + backgroundWorkerId: workerInstance.deployment.workerId, + environment: workerInstance.environment, + }); + } + + private async getOrCreateWorkerInstance({ + workerGroup, + instanceName, + deploymentId, + }: { + workerGroup: WorkerInstanceGroup; + instanceName: string; + deploymentId?: string; + }) { + return await $transaction(this._prisma, async (tx) => { + const resourceIdentifier = deploymentId ? `${deploymentId}:${instanceName}` : instanceName; + + const workerInstance = await tx.workerInstance.findUnique({ + where: { + workerGroupId_resourceIdentifier: { + workerGroupId: workerGroup.id, + resourceIdentifier, + }, + }, + include: { + deployment: true, + environment: true, + }, + }); + + if (workerInstance) { + return workerInstance; + } + + if (workerGroup.type === WorkerInstanceGroupType.MANAGED) { + if (deploymentId) { + logger.warn( + "[WorkerGroupTokenService] Shared worker group instances should not authenticate with a deployment ID", + { + workerGroup, + workerInstance, + deploymentId, + } + ); + } + + try { + const newWorkerInstance = await tx.workerInstance.create({ + data: { + workerGroupId: workerGroup.id, + name: instanceName, + resourceIdentifier, + }, + include: { + // This will always be empty for shared worker instances, but required for types + deployment: true, + environment: true, + }, + }); + return newWorkerInstance; + } catch (error) { + // Gracefully handle race conditions when connecting for the first time + if (error instanceof Prisma.PrismaClientKnownRequestError) { + // Unique constraint violation + if (error.code === "P2002") { + try { + const existingWorkerInstance = await tx.workerInstance.findUnique({ + where: { + workerGroupId_resourceIdentifier: { + workerGroupId: workerGroup.id, + resourceIdentifier, + }, + }, + include: { + deployment: true, + environment: true, + }, + }); + return existingWorkerInstance; + } catch (error) { + logger.error("[WorkerGroupTokenService] Failed to find worker instance", { + workerGroup, + workerInstance, + deploymentId, + }); + return; + } + } + } + } + } + + if (!workerGroup.projectId || !workerGroup.organizationId) { + logger.error( + "[WorkerGroupTokenService] Unmanaged worker group missing project or organization", + { + workerGroup, + workerInstance, + deploymentId, + } + ); + return; + } + + if (!deploymentId) { + logger.error("[WorkerGroupTokenService] Unmanaged worker group required deployment ID", { + workerGroup, + workerInstance, + }); + return; + } + + // Unmanaged workers instances are locked to a specific deployment version + + const deployment = await tx.workerDeployment.findUnique({ + where: { + ...(deploymentId.startsWith("deployment_") + ? { + friendlyId: deploymentId, + } + : { + id: deploymentId, + }), + }, + }); + + if (!deployment) { + logger.error("[WorkerGroupTokenService] Deployment not found", { + workerGroup, + workerInstance, + deploymentId, + }); + return; + } + + if (deployment.projectId !== workerGroup.projectId) { + logger.error("[WorkerGroupTokenService] Deployment does not match worker group project", { + deployment, + workerGroup, + workerInstance, + }); + return; + } + + if (deployment.status === "DEPLOYING") { + // This is the first instance to be created for this deployment, so mark it as deployed + await tx.workerDeployment.update({ + where: { + id: deployment.id, + }, + data: { + status: "DEPLOYED", + deployedAt: new Date(), + }, + }); + + // Check if the deployment should be promoted + const workerPromotion = await tx.workerDeploymentPromotion.findFirst({ + where: { + label: CURRENT_UNMANAGED_DEPLOYMENT_LABEL, + environmentId: deployment.environmentId, + }, + include: { + deployment: true, + }, + }); + + const shouldPromote = + !workerPromotion || deployment.createdAt > workerPromotion.deployment.createdAt; + + if (shouldPromote) { + // Promote the deployment + await tx.workerDeploymentPromotion.upsert({ + where: { + environmentId_label: { + environmentId: deployment.environmentId, + label: CURRENT_UNMANAGED_DEPLOYMENT_LABEL, + }, + }, + create: { + deploymentId: deployment.id, + environmentId: deployment.environmentId, + label: CURRENT_UNMANAGED_DEPLOYMENT_LABEL, + }, + update: { + deploymentId: deployment.id, + }, + }); + } + } else if (deployment.status !== "DEPLOYED") { + logger.error("[WorkerGroupTokenService] Deployment not deploying or deployed", { + deployment, + workerGroup, + workerInstance, + }); + return; + } + + const nonSharedWorkerInstance = tx.workerInstance.create({ + data: { + workerGroupId: workerGroup.id, + name: instanceName, + resourceIdentifier, + environmentId: deployment.environmentId, + deploymentId: deployment.id, + }, + include: { + deployment: true, + environment: true, + }, + }); + + return nonSharedWorkerInstance; + }); + } + + private sanitizeHeaders(request: Request, skipHeaders = ["authorization"]) { + const sanitizedHeaders: Partial> = {}; + + for (const [key, value] of request.headers.entries()) { + if (!skipHeaders.includes(key.toLowerCase())) { + sanitizedHeaders[key] = value; + } + } + + return sanitizedHeaders; + } +} + +export const WorkerInstanceEnv = z.enum(["dev", "staging", "prod"]).default("prod"); +export type WorkerInstanceEnv = z.infer; + +export type AuthenticatedWorkerInstanceOptions = WithRunEngineOptions<{ + type: WorkerInstanceGroupType; + name: string; + workerGroupId: string; + workerInstanceId: string; + masterQueue: string; + environmentId?: string; + deploymentId?: string; + backgroundWorkerId?: string; + environment: RuntimeEnvironment | null; +}>; + +export class AuthenticatedWorkerInstance extends WithRunEngine { + readonly type: WorkerInstanceGroupType; + readonly name: string; + readonly workerGroupId: string; + readonly workerInstanceId: string; + readonly masterQueue: string; + readonly environment: RuntimeEnvironment | null; + readonly deploymentId?: string; + readonly backgroundWorkerId?: string; + + // FIXME: Required for unmanaged workers + readonly isLatestDeployment = true; + + constructor(opts: AuthenticatedWorkerInstanceOptions) { + super({ prisma: opts.prisma, engine: opts.engine }); + + this.type = opts.type; + this.name = opts.name; + this.workerGroupId = opts.workerGroupId; + this.workerInstanceId = opts.workerInstanceId; + this.masterQueue = opts.masterQueue; + this.environment = opts.environment; + this.deploymentId = opts.deploymentId; + this.backgroundWorkerId = opts.backgroundWorkerId; + } + + async connect(metadata: Record): Promise { + await this._prisma.workerInstance.update({ + where: { + id: this.workerInstanceId, + }, + data: { + metadata, + }, + }); + } + + async dequeue(maxRunCount = 10): Promise { + if (this.type === WorkerInstanceGroupType.MANAGED) { + return await this._engine.dequeueFromMasterQueue({ + consumerId: this.workerInstanceId, + masterQueue: this.masterQueue, + maxRunCount, + }); + } + + if (!this.environment || !this.deploymentId || !this.backgroundWorkerId) { + logger.error("[AuthenticatedWorkerInstance] Missing environment or deployment", { + ...this.toJSON(), + }); + return []; + } + + await this._prisma.workerInstance.update({ + where: { + id: this.workerInstanceId, + }, + data: { + lastDequeueAt: new Date(), + }, + }); + + if (this.isLatestDeployment) { + return await this._engine.dequeueFromEnvironmentMasterQueue({ + consumerId: this.workerInstanceId, + environmentId: this.environment.id, + maxRunCount, + backgroundWorkerId: this.backgroundWorkerId, + }); + } + + return await this._engine.dequeueFromBackgroundWorkerMasterQueue({ + consumerId: this.workerInstanceId, + backgroundWorkerId: this.backgroundWorkerId, + maxRunCount, + }); + } + + /** Allows managed workers to dequeue from a specific version */ + async dequeueFromVersion( + backgroundWorkerId: string, + maxRunCount = 1 + ): Promise { + if (this.type !== WorkerInstanceGroupType.MANAGED) { + logger.error("[AuthenticatedWorkerInstance] Worker instance is not managed", { + ...this.toJSON(), + }); + return []; + } + + return await this._engine.dequeueFromBackgroundWorkerMasterQueue({ + consumerId: this.workerInstanceId, + backgroundWorkerId, + maxRunCount, + }); + } + + /** Allows managed workers to dequeue from a specific environment */ + async dequeueFromEnvironment( + backgroundWorkerId: string, + environmentId: string, + maxRunCount = 1 + ): Promise { + if (this.type !== WorkerInstanceGroupType.MANAGED) { + logger.error("[AuthenticatedWorkerInstance] Worker instance is not managed", { + ...this.toJSON(), + }); + return []; + } + + return await this._engine.dequeueFromEnvironmentMasterQueue({ + consumerId: this.workerInstanceId, + backgroundWorkerId, + environmentId, + maxRunCount, + }); + } + + async heartbeatWorkerInstance() { + await this._prisma.workerInstance.update({ + where: { + id: this.workerInstanceId, + }, + data: { + lastHeartbeatAt: new Date(), + }, + }); + } + + async heartbeatRun({ + runFriendlyId, + snapshotFriendlyId, + }: { + runFriendlyId: string; + snapshotFriendlyId: string; + }): Promise { + return await this._engine.heartbeatRun({ + runId: fromFriendlyId(runFriendlyId), + snapshotId: fromFriendlyId(snapshotFriendlyId), + }); + } + + async startRunAttempt({ + runFriendlyId, + snapshotFriendlyId, + isWarmStart, + }: { + runFriendlyId: string; + snapshotFriendlyId: string; + isWarmStart?: boolean; + }): Promise< + StartRunAttemptResult & { + envVars: Record; + } + > { + const engineResult = await this._engine.startRunAttempt({ + runId: fromFriendlyId(runFriendlyId), + snapshotId: fromFriendlyId(snapshotFriendlyId), + isWarmStart, + }); + + const defaultMachinePreset = machinePresetFromName(defaultMachine); + + const environment = + this.environment ?? + (await this._prisma.runtimeEnvironment.findUnique({ + where: { + id: engineResult.execution.environment.id, + }, + })); + + const envVars = environment + ? await this.getEnvVars( + environment, + engineResult.run.id, + engineResult.execution.machine ?? defaultMachinePreset + ) + : {}; + + return { + ...engineResult, + envVars, + }; + } + + async completeRunAttempt({ + runFriendlyId, + snapshotFriendlyId, + completion, + }: { + runFriendlyId: string; + snapshotFriendlyId: string; + completion: TaskRunExecutionResult; + }): Promise { + return await this._engine.completeRunAttempt({ + runId: fromFriendlyId(runFriendlyId), + snapshotId: fromFriendlyId(snapshotFriendlyId), + completion, + }); + } + + async waitForDuration({ + runFriendlyId, + snapshotFriendlyId, + date, + }: { + runFriendlyId: string; + snapshotFriendlyId: string; + date: Date; + }): Promise { + return await this._engine.waitForDuration({ + runId: fromFriendlyId(runFriendlyId), + snapshotId: fromFriendlyId(snapshotFriendlyId), + date, + }); + } + + async getLatestSnapshot({ runFriendlyId }: { runFriendlyId: string }) { + return await this._engine.getRunExecutionData({ + runId: fromFriendlyId(runFriendlyId), + }); + } + + toJSON(): WorkerGroupTokenAuthenticationResponse { + if (this.type === WorkerInstanceGroupType.MANAGED) { + return { + type: WorkerInstanceGroupType.MANAGED, + name: this.name, + workerGroupId: this.workerGroupId, + workerInstanceId: this.workerInstanceId, + masterQueue: this.masterQueue, + }; + } + + return { + type: WorkerInstanceGroupType.UNMANAGED, + name: this.name, + workerGroupId: this.workerGroupId, + workerInstanceId: this.workerInstanceId, + masterQueue: this.masterQueue, + environmentId: this.environment?.id!, + deploymentId: this.deploymentId!, + }; + } + + private async getEnvVars( + environment: RuntimeEnvironment, + runId: string, + machinePreset: MachinePreset + ): Promise> { + const variables = await resolveVariablesForEnvironment(environment); + + const jwt = await generateJWTTokenForEnvironment(environment, { + run_id: runId, + machine_preset: machinePreset.name, + }); + + variables.push( + ...[ + { key: "TRIGGER_JWT", value: jwt }, + { key: "TRIGGER_RUN_ID", value: runId }, + { key: "TRIGGER_MACHINE_PRESET", value: machinePreset.name }, + ] + ); + + return variables.reduce((acc: Record, curr) => { + acc[curr.key] = curr.value; + return acc; + }, {}); + } +} + +export type WorkerGroupTokenAuthenticationResponse = + | { + type: typeof WorkerInstanceGroupType.MANAGED; + name: string; + workerGroupId: string; + workerInstanceId: string; + masterQueue: string; + } + | { + type: typeof WorkerInstanceGroupType.UNMANAGED; + name: string; + workerGroupId: string; + workerInstanceId: string; + masterQueue: string; + environmentId: string; + deploymentId: string; + }; diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 9cb11c3d4d..986ae25e31 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -49,6 +49,7 @@ "@electric-sql/react": "^0.3.5", "@headlessui/react": "^1.7.8", "@heroicons/react": "^2.0.12", + "@internal/run-engine": "workspace:*", "@internal/zod-worker": "workspace:*", "@internationalized/date": "^3.5.1", "@lezer/highlight": "^1.1.6", @@ -100,6 +101,7 @@ "@trigger.dev/otlp-importer": "workspace:*", "@trigger.dev/platform": "1.0.14", "@trigger.dev/sdk": "workspace:*", + "@trigger.dev/worker": "workspace:*", "@trigger.dev/yalt": "npm:@trigger.dev/yalt", "@types/pg": "8.6.6", "@uiw/react-codemirror": "^4.19.5", @@ -206,7 +208,6 @@ "@types/lodash.omit": "^4.5.7", "@types/marked": "^4.0.3", "@types/morgan": "^1.9.3", - "@types/node": "^18.11.15", "@types/node-fetch": "^2.6.2", "@types/prismjs": "^1.26.0", "@types/qs": "^6.9.7", @@ -248,7 +249,6 @@ "tailwindcss": "3.4.1", "ts-node": "^10.7.0", "tsconfig-paths": "^3.14.1", - "typescript": "^5.1.6", "vite-tsconfig-paths": "^4.0.5", "vitest": "^1.4.0" }, diff --git a/apps/webapp/prisma/populate.ts b/apps/webapp/prisma/populate.ts index 6b3f277d39..fb31a1e978 100644 --- a/apps/webapp/prisma/populate.ts +++ b/apps/webapp/prisma/populate.ts @@ -4,12 +4,212 @@ // 2. pnpm run db:populate -- --projectRef=proj_liazlkfgmfcusswwgohl --taskIdentifier=child-task --runCount=100000 import { generateFriendlyId } from "~/v3/friendlyIdentifiers"; import { prisma } from "../app/db.server"; +import { createHash } from "crypto"; +import { + BackgroundWorker, + BackgroundWorkerTask, + RuntimeEnvironmentType, + WorkerInstanceGroupType, +} from "@trigger.dev/database"; +import { nanoid } from "nanoid"; async function populate() { if (process.env.NODE_ENV !== "development") { return; } + const project = await getProject(); + + await generateRuns(project); + await createWorkerGroup(project); + const { worker, tasks } = await createBackgroundWorker(project, getEnvTypeFromArg()); + await createWorkerDeployment(project, worker, getEnvTypeFromArg()); +} + +function getEnvironment( + project: ProjectWithEnvironment, + envType: RuntimeEnvironmentType = "PRODUCTION" +) { + const env = project.environments.find((e) => e.type === envType); + + if (!env) { + throw new Error(`No environment of type "${envType}" found for project ${project.id}`); + } + + return env; +} + +async function createWorkerDeployment( + project: ProjectWithEnvironment, + worker: BackgroundWorker, + envType: RuntimeEnvironmentType = "PRODUCTION" +) { + const env = getEnvironment(project, envType); + const deploymentId = `cm3c821sk00032v6is7ufqy3d-${env.slug}`; + + if (env.type === "DEVELOPMENT") { + console.warn("Skipping deployment creation for development environment"); + return; + } + + let deployment = await prisma.workerDeployment.findUnique({ + where: { + id: deploymentId, + }, + }); + + if (deployment) { + console.log(`Deployment "${deploymentId}" already exists`); + return deployment; + } + + const firstOrgMember = project.organization.members[0]; + + deployment = await prisma.workerDeployment.create({ + data: { + id: deploymentId, + friendlyId: generateFriendlyId("deployment"), + contentHash: worker.contentHash, + version: worker.version, + shortCode: nanoid(8), + imageReference: `trigger/${project.externalRef}:${worker.version}.${env.slug}`, + status: "DEPLOYING", + projectId: project.id, + environmentId: env.id, + workerId: worker.id, + triggeredById: firstOrgMember.userId, + }, + }); + + console.log(`Created deployment "${deploymentId}"`); + + return deployment; +} + +async function createBackgroundWorker( + project: ProjectWithEnvironment, + envType: RuntimeEnvironmentType = "PRODUCTION" +) { + const env = getEnvironment(project, envType); + const taskIdentifier = "seed-task"; + const backgroundWorkerId = `cm3c8fmiv00042v6imoqwxst1-${env.slug}`; + + let worker = await prisma.backgroundWorker.findUnique({ + where: { + id: backgroundWorkerId, + }, + include: { + tasks: true, + }, + }); + + if (worker) { + console.log(`Worker "${backgroundWorkerId}" already exists`); + + return { + worker, + tasks: worker.tasks, + }; + } + + worker = await prisma.backgroundWorker.create({ + data: { + id: backgroundWorkerId, + friendlyId: generateFriendlyId("worker"), + contentHash: "hash", + projectId: project.id, + runtimeEnvironmentId: env.id, + version: "20241111.1", + metadata: {}, + }, + include: { + tasks: true, + }, + }); + + console.log(`Created worker "${backgroundWorkerId}"`); + + const taskIdentifiers = Array.isArray(taskIdentifier) ? taskIdentifier : [taskIdentifier]; + + const tasks: BackgroundWorkerTask[] = []; + + for (const identifier of taskIdentifiers) { + const task = await prisma.backgroundWorkerTask.create({ + data: { + friendlyId: generateFriendlyId("task"), + slug: identifier, + filePath: `/trigger/${identifier}.ts`, + exportName: identifier, + workerId: worker.id, + runtimeEnvironmentId: env.id, + projectId: project.id, + }, + }); + + tasks.push(task); + } + + return { + worker, + tasks, + }; +} + +async function createWorkerGroup(project: ProjectWithEnvironment) { + const workerGroupName = "seed-unmanaged"; + const rawToken = "tr_wgt_15480aa1712cae4b8db8c7a49707d69d"; + + const existingWorkerGroup = await prisma.workerInstanceGroup.findFirst({ + where: { + projectId: project.id, + name: workerGroupName, + }, + }); + + if (existingWorkerGroup) { + console.log(`Worker group "${workerGroupName}" already exists`); + + await setAsDefaultWorkerGroup(project, existingWorkerGroup.id); + + return existingWorkerGroup; + } + + const token = await prisma.workerGroupToken.create({ + data: { + tokenHash: createHash("sha256").update(rawToken).digest("hex"), + }, + }); + + const workerGroup = await prisma.workerInstanceGroup.create({ + data: { + projectId: project.id, + organizationId: project.organizationId, + type: WorkerInstanceGroupType.UNMANAGED, + masterQueue: `${project.id}-${workerGroupName}`, + tokenId: token.id, + description: "Seeded worker group", + name: workerGroupName, + }, + }); + + await setAsDefaultWorkerGroup(project, workerGroup.id); + + return workerGroup; +} + +async function setAsDefaultWorkerGroup(project: ProjectWithEnvironment, workerGroupId: string) { + // Set as default worker group + await prisma.project.update({ + where: { + id: project.id, + }, + data: { + defaultWorkerGroupId: workerGroupId, + }, + }); +} + +async function getProject() { const projectRef = getArg("projectRef"); if (!projectRef) { throw new Error("projectRef is required"); @@ -18,15 +218,27 @@ async function populate() { const project = await prisma.project.findUnique({ include: { environments: true, + organization: { + include: { + members: true, + }, + }, }, where: { externalRef: projectRef, }, }); + if (!project) { throw new Error("Project not found"); } + return project; +} + +type ProjectWithEnvironment = Awaited>; + +async function generateRuns(project: ProjectWithEnvironment) { const taskIdentifier = getArg("taskIdentifier"); if (!taskIdentifier) { throw new Error("taskIdentifier is required"); @@ -74,6 +286,25 @@ async function populate() { console.log(`Added ${runs.count} runs`); } +function getEnvTypeFromArg(): RuntimeEnvironmentType { + const env = getArg("env"); + + if (!env) { + return RuntimeEnvironmentType.PRODUCTION; + } + + switch (env) { + case "dev": + return RuntimeEnvironmentType.DEVELOPMENT; + case "prod": + return RuntimeEnvironmentType.PRODUCTION; + case "stg": + return RuntimeEnvironmentType.STAGING; + default: + throw new Error(`Invalid environment: ${env}`); + } +} + function getArg(name: string) { const args = process.argv.slice(2); diff --git a/apps/webapp/remix.config.js b/apps/webapp/remix.config.js index d2417a3eb5..32af713449 100644 --- a/apps/webapp/remix.config.js +++ b/apps/webapp/remix.config.js @@ -11,6 +11,7 @@ module.exports = { /^remix-utils.*/, "marked", "axios", + "@internal/redis-worker", "@trigger.dev/core", "@trigger.dev/sdk", "@trigger.dev/platform", diff --git a/apps/webapp/tsconfig.json b/apps/webapp/tsconfig.json index af3d25eb48..b402860b1b 100644 --- a/apps/webapp/tsconfig.json +++ b/apps/webapp/tsconfig.json @@ -31,10 +31,14 @@ "@trigger.dev/yalt/*": ["../../packages/yalt/src/*"], "@trigger.dev/otlp-importer": ["../../internal-packages/otlp-importer/src/index"], "@trigger.dev/otlp-importer/*": ["../../internal-packages/otlp-importer/src/*"], + "@trigger.dev/worker": ["../../packages/worker/src/index"], + "@trigger.dev/worker/*": ["../../packages/worker/src/*"], "emails": ["../../internal-packages/emails/src/index"], "emails/*": ["../../internal-packages/emails/src/*"], "@internal/zod-worker": ["../../internal-packages/zod-worker/src/index"], - "@internal/zod-worker/*": ["../../internal-packages/zod-worker/src/*"] + "@internal/zod-worker/*": ["../../internal-packages/zod-worker/src/*"], + "@internal/run-engine": ["../../internal-packages/run-engine/src/index"], + "@internal/run-engine/*": ["../../internal-packages/run-engine/src/*"] }, "noEmit": true } diff --git a/internal-packages/database/package.json b/internal-packages/database/package.json index 41afe0e5ef..427cdf1829 100644 --- a/internal-packages/database/package.json +++ b/internal-packages/database/package.json @@ -5,8 +5,7 @@ "main": "./src/index.ts", "types": "./src/index.ts", "dependencies": { - "@prisma/client": "5.4.1", - "typescript": "^4.8.4" + "@prisma/client": "5.4.1" }, "devDependencies": { "prisma": "5.4.1" diff --git a/internal-packages/database/prisma/migrations/20250103152909_add_run_engine_v2/migration.sql b/internal-packages/database/prisma/migrations/20250103152909_add_run_engine_v2/migration.sql new file mode 100644 index 0000000000..5161459ff5 --- /dev/null +++ b/internal-packages/database/prisma/migrations/20250103152909_add_run_engine_v2/migration.sql @@ -0,0 +1,288 @@ +-- CreateEnum +CREATE TYPE "RunEngineVersion" AS ENUM ('V1', 'V2'); + +-- CreateEnum +CREATE TYPE "TaskRunExecutionStatus" AS ENUM ('RUN_CREATED', 'QUEUED', 'PENDING_EXECUTING', 'EXECUTING', 'EXECUTING_WITH_WAITPOINTS', 'BLOCKED_BY_WAITPOINTS', 'PENDING_CANCEL', 'FINISHED'); + +-- CreateEnum +CREATE TYPE "TaskRunCheckpointType" AS ENUM ('DOCKER', 'KUBERNETES'); + +-- CreateEnum +CREATE TYPE "WaitpointType" AS ENUM ('RUN', 'DATETIME', 'MANUAL'); + +-- CreateEnum +CREATE TYPE "WaitpointStatus" AS ENUM ('PENDING', 'COMPLETED'); + +-- CreateEnum +CREATE TYPE "WorkerInstanceGroupType" AS ENUM ('MANAGED', 'UNMANAGED'); + +-- CreateEnum +CREATE TYPE "WorkerDeploymentType" AS ENUM ('MANAGED', 'UNMANAGED', 'V1'); + +-- AlterTable +ALTER TABLE "BackgroundWorker" ADD COLUMN "workerGroupId" TEXT; + +-- AlterTable +ALTER TABLE "Project" ADD COLUMN "defaultWorkerGroupId" TEXT, +ADD COLUMN "engine" "RunEngineVersion" NOT NULL DEFAULT 'V1'; + +-- AlterTable +ALTER TABLE "TaskEvent" ADD COLUMN "isDebug" BOOLEAN NOT NULL DEFAULT false; + +-- AlterTable +ALTER TABLE "TaskRun" ADD COLUMN "attemptNumber" INTEGER, +ADD COLUMN "engine" "RunEngineVersion" NOT NULL DEFAULT 'V1', +ADD COLUMN "firstAttemptStartedAt" TIMESTAMP(3), +ADD COLUMN "masterQueue" TEXT NOT NULL DEFAULT 'main', +ADD COLUMN "priorityMs" INTEGER NOT NULL DEFAULT 0, +ADD COLUMN "secondaryMasterQueue" TEXT; + +-- AlterTable +ALTER TABLE "WorkerDeployment" ADD COLUMN "type" "WorkerDeploymentType" NOT NULL DEFAULT 'V1'; + +-- CreateTable +CREATE TABLE "TaskRunExecutionSnapshot" ( + "id" TEXT NOT NULL, + "engine" "RunEngineVersion" NOT NULL DEFAULT 'V2', + "executionStatus" "TaskRunExecutionStatus" NOT NULL, + "description" TEXT NOT NULL, + "isValid" BOOLEAN NOT NULL DEFAULT true, + "error" TEXT, + "runId" TEXT NOT NULL, + "runStatus" "TaskRunStatus" NOT NULL, + "attemptNumber" INTEGER, + "checkpointId" TEXT, + "workerId" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "lastHeartbeatAt" TIMESTAMP(3), + + CONSTRAINT "TaskRunExecutionSnapshot_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "TaskRunCheckpoint" ( + "id" TEXT NOT NULL, + "friendlyId" TEXT NOT NULL, + "type" "TaskRunCheckpointType" NOT NULL, + "location" TEXT NOT NULL, + "imageRef" TEXT NOT NULL, + "reason" TEXT, + "metadata" TEXT, + "projectId" TEXT NOT NULL, + "runtimeEnvironmentId" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "TaskRunCheckpoint_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "Waitpoint" ( + "id" TEXT NOT NULL, + "friendlyId" TEXT NOT NULL, + "type" "WaitpointType" NOT NULL, + "status" "WaitpointStatus" NOT NULL DEFAULT 'PENDING', + "completedAt" TIMESTAMP(3), + "idempotencyKey" TEXT NOT NULL, + "userProvidedIdempotencyKey" BOOLEAN NOT NULL, + "inactiveIdempotencyKey" TEXT, + "completedByTaskRunId" TEXT, + "completedAfter" TIMESTAMP(3), + "output" TEXT, + "outputType" TEXT NOT NULL DEFAULT 'application/json', + "outputIsError" BOOLEAN NOT NULL DEFAULT false, + "projectId" TEXT NOT NULL, + "environmentId" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "Waitpoint_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "TaskRunWaitpoint" ( + "id" TEXT NOT NULL, + "taskRunId" TEXT NOT NULL, + "waitpointId" TEXT NOT NULL, + "projectId" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "TaskRunWaitpoint_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "FeatureFlag" ( + "id" TEXT NOT NULL, + "key" TEXT NOT NULL, + "value" JSONB, + + CONSTRAINT "FeatureFlag_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "WorkerInstance" ( + "id" TEXT NOT NULL, + "name" TEXT NOT NULL, + "resourceIdentifier" TEXT NOT NULL, + "metadata" JSONB, + "workerGroupId" TEXT NOT NULL, + "organizationId" TEXT, + "projectId" TEXT, + "environmentId" TEXT, + "deploymentId" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "lastDequeueAt" TIMESTAMP(3), + "lastHeartbeatAt" TIMESTAMP(3), + + CONSTRAINT "WorkerInstance_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "WorkerInstanceGroup" ( + "id" TEXT NOT NULL, + "type" "WorkerInstanceGroupType" NOT NULL, + "name" TEXT NOT NULL, + "masterQueue" TEXT NOT NULL, + "description" TEXT, + "hidden" BOOLEAN NOT NULL DEFAULT false, + "tokenId" TEXT NOT NULL, + "organizationId" TEXT, + "projectId" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "WorkerInstanceGroup_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "WorkerGroupToken" ( + "id" TEXT NOT NULL, + "tokenHash" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "WorkerGroupToken_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "_completedWaitpoints" ( + "A" TEXT NOT NULL, + "B" TEXT NOT NULL +); + +-- CreateIndex +CREATE INDEX "TaskRunExecutionSnapshot_runId_isValid_createdAt_idx" ON "TaskRunExecutionSnapshot"("runId", "isValid", "createdAt" DESC); + +-- CreateIndex +CREATE UNIQUE INDEX "TaskRunCheckpoint_friendlyId_key" ON "TaskRunCheckpoint"("friendlyId"); + +-- CreateIndex +CREATE UNIQUE INDEX "Waitpoint_friendlyId_key" ON "Waitpoint"("friendlyId"); + +-- CreateIndex +CREATE UNIQUE INDEX "Waitpoint_completedByTaskRunId_key" ON "Waitpoint"("completedByTaskRunId"); + +-- CreateIndex +CREATE UNIQUE INDEX "Waitpoint_environmentId_idempotencyKey_key" ON "Waitpoint"("environmentId", "idempotencyKey"); + +-- CreateIndex +CREATE INDEX "TaskRunWaitpoint_taskRunId_idx" ON "TaskRunWaitpoint"("taskRunId"); + +-- CreateIndex +CREATE INDEX "TaskRunWaitpoint_waitpointId_idx" ON "TaskRunWaitpoint"("waitpointId"); + +-- CreateIndex +CREATE UNIQUE INDEX "TaskRunWaitpoint_taskRunId_waitpointId_key" ON "TaskRunWaitpoint"("taskRunId", "waitpointId"); + +-- CreateIndex +CREATE UNIQUE INDEX "FeatureFlag_key_key" ON "FeatureFlag"("key"); + +-- CreateIndex +CREATE UNIQUE INDEX "WorkerInstance_workerGroupId_resourceIdentifier_key" ON "WorkerInstance"("workerGroupId", "resourceIdentifier"); + +-- CreateIndex +CREATE UNIQUE INDEX "WorkerInstanceGroup_masterQueue_key" ON "WorkerInstanceGroup"("masterQueue"); + +-- CreateIndex +CREATE UNIQUE INDEX "WorkerInstanceGroup_tokenId_key" ON "WorkerInstanceGroup"("tokenId"); + +-- CreateIndex +CREATE UNIQUE INDEX "WorkerGroupToken_tokenHash_key" ON "WorkerGroupToken"("tokenHash"); + +-- CreateIndex +CREATE UNIQUE INDEX "_completedWaitpoints_AB_unique" ON "_completedWaitpoints"("A", "B"); + +-- CreateIndex +CREATE INDEX "_completedWaitpoints_B_index" ON "_completedWaitpoints"("B"); + +-- AddForeignKey +ALTER TABLE "Project" ADD CONSTRAINT "Project_defaultWorkerGroupId_fkey" FOREIGN KEY ("defaultWorkerGroupId") REFERENCES "WorkerInstanceGroup"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "BackgroundWorker" ADD CONSTRAINT "BackgroundWorker_workerGroupId_fkey" FOREIGN KEY ("workerGroupId") REFERENCES "WorkerInstanceGroup"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "TaskRunExecutionSnapshot" ADD CONSTRAINT "TaskRunExecutionSnapshot_runId_fkey" FOREIGN KEY ("runId") REFERENCES "TaskRun"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "TaskRunExecutionSnapshot" ADD CONSTRAINT "TaskRunExecutionSnapshot_checkpointId_fkey" FOREIGN KEY ("checkpointId") REFERENCES "TaskRunCheckpoint"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "TaskRunExecutionSnapshot" ADD CONSTRAINT "TaskRunExecutionSnapshot_workerId_fkey" FOREIGN KEY ("workerId") REFERENCES "WorkerInstance"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "TaskRunCheckpoint" ADD CONSTRAINT "TaskRunCheckpoint_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "TaskRunCheckpoint" ADD CONSTRAINT "TaskRunCheckpoint_runtimeEnvironmentId_fkey" FOREIGN KEY ("runtimeEnvironmentId") REFERENCES "RuntimeEnvironment"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Waitpoint" ADD CONSTRAINT "Waitpoint_completedByTaskRunId_fkey" FOREIGN KEY ("completedByTaskRunId") REFERENCES "TaskRun"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Waitpoint" ADD CONSTRAINT "Waitpoint_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Waitpoint" ADD CONSTRAINT "Waitpoint_environmentId_fkey" FOREIGN KEY ("environmentId") REFERENCES "RuntimeEnvironment"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "TaskRunWaitpoint" ADD CONSTRAINT "TaskRunWaitpoint_taskRunId_fkey" FOREIGN KEY ("taskRunId") REFERENCES "TaskRun"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "TaskRunWaitpoint" ADD CONSTRAINT "TaskRunWaitpoint_waitpointId_fkey" FOREIGN KEY ("waitpointId") REFERENCES "Waitpoint"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "TaskRunWaitpoint" ADD CONSTRAINT "TaskRunWaitpoint_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "WorkerInstance" ADD CONSTRAINT "WorkerInstance_workerGroupId_fkey" FOREIGN KEY ("workerGroupId") REFERENCES "WorkerInstanceGroup"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "WorkerInstance" ADD CONSTRAINT "WorkerInstance_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "WorkerInstance" ADD CONSTRAINT "WorkerInstance_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "WorkerInstance" ADD CONSTRAINT "WorkerInstance_environmentId_fkey" FOREIGN KEY ("environmentId") REFERENCES "RuntimeEnvironment"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "WorkerInstance" ADD CONSTRAINT "WorkerInstance_deploymentId_fkey" FOREIGN KEY ("deploymentId") REFERENCES "WorkerDeployment"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "WorkerInstanceGroup" ADD CONSTRAINT "WorkerInstanceGroup_tokenId_fkey" FOREIGN KEY ("tokenId") REFERENCES "WorkerGroupToken"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "WorkerInstanceGroup" ADD CONSTRAINT "WorkerInstanceGroup_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "WorkerInstanceGroup" ADD CONSTRAINT "WorkerInstanceGroup_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "_completedWaitpoints" ADD CONSTRAINT "_completedWaitpoints_A_fkey" FOREIGN KEY ("A") REFERENCES "TaskRunExecutionSnapshot"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "_completedWaitpoints" ADD CONSTRAINT "_completedWaitpoints_B_fkey" FOREIGN KEY ("B") REFERENCES "Waitpoint"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index ceaa349bb0..4860f6d61b 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -148,6 +148,8 @@ model Organization { integrations Integration[] sources TriggerSource[] organizationIntegrations OrganizationIntegration[] + workerGroups WorkerInstanceGroup[] + workerInstances WorkerInstance[] } model ExternalAccount { @@ -419,6 +421,9 @@ model RuntimeEnvironment { currentSession RuntimeEnvironmentSession? @relation("currentSession", fields: [currentSessionId], references: [id], onDelete: SetNull, onUpdate: Cascade) currentSessionId String? taskRunNumberCounter TaskRunNumberCounter[] + taskRunCheckpoints TaskRunCheckpoint[] + waitpoints Waitpoint[] + workerInstances WorkerInstance[] @@unique([projectId, slug, orgMemberId]) @@unique([projectId, shortcode]) @@ -445,10 +450,17 @@ model Project { updatedAt DateTime @updatedAt deletedAt DateTime? - version ProjectVersion @default(V2) + version ProjectVersion @default(V2) + engine RunEngineVersion @default(V1) builderProjectId String? + workerGroups WorkerInstanceGroup[] + workers WorkerInstance[] + + defaultWorkerGroup WorkerInstanceGroup? @relation("ProjectDefaultWorkerGroup", fields: [defaultWorkerGroupId], references: [id]) + defaultWorkerGroupId String? + environments RuntimeEnvironment[] endpoints Endpoint[] jobs Job[] @@ -473,6 +485,9 @@ model Project { alertStorages ProjectAlertStorage[] bulkActionGroups BulkActionGroup[] BackgroundWorkerFile BackgroundWorkerFile[] + waitpoints Waitpoint[] + taskRunWaitpoints TaskRunWaitpoint[] + taskRunCheckpoints TaskRunCheckpoint[] } enum ProjectVersion { @@ -1580,6 +1595,9 @@ model BackgroundWorker { deployment WorkerDeployment? + workerGroup WorkerInstanceGroup? @relation(fields: [workerGroupId], references: [id], onDelete: SetNull, onUpdate: Cascade) + workerGroupId String? + supportsLazyAttempts Boolean @default(false) @@unique([projectId, runtimeEnvironmentId, version]) @@ -1659,6 +1677,8 @@ model TaskRun { number Int @default(0) friendlyId String @unique + engine RunEngineVersion @default(V1) + status TaskRunStatus @default(PENDING) idempotencyKey String? @@ -1681,8 +1701,16 @@ model TaskRun { project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) projectId String + // The specific queue this run is in queue String + /// The main queue that this run is part of + masterQueue String @default("main") + secondaryMasterQueue String? + + /// From engine v2+ this will be defined after a run has been dequeued (starting at 1) + attemptNumber Int? + createdAt DateTime @default(now()) updatedAt DateTime @updatedAt @@ -1703,6 +1731,9 @@ model TaskRun { completedAt DateTime? machinePreset String? + /// Run Engine 2.0+ + firstAttemptStartedAt DateTime? + usageDurationMs Int @default(0) costInCents Float @default(0) baseCostInCents Float @default(0) @@ -1714,6 +1745,10 @@ model TaskRun { lockedToVersion BackgroundWorker? @relation(fields: [lockedToVersionId], references: [id]) lockedToVersionId String? + /// The "priority" of the run. This is just a negative offset in ms for the queue timestamp + /// E.g. a value of 60_000 would put the run into the queue 60s ago. + priorityMs Int @default(0) + concurrencyKey String? delayUntil DateTime? @@ -1725,9 +1760,16 @@ model TaskRun { /// optional token that can be used to authenticate the task run oneTimeUseToken String? + ///When this run is finished, the waitpoint will be marked as completed + associatedWaitpoint Waitpoint? + + ///If there are any blocked waitpoints, the run won't be executed + blockedByWaitpoints TaskRunWaitpoint[] + batchItems BatchTaskRunItem[] dependency TaskRunDependency? CheckpointRestoreEvent CheckpointRestoreEvent[] + executionSnapshots TaskRunExecutionSnapshot[] alerts ProjectAlert[] @@ -1872,6 +1914,288 @@ enum TaskRunStatus { TIMED_OUT } +enum RunEngineVersion { + /// The original version that uses marqs v1 and Graphile + V1 + V2 +} + +/// Used by the RunEngine during TaskRun execution +/// It has the required information to transactionally progress a run through states, +/// and prevent side effects like heartbeats failing a run that has progressed. +/// It is optimised for performance and is designed to be cleared at some point, +/// so there are no cascading relationships to other models. +model TaskRunExecutionSnapshot { + id String @id @default(cuid()) + + /// This should always be 2+ (V1 didn't use the run engine or snapshots) + engine RunEngineVersion @default(V2) + + /// The execution status + executionStatus TaskRunExecutionStatus + /// For debugging + description String + + /// We store invalid snapshots as a record of the run state when we tried to move + isValid Boolean @default(true) + error String? + + /// Run + runId String + run TaskRun @relation(fields: [runId], references: [id]) + runStatus TaskRunStatus + + /// This is the current run attempt number. Users can define how many attempts they want for a run. + attemptNumber Int? + + /// Waitpoints that have been completed for this execution + completedWaitpoints Waitpoint[] @relation("completedWaitpoints") + + /// Checkpoint + checkpointId String? + checkpoint TaskRunCheckpoint? @relation(fields: [checkpointId], references: [id]) + + /// Worker + workerId String? + worker WorkerInstance? @relation(fields: [workerId], references: [id]) + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + lastHeartbeatAt DateTime? + + /// Used to get the latest valid snapshot quickly + @@index([runId, isValid, createdAt(sort: Desc)]) +} + +enum TaskRunExecutionStatus { + /// Run has been created + RUN_CREATED + /// Run is in the RunQueue + QUEUED + /// Run has been pulled from the queue, but isn't executing yet + PENDING_EXECUTING + /// Run is executing on a worker + EXECUTING + /// Run is executing on a worker but is waiting for waitpoints to complete + EXECUTING_WITH_WAITPOINTS + /// Run is not executing and is waiting for waitpoints to complete + BLOCKED_BY_WAITPOINTS + /// Run has been scheduled for cancellation + PENDING_CANCEL + /// Run is finished (success of failure) + FINISHED +} + +model TaskRunCheckpoint { + id String @id @default(cuid()) + + friendlyId String @unique + + type TaskRunCheckpointType + location String + imageRef String + reason String? + metadata String? + + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) + projectId String + + runtimeEnvironment RuntimeEnvironment @relation(fields: [runtimeEnvironmentId], references: [id], onDelete: Cascade, onUpdate: Cascade) + runtimeEnvironmentId String + + executionSnapshot TaskRunExecutionSnapshot[] + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + +enum TaskRunCheckpointType { + DOCKER + KUBERNETES +} + +/// A Waitpoint blocks a run from continuing until it's completed +/// If there's a waitpoint blocking a run, it shouldn't be in the queue +model Waitpoint { + id String @id @default(cuid()) + + friendlyId String @unique + + type WaitpointType + status WaitpointStatus @default(PENDING) + + completedAt DateTime? + + /// If it's an Event type waitpoint, this is the event. It can also be provided for the DATETIME type + idempotencyKey String + /// If this is true then we can show it in the dashboard/return it from the SDK + userProvidedIdempotencyKey Boolean + + //todo + /// Will automatically deactivate the idempotencyKey when the waitpoint is completed + /// "Deactivating" means moving it to the inactiveIdempotencyKey field and generating a random new one for the main column + /// deactivateIdempotencyKeyWhenCompleted Boolean @default(false) + + /// If an idempotencyKey is no longer active, we store it here and generate a new one for the idempotencyKey field. + /// Clearing an idempotencyKey is useful for debounce or cancelling child runs. + /// This is a workaround because Prisma doesn't support partial indexes. + inactiveIdempotencyKey String? + + /// If it's a RUN type waitpoint, this is the associated run + completedByTaskRunId String? @unique + completedByTaskRun TaskRun? @relation(fields: [completedByTaskRunId], references: [id], onDelete: SetNull) + + /// If it's a DATETIME type waitpoint, this is the date + completedAfter DateTime? + + /// The runs this waitpoint is blocking + blockingTaskRuns TaskRunWaitpoint[] + + /// When a waitpoint is complete + completedExecutionSnapshots TaskRunExecutionSnapshot[] @relation("completedWaitpoints") + + /// When completed, an output can be stored here + output String? + outputType String @default("application/json") + outputIsError Boolean @default(false) + + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) + projectId String + + environment RuntimeEnvironment @relation(fields: [environmentId], references: [id], onDelete: Cascade, onUpdate: Cascade) + environmentId String + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([environmentId, idempotencyKey]) +} + +enum WaitpointType { + RUN + DATETIME + MANUAL +} + +enum WaitpointStatus { + PENDING + COMPLETED +} + +model TaskRunWaitpoint { + id String @id @default(cuid()) + + taskRun TaskRun @relation(fields: [taskRunId], references: [id]) + taskRunId String + + waitpoint Waitpoint @relation(fields: [waitpointId], references: [id]) + waitpointId String + + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) + projectId String + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([taskRunId, waitpointId]) + @@index([taskRunId]) + @@index([waitpointId]) +} + +model FeatureFlag { + id String @id @default(cuid()) + + key String @unique + value Json? +} + +model WorkerInstance { + id String @id @default(cuid()) + + /// For example "worker-1" + name String + + /// If managed, it will default to the name, e.g. "worker-1" + /// If unmanged, it will be prefixed with the deployment ID e.g. "deploy-123-worker-1" + resourceIdentifier String + + metadata Json? + + workerGroup WorkerInstanceGroup @relation(fields: [workerGroupId], references: [id]) + workerGroupId String + + TaskRunExecutionSnapshot TaskRunExecutionSnapshot[] + + organization Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade, onUpdate: Cascade) + organizationId String? + + project Project? @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) + projectId String? + + environment RuntimeEnvironment? @relation(fields: [environmentId], references: [id], onDelete: Cascade, onUpdate: Cascade) + environmentId String? + + deployment WorkerDeployment? @relation(fields: [deploymentId], references: [id], onDelete: SetNull, onUpdate: Cascade) + deploymentId String? + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + lastDequeueAt DateTime? + lastHeartbeatAt DateTime? + + @@unique([workerGroupId, resourceIdentifier]) +} + +enum WorkerInstanceGroupType { + MANAGED + UNMANAGED +} + +model WorkerInstanceGroup { + id String @id @default(cuid()) + type WorkerInstanceGroupType + + /// For example "us-east-1" + name String + + /// If managed, it will default to the name, e.g. "us-east-1" + /// If unmanged, it will be prefixed with the project ID e.g. "project_1-us-east-1" + masterQueue String @unique + + description String? + hidden Boolean @default(false) + + token WorkerGroupToken @relation(fields: [tokenId], references: [id], onDelete: Cascade, onUpdate: Cascade) + tokenId String @unique + + workers WorkerInstance[] + backgroundWorkers BackgroundWorker[] + + defaultForProjects Project[] @relation("ProjectDefaultWorkerGroup") + + organization Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade, onUpdate: Cascade) + organizationId String? + + project Project? @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) + projectId String? + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + +model WorkerGroupToken { + id String @id @default(cuid()) + + tokenHash String @unique + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + workerGroup WorkerInstanceGroup? +} + model TaskRunTag { id String @id @default(cuid()) name String @@ -1932,6 +2256,7 @@ model TaskRunNumberCounter { @@unique([taskIdentifier, environmentId]) } +/// This is not used from engine v2+, attempts use the TaskRunExecutionSnapshot and TaskRun model TaskRunAttempt { id String @id @default(cuid()) number Int @default(0) @@ -2006,6 +2331,7 @@ model TaskEvent { isError Boolean @default(false) isPartial Boolean @default(false) isCancelled Boolean @default(false) + isDebug Boolean @default(false) serviceName String serviceNamespace String @@ -2158,6 +2484,7 @@ model BatchTaskRun { updatedAt DateTime @updatedAt // new columns + /// Friendly IDs runIds String[] @default([]) runCount Int @default(0) payload String? @@ -2187,6 +2514,7 @@ enum BatchTaskRunStatus { COMPLETED } +///Used in engine V1 only model BatchTaskRunItem { id String @id @default(cuid()) @@ -2315,6 +2643,12 @@ enum CheckpointRestoreEventType { RESTORE } +enum WorkerDeploymentType { + MANAGED + UNMANAGED + V1 +} + model WorkerDeployment { id String @id @default(cuid()) @@ -2328,6 +2662,7 @@ model WorkerDeployment { externalBuildData Json? status WorkerDeploymentStatus @default(PENDING) + type WorkerDeploymentType @default(V1) project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) projectId String @@ -2350,8 +2685,9 @@ model WorkerDeployment { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt - promotions WorkerDeploymentPromotion[] - alerts ProjectAlert[] + promotions WorkerDeploymentPromotion[] + alerts ProjectAlert[] + workerInstance WorkerInstance[] @@unique([projectId, shortCode]) @@unique([environmentId, version]) diff --git a/internal-packages/emails/package.json b/internal-packages/emails/package.json index d7fa150fbb..85cb7abe01 100644 --- a/internal-packages/emails/package.json +++ b/internal-packages/emails/package.json @@ -20,10 +20,8 @@ "zod": "3.23.8" }, "devDependencies": { - "@types/node": "^18", "@types/nodemailer": "^6.4.17", - "@types/react": "18.2.69", - "typescript": "^4.9.4" + "@types/react": "18.2.69" }, "engines": { "node": ">=18.0.0" diff --git a/internal-packages/otlp-importer/package.json b/internal-packages/otlp-importer/package.json index 49d2fdbae5..72e46c2f9d 100644 --- a/internal-packages/otlp-importer/package.json +++ b/internal-packages/otlp-importer/package.json @@ -29,8 +29,7 @@ "devDependencies": { "@types/node": "^20", "rimraf": "^3.0.2", - "ts-proto": "^1.167.3", - "typescript": "^5.5.0" + "ts-proto": "^1.167.3" }, "engines": { "node": ">=18.0.0" diff --git a/internal-packages/redis-worker/package.json b/internal-packages/redis-worker/package.json index 33e7bbea42..bf44ab71cb 100644 --- a/internal-packages/redis-worker/package.json +++ b/internal-packages/redis-worker/package.json @@ -11,7 +11,6 @@ "ioredis": "^5.3.2", "lodash.omit": "^4.5.0", "nanoid": "^5.0.7", - "typescript": "^5.5.4", "zod": "3.23.8" }, "devDependencies": { diff --git a/internal-packages/redis-worker/src/index.ts b/internal-packages/redis-worker/src/index.ts new file mode 100644 index 0000000000..a5893efc83 --- /dev/null +++ b/internal-packages/redis-worker/src/index.ts @@ -0,0 +1,2 @@ +export * from "./queue"; +export * from "./worker"; diff --git a/internal-packages/redis-worker/src/queue.ts b/internal-packages/redis-worker/src/queue.ts index 04c08a30d2..4c7fa7e25e 100644 --- a/internal-packages/redis-worker/src/queue.ts +++ b/internal-packages/redis-worker/src/queue.ts @@ -128,7 +128,7 @@ export class SimpleQueue { const dequeuedItems = []; for (const [id, serializedItem] of results) { - const parsedItem = JSON.parse(serializedItem); + const parsedItem = JSON.parse(serializedItem) as any; if (typeof parsedItem.job !== "string") { this.logger.error(`Invalid item in queue`, { queue: this.name, id, item: parsedItem }); continue; diff --git a/internal-packages/redis-worker/src/worker.test.ts b/internal-packages/redis-worker/src/worker.test.ts index a55a653887..de2e78a7b0 100644 --- a/internal-packages/redis-worker/src/worker.test.ts +++ b/internal-packages/redis-worker/src/worker.test.ts @@ -274,11 +274,4 @@ describe("Worker", () => { } } ); - - //todo test that throwing an error doesn't screw up the other items - //todo process more items when finished - - //todo add a Dead Letter Queue when items are failed, with the error - //todo add a function on the worker to redrive them - //todo add an API endpoint to redrive with an ID }); diff --git a/internal-packages/redis-worker/src/worker.ts b/internal-packages/redis-worker/src/worker.ts index 601f5e1708..b9408d1224 100644 --- a/internal-packages/redis-worker/src/worker.ts +++ b/internal-packages/redis-worker/src/worker.ts @@ -9,11 +9,11 @@ import { SimpleQueue } from "./queue.js"; import Redis from "ioredis"; -type WorkerCatalog = { +export type WorkerCatalog = { [key: string]: { schema: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion; visibilityTimeoutMs: number; - retry: RetryOptions; + retry?: RetryOptions; }; }; @@ -28,6 +28,11 @@ type JobHandler = (param attempt: number; }) => Promise; +export type WorkerConcurrencyOptions = { + workers?: number; + tasksPerWorker?: number; +}; + type WorkerOptions = { name: string; redisOptions: RedisOptions; @@ -35,14 +40,22 @@ type WorkerOptions = { jobs: { [K in keyof TCatalog]: JobHandler; }; - concurrency?: { - workers?: number; - tasksPerWorker?: number; - }; + concurrency?: WorkerConcurrencyOptions; pollIntervalMs?: number; logger?: Logger; }; +// This results in attempt 12 being a delay of 1 hour +const defaultRetrySettings = { + maxAttempts: 12, + factor: 2, + //one second + minTimeoutInMs: 1_000, + //one hour + maxTimeoutInMs: 3_600_000, + randomize: true, +}; + class Worker { private subscriber: Redis; @@ -83,16 +96,28 @@ class Worker { this.setupSubscriber(); } + /** + * Enqueues a job for processing. + * @param options - The enqueue options. + * @param options.id - Optional unique identifier for the job. If not provided, one will be generated. It prevents duplication. + * @param options.job - The job type from the worker catalog. + * @param options.payload - The job payload that matches the schema defined in the catalog. + * @param options.visibilityTimeoutMs - Optional visibility timeout in milliseconds. Defaults to value from catalog. + * @param options.availableAt - Optional date when the job should become available for processing. Defaults to now. + * @returns A promise that resolves when the job is enqueued. + */ enqueue({ id, job, payload, visibilityTimeoutMs, + availableAt, }: { id?: string; job: K; payload: z.infer; visibilityTimeoutMs?: number; + availableAt?: Date; }) { const timeout = visibilityTimeoutMs ?? this.options.catalog[job].visibilityTimeoutMs; return this.queue.enqueue({ @@ -100,9 +125,14 @@ class Worker { job, item: payload, visibilityTimeoutMs: timeout, + availableAt, }); } + ack(id: string) { + return this.queue.ack(id); + } + private createWorker(tasksPerWorker: number) { const worker = new NodeWorker( ` @@ -182,7 +212,12 @@ class Worker { try { attempt = attempt + 1; - const retryDelay = calculateNextRetryDelay(catalogItem.retry, attempt); + const retrySettings = { + ...defaultRetrySettings, + ...catalogItem.retry, + }; + + const retryDelay = calculateNextRetryDelay(retrySettings, attempt); if (!retryDelay) { this.logger.error( @@ -262,7 +297,7 @@ class Worker { private async handleRedriveMessage(channel: string, message: string) { try { - const { id } = JSON.parse(message); + const { id } = JSON.parse(message) as any; if (typeof id !== "string") { throw new Error("Invalid message format: id must be a string"); } @@ -283,9 +318,7 @@ class Worker { this.isShuttingDown = true; this.logger.log("Shutting down workers..."); - for (const worker of this.workers) { - worker.terminate(); - } + await Promise.all(this.workers.map((worker) => worker.terminate())); await this.subscriber.unsubscribe(); await this.subscriber.quit(); @@ -301,8 +334,8 @@ class Worker { } } - public stop() { - this.shutdown(); + public async stop() { + await this.shutdown(); } } diff --git a/internal-packages/run-engine/README.md b/internal-packages/run-engine/README.md new file mode 100644 index 0000000000..a2ca8fda22 --- /dev/null +++ b/internal-packages/run-engine/README.md @@ -0,0 +1,189 @@ +# Trigger.dev Run Engine + +The Run Engine process runs from triggering, to executing, retrying, and completing them. + +It is responsible for: + +- Creating, updating, and completing runs as they progress. +- Operating the run queue, including handling concurrency. +- Heartbeats which detects stalled runs and attempts to automatically recover them. +- Registering checkpoints which enable pausing/resuming of runs. + +## Glossary + +- **Platform**: The main Trigger.dev API, dashboard, database. The Run Engine is part of the platform. +- **Worker group**: A group of workers that all pull from the same queue, e.g. "us-east-1", "my-self-hosted-workers". + - **Worker**: A worker is a 'server' that connects to the platform and receives runs. + - **Supervisor**: Pulls new runs from the queue, communicates with the platform, spins up new Deploy executors. + - **Deploy container**: Container that comes from a specific deploy from a user's project. + - **Run controller**: The code that manages running the task. + - **Run executor**: The actual task running. + +## Run locking + +Many operations on the run are "atomic" in the sense that only a single operation can mutate them at a time. We use RedLock to create a distributed lock to ensure this. Postgres locking is not enough on its own because we have multiple API instances and Redis is used for the queue. + +There are race conditions we need to deal with: +- When checkpointing the run continues to execute until the checkpoint has been stored. At the same time the run continues and the checkpoint can become irrelevant if the waitpoint is completed. Both can happen at the same time, so we must lock the run and protect against outdated checkpoints. + +## Run execution + +The execution state of a run is stored in the `TaskRunExecutionSnapshot` table in Postgres. This is separate from the `TaskRun` status which is exposed to users via the dashboard and API. + +![The execution states](./execution-states.png) + +The `TaskRunExecutionSnapshot` `executionStatus` is used to determine the execution status and is internal to the run engine. It is a log of events that impact run execution – the data is used to execute the run. + +A common pattern we use is to read the current state and check that the passed in `snapshotId` matches the current `snapshotId`. If it doesn't, we know that the state has moved on. In the case of a checkpoint coming in, we know we can just ignore it. + +We can also store invalid states by setting an error. These invalid states are purely used for debugging and are ignored for execution purposes. + +## Workers + +A worker is a server that runs tasks. There are two types of workers: +- Hosted workers (serverless, managed and cloud-only) +- Self-hosted workers + +In the dashboard under the "Workers" page, you can see all worker groups including the "main" group which is the default and not self-hosted. You can also see alternative worker groups that are available to you, such as "EU", "v3.2 (beta)", and any self-hosted worker groups you have created. + +You add a new self-hosted worker group by clicking "Add" and choosing an `id` that is unique to your project. + +Then when triggering runs, you can specify the `workerGroup` to use. It defaults to "main". The workerGroup is used internally to set the `masterQueue` that a run is placed in, this allows pulling runs only for that worker group. + +On the "Workers" page, you can see the status of each worker group, including the number of workers in the group, the number of runs that are queued. + +## Pulling from the queue + +A worker will call the Trigger.dev API with it's `workerGroup`. + +For warm starts, self-hosted workers we will also pass the `BackgroundWorker` id and `environment` id. This allow pulling relevant runs. + +For dev environments, we will pass the `environment` id. + +If there's only a `workerGroup`, we can just `dequeueFromMasterQueue()` to get runs. If there's a `BackgroundWorker` id, we need to determine if that `BackgroundWorker` is the latest. If it's the latest we call `dequeueFromEnvironmentMasterQueue()` to get any runs that aren't locked to a version. If it's not the latest, we call `dequeueFromBackgroundWorkerMasterQueue()` to get runs that are locked to that version. + +## Run Queue + +This is a fair multi-tenant queue. It is designed to fairly select runs, respect concurrency limits, and have high throughput. It provides visibility into the current concurrency for the env, org, etc. + +It has built-in reliability features: +- When nacking we increment the `attempt` and if it continually fails we will move it to a Dead Letter Queue (DLQ). +- If a run is in the DLQ you can redrive it. + +## Heartbeats + +Heartbeats are used to determine if a run has become stalled. Depending on the current execution status, we do different things. For example, if the run has been dequeued but the attempt hasn't been started we requeue it. + +## Checkpoints + +Checkpoints allow pausing an executing run and then resuming it later. This is an optimization to avoid wasted compute and is especially useful with "Waitpoints". + +## Waitpoints + +A "Waitpoint" is something that can block a run from continuing: + +A single Waitpoint can block many runs, the same waitpoint can only block a run once (there's a unique constraint). They block run execution from continuing until all of them are completed. + +They can have output data associated with them, e.g. the finished run payload. That includes an error, e.g. a failed run. + +There are currently three types: + - `RUN` which gets completed when the associated run completes. Every run has an `associatedWaitpoint` that matches the lifetime of the run. + - `DATETIME` which gets completed when the datetime is reached. + - `MANUAL` which gets completed when that event occurs. + +Waitpoints can have an idempotencyKey which allows stops them from being created multiple times. This is especially useful for event waitpoints, where you don't want to create a new waitpoint for the same event twice. + +### `wait.for()` or `wait.until()` +Wait for a future time, then continue. We should add the option to pass an `idempotencyKey` so a second attempt doesn't wait again. By default it would wait again. + +```ts +//Note if the idempotency key is a string, it will get prefixed with the run id. +//you can explicitly pass in an idempotency key created with the the global scope. +await wait.until(new Date('2022-01-01T00:00:00Z'), { idempotencyKey: "first-wait" }); +await wait.until(new Date('2022-01-01T00:00:00Z'), { idempotencyKey: "second-wait" }); +``` + +### `triggerAndWait()` or `batchTriggerAndWait()` +Trigger and then wait for run(s) to finish. If the run fails it will still continue but with the errors so the developer can decide what to do. + +### The `trigger` `delay` option + +When triggering a run and passing the `delay` option, we use a `DATETIME` waitpoint to block the run from starting. + +### `wait.forRequest()` +Wait until a request has been received at the URL that you are given. This is useful for pausing a run and then continuing it again when some external event occurs on another service. For example, Replicate have an API where they will callback when their work is complete. + +### `wait.forWaitpoint(waitpointId)` + +A more advanced SDK which would require uses to explicitly create a waitpoint. We would also need `createWaitpoint()`, `completeWaitpoint()`, and `failWaitpoint()`. + +```ts +const waitpoint = await waitpoints.create({ idempotencyKey: `purchase-${payload.cart.id}` }); +const waitpoint = await waitpoints.retrieve(waitpoint.id); +const waitpoint = await waitpoints.complete(waitpoint.id, result); +const waitpoint = await waitpoints.fail(waitpoint.id, error); + +export const approvalFlow = task({ + id: "approvalFlow", + run: async (payload) => { + //...do stuff + + const result = await wait.forWaitpoint(waitpoint.id, { timeout: "1h" }); + if (!result.ok) { + //...timeout + } + + //...do more stuff + }, +}); +``` + +### `wait.forRunToComplete(runId)` + +You could wait for another run (or runs) using their run ids. This would allow you to wait for runs that you haven't triggered inside that run. + +## Run flow control + +There are several ways to control when a run will execute (or not). Each of these should be configurable on a task, a named queue that is shared between tasks, and at trigger time including the ability to pass a `key` so you can have per-tenant controls. + +### Concurrency limits + +When `trigger` is called the run is added to the queue. We only dequeue when the concurrency limit hasn't been exceeded for that task/queue. + +### Rate limiting + +When `trigger` is called, we check if the rate limit has been exceeded. If it has then we ignore the trigger. The run is thrown away and an appropriate error is returned. + +This is useful: +- To prevent abuse. +- To control how many executions a user can do (using a `key` with rate limiting). + +### Debouncing + +When `trigger` is called, we prevent too many runs happening in a period by collapsing into a single run. This is done by discarding some runs in a period. + +This is useful: +- To prevent too many runs happening in a short period. + +We should mark the run as `"DELAYED"` with the correct `delayUntil` time. This will allow the user to see that the run is delayed and why. + +### Throttling + +When `trigger` is called the run is added to the queue. We only run them when they don't exceed the limit in that time period, by controlling the timing of when they are dequeued. + +This is useful: +- To prevent too many runs happening in a short period. +- To control how many executions a user can do (using a `key` with throttling). +- When you need to execute every run but not too many in a short period, e.g. avoiding rate limits. + +### Batching + +When `trigger` is called, we batch the runs together. This means the payload of the run is an array of items, each being a single payload. + +This is useful: +- For performance, as it reduces the number of runs in the system. +- It can be useful when using 3rd party APIs that support batching. + +## Emitting events + +The Run Engine emits events using its `eventBus`. This is used for runs completing, failing, or things that any workers should be aware of. diff --git a/internal-packages/run-engine/execution-states.png b/internal-packages/run-engine/execution-states.png new file mode 100644 index 0000000000..cc156dd7de Binary files /dev/null and b/internal-packages/run-engine/execution-states.png differ diff --git a/internal-packages/run-engine/package.json b/internal-packages/run-engine/package.json new file mode 100644 index 0000000000..b0a2dc9eb6 --- /dev/null +++ b/internal-packages/run-engine/package.json @@ -0,0 +1,27 @@ +{ + "name": "@internal/run-engine", + "private": true, + "version": "0.0.1", + "main": "./src/index.ts", + "types": "./src/index.ts", + "dependencies": { + "@internal/redis-worker": "workspace:*", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/semantic-conventions": "^1.27.0", + "@trigger.dev/core": "workspace:*", + "@trigger.dev/database": "workspace:*", + "assert-never": "^1.2.1", + "ioredis": "^5.3.2", + "nanoid": "^3.3.4", + "redlock": "5.0.0-beta.2", + "zod": "3.23.8" + }, + "devDependencies": { + "@internal/testcontainers": "workspace:*", + "vitest": "^1.4.0" + }, + "scripts": { + "typecheck": "tsc --noEmit", + "test": "vitest --sequence.concurrent=false" + } +} diff --git a/internal-packages/run-engine/src/engine/consts.ts b/internal-packages/run-engine/src/engine/consts.ts new file mode 100644 index 0000000000..6ea6f54c38 --- /dev/null +++ b/internal-packages/run-engine/src/engine/consts.ts @@ -0,0 +1 @@ +export const MAX_TASK_RUN_ATTEMPTS = 250; diff --git a/internal-packages/run-engine/src/engine/db/worker.ts b/internal-packages/run-engine/src/engine/db/worker.ts new file mode 100644 index 0000000000..684bad6f50 --- /dev/null +++ b/internal-packages/run-engine/src/engine/db/worker.ts @@ -0,0 +1,245 @@ +import { + BackgroundWorker, + BackgroundWorkerTask, + Prisma, + PrismaClientOrTransaction, + WorkerDeployment, +} from "@trigger.dev/database"; +import { CURRENT_DEPLOYMENT_LABEL } from "@trigger.dev/core/v3/apps"; + +type RunWithMininimalEnvironment = Prisma.TaskRunGetPayload<{ + include: { + runtimeEnvironment: { + select: { + id: true; + type: true; + }; + }; + }; +}>; + +type RunWithBackgroundWorkerTasksResult = + | { + success: false; + code: "NO_RUN"; + message: string; + } + | { + success: false; + code: + | "NO_WORKER" + | "TASK_NOT_IN_LATEST" + | "TASK_NEVER_REGISTERED" + | "BACKGROUND_WORKER_MISMATCH"; + message: string; + run: RunWithMininimalEnvironment; + } + | { + success: false; + code: "BACKGROUND_WORKER_MISMATCH"; + message: string; + backgroundWorker: { + expected: string; + received: string; + }; + run: RunWithMininimalEnvironment; + } + | { + success: true; + run: RunWithMininimalEnvironment; + worker: BackgroundWorker; + task: BackgroundWorkerTask; + deployment: WorkerDeployment | null; + }; + +export async function getRunWithBackgroundWorkerTasks( + prisma: PrismaClientOrTransaction, + runId: string, + backgroundWorkerId?: string +): Promise { + const run = await prisma.taskRun.findFirst({ + where: { + id: runId, + }, + include: { + runtimeEnvironment: { + select: { + id: true, + type: true, + }, + }, + lockedToVersion: { + include: { + deployment: true, + tasks: true, + }, + }, + }, + }); + + if (!run) { + return { + success: false as const, + code: "NO_RUN", + message: `No run found with id: ${runId}`, + }; + } + + const workerId = run.lockedToVersionId ?? backgroundWorkerId; + + //get the relevant BackgroundWorker with tasks and deployment (if not DEV) + const workerWithTasks = workerId + ? await getWorkerDeploymentFromWorker(prisma, workerId) + : run.runtimeEnvironment.type === "DEVELOPMENT" + ? await getMostRecentWorker(prisma, run.runtimeEnvironmentId) + : await getWorkerFromCurrentlyPromotedDeployment(prisma, run.runtimeEnvironmentId); + + if (!workerWithTasks) { + return { + success: false as const, + code: "NO_WORKER", + message: `No worker found for run: ${run.id}`, + run, + }; + } + + if (backgroundWorkerId) { + if (backgroundWorkerId !== workerWithTasks.worker.id) { + return { + success: false as const, + code: "BACKGROUND_WORKER_MISMATCH", + message: `Background worker mismatch for run: ${run.id}`, + backgroundWorker: { + expected: backgroundWorkerId, + received: workerWithTasks.worker.id, + }, + run, + }; + } + } + + const backgroundTask = workerWithTasks.tasks.find((task) => task.slug === run.taskIdentifier); + + if (!backgroundTask) { + const nonCurrentTask = await prisma.backgroundWorkerTask.findFirst({ + where: { + slug: run.taskIdentifier, + projectId: run.projectId, + runtimeEnvironmentId: run.runtimeEnvironmentId, + }, + include: { + worker: true, + }, + }); + + if (nonCurrentTask) { + return { + success: false as const, + code: "TASK_NOT_IN_LATEST", + message: `Task not found in latest version: ${run.taskIdentifier}. Found in ${nonCurrentTask.worker.version}`, + run, + }; + } else { + return { + success: false as const, + code: "TASK_NEVER_REGISTERED", + message: `Task has never been registered (in dev or deployed): ${run.taskIdentifier}`, + run, + }; + } + } + + return { + success: true as const, + run, + worker: workerWithTasks.worker, + task: backgroundTask, + deployment: workerWithTasks.deployment, + }; +} + +type WorkerDeploymentWithWorkerTasks = { + worker: BackgroundWorker; + tasks: BackgroundWorkerTask[]; + deployment: WorkerDeployment | null; +}; + +export async function getWorkerDeploymentFromWorker( + prisma: PrismaClientOrTransaction, + workerId: string +): Promise { + const worker = await prisma.backgroundWorker.findUnique({ + where: { + id: workerId, + }, + include: { + deployment: true, + tasks: true, + }, + }); + + if (!worker) { + return null; + } + + return { worker, tasks: worker.tasks, deployment: worker.deployment }; +} + +export async function getMostRecentWorker( + prisma: PrismaClientOrTransaction, + environmentId: string +): Promise { + const worker = await prisma.backgroundWorker.findFirst({ + where: { + runtimeEnvironmentId: environmentId, + }, + include: { + deployment: true, + tasks: true, + }, + orderBy: { + id: "desc", + }, + }); + + if (!worker) { + return null; + } + + return { worker, tasks: worker.tasks, deployment: worker.deployment }; +} + +export async function getWorkerFromCurrentlyPromotedDeployment( + prisma: PrismaClientOrTransaction, + environmentId: string +): Promise { + const promotion = await prisma.workerDeploymentPromotion.findUnique({ + where: { + environmentId_label: { + environmentId, + label: CURRENT_DEPLOYMENT_LABEL, + }, + }, + include: { + deployment: { + include: { + worker: { + include: { + tasks: true, + }, + }, + }, + }, + }, + }); + + if (!promotion || !promotion.deployment.worker) { + return null; + } + + return { + worker: promotion.deployment.worker, + tasks: promotion.deployment.worker.tasks, + deployment: promotion.deployment, + }; +} diff --git a/internal-packages/run-engine/src/engine/errors.ts b/internal-packages/run-engine/src/engine/errors.ts new file mode 100644 index 0000000000..b7ef0b4d61 --- /dev/null +++ b/internal-packages/run-engine/src/engine/errors.ts @@ -0,0 +1,53 @@ +import { assertExhaustive } from "@trigger.dev/core"; +import { TaskRunError } from "@trigger.dev/core/v3"; +import { TaskRunStatus } from "@trigger.dev/database"; + +export function runStatusFromError(error: TaskRunError): TaskRunStatus { + if (error.type !== "INTERNAL_ERROR") { + return "COMPLETED_WITH_ERRORS"; + } + + //"CRASHED" should be used if it's a user-error or something they've misconfigured + //e.g. not enough memory + //"SYSTEM_FAILURE" should be used if it's an error from our system + //e.g. a bug + switch (error.code) { + case "TASK_RUN_CANCELLED": + return "CANCELED"; + case "MAX_DURATION_EXCEEDED": + return "TIMED_OUT"; + case "TASK_PROCESS_OOM_KILLED": + case "TASK_PROCESS_MAYBE_OOM_KILLED": + case "TASK_PROCESS_SIGSEGV": + case "DISK_SPACE_EXCEEDED": + case "OUTDATED_SDK_VERSION": + case "HANDLE_ERROR_ERROR": + case "TASK_RUN_CRASHED": + case "TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE": + return "CRASHED"; + case "COULD_NOT_FIND_EXECUTOR": + case "COULD_NOT_FIND_TASK": + case "COULD_NOT_IMPORT_TASK": + case "CONFIGURED_INCORRECTLY": + case "TASK_ALREADY_RUNNING": + case "TASK_PROCESS_SIGKILL_TIMEOUT": + case "TASK_RUN_HEARTBEAT_TIMEOUT": + case "TASK_DEQUEUED_INVALID_STATE": + case "TASK_DEQUEUED_QUEUE_NOT_FOUND": + case "TASK_RUN_DEQUEUED_MAX_RETRIES": + case "TASK_RUN_STALLED_EXECUTING": + case "TASK_RUN_STALLED_EXECUTING_WITH_WAITPOINTS": + case "TASK_HAS_N0_EXECUTION_SNAPSHOT": + case "GRACEFUL_EXIT_TIMEOUT": + case "TASK_INPUT_ERROR": + case "TASK_OUTPUT_ERROR": + case "POD_EVICTED": + case "POD_UNKNOWN_ERROR": + case "TASK_EXECUTION_ABORTED": + case "TASK_EXECUTION_FAILED": + case "TASK_PROCESS_SIGTERM": + return "SYSTEM_FAILURE"; + default: + assertExhaustive(error.code); + } +} diff --git a/internal-packages/run-engine/src/engine/eventBus.ts b/internal-packages/run-engine/src/engine/eventBus.ts new file mode 100644 index 0000000000..7c54638935 --- /dev/null +++ b/internal-packages/run-engine/src/engine/eventBus.ts @@ -0,0 +1,111 @@ +import { TaskRunExecutionStatus, TaskRunStatus } from "@trigger.dev/database"; +import { AuthenticatedEnvironment } from "../shared"; +import { TaskRunError } from "@trigger.dev/core/v3"; + +export type EventBusEvents = { + runAttemptStarted: [ + { + time: Date; + run: { + id: string; + attemptNumber: number; + baseCostInCents: number; + }; + organization: { + id: string; + }; + }, + ]; + runExpired: [ + { + time: Date; + run: { + id: string; + spanId: string; + ttl: string | null; + }; + }, + ]; + runSucceeded: [ + { + time: Date; + run: { + id: string; + spanId: string; + output: string | undefined; + outputType: string; + }; + }, + ]; + runFailed: [ + { + time: Date; + run: { + id: string; + status: TaskRunStatus; + spanId: string; + error: TaskRunError; + }; + }, + ]; + runRetryScheduled: [ + { + time: Date; + run: { + id: string; + friendlyId: string; + spanId: string; + attemptNumber: number; + queue: string; + traceContext: Record; + taskIdentifier: string; + baseCostInCents: number; + }; + organization: { + id: string; + }; + environment: AuthenticatedEnvironment; + retryAt: Date; + }, + ]; + runCancelled: [ + { + time: Date; + run: { + id: string; + friendlyId: string; + spanId: string; + error: TaskRunError; + }; + }, + ]; + workerNotification: [ + { + time: Date; + run: { + id: string; + }; + }, + ]; + executionSnapshotCreated: [ + { + time: Date; + run: { + id: string; + }; + snapshot: { + id: string; + executionStatus: TaskRunExecutionStatus; + description: string; + runStatus: string; + attemptNumber: number | null; + checkpointId: string | null; + completedWaitpointIds: string[]; + isValid: boolean; + error: string | null; + }; + }, + ]; +}; + +export type EventBusEventArgs = EventBusEvents[T]; diff --git a/internal-packages/run-engine/src/engine/executionSnapshots.ts b/internal-packages/run-engine/src/engine/executionSnapshots.ts new file mode 100644 index 0000000000..eb2dfcf42e --- /dev/null +++ b/internal-packages/run-engine/src/engine/executionSnapshots.ts @@ -0,0 +1,104 @@ +import { CompletedWaitpoint, ExecutionResult } from "@trigger.dev/core/v3"; +import { RunId, SnapshotId } from "@trigger.dev/core/v3/apps"; +import { + PrismaClientOrTransaction, + TaskRunCheckpoint, + TaskRunExecutionSnapshot, +} from "@trigger.dev/database"; + +interface LatestExecutionSnapshot extends TaskRunExecutionSnapshot { + friendlyId: string; + runFriendlyId: string; + checkpoint: TaskRunCheckpoint | null; + completedWaitpoints: CompletedWaitpoint[]; +} + +/* Gets the most recent valid snapshot for a run */ +export async function getLatestExecutionSnapshot( + prisma: PrismaClientOrTransaction, + runId: string +): Promise { + const snapshot = await prisma.taskRunExecutionSnapshot.findFirst({ + where: { runId, isValid: true }, + include: { + completedWaitpoints: true, + checkpoint: true, + }, + orderBy: { createdAt: "desc" }, + }); + + if (!snapshot) { + throw new Error(`No execution snapshot found for TaskRun ${runId}`); + } + + return { + ...snapshot, + friendlyId: SnapshotId.toFriendlyId(snapshot.id), + runFriendlyId: RunId.toFriendlyId(snapshot.runId), + completedWaitpoints: snapshot.completedWaitpoints.map( + (w) => + ({ + id: w.id, + friendlyId: w.friendlyId, + type: w.type, + completedAt: w.completedAt ?? new Date(), + idempotencyKey: + w.userProvidedIdempotencyKey && !w.inactiveIdempotencyKey + ? w.idempotencyKey + : undefined, + completedByTaskRun: w.completedByTaskRunId + ? { + id: w.completedByTaskRunId, + friendlyId: RunId.toFriendlyId(w.completedByTaskRunId), + } + : undefined, + completedAfter: w.completedAfter ?? undefined, + output: w.output ?? undefined, + outputType: w.outputType, + outputIsError: w.outputIsError, + }) satisfies CompletedWaitpoint + ), + }; +} + +export async function getExecutionSnapshotCompletedWaitpoints( + prisma: PrismaClientOrTransaction, + snapshotId: string +) { + const waitpoints = await prisma.taskRunExecutionSnapshot.findFirst({ + where: { id: snapshotId }, + include: { + completedWaitpoints: true, + }, + }); + + //deduplicate waitpoints + const waitpointIds = new Set(); + return ( + waitpoints?.completedWaitpoints.filter((waitpoint) => { + if (waitpointIds.has(waitpoint.id)) { + return false; + } else { + waitpointIds.add(waitpoint.id); + return true; + } + }) ?? [] + ); +} + +export function executionResultFromSnapshot(snapshot: TaskRunExecutionSnapshot): ExecutionResult { + return { + snapshot: { + id: snapshot.id, + friendlyId: SnapshotId.toFriendlyId(snapshot.id), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + }, + run: { + id: snapshot.runId, + friendlyId: RunId.toFriendlyId(snapshot.runId), + status: snapshot.runStatus, + attemptNumber: snapshot.attemptNumber, + }, + }; +} diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts new file mode 100644 index 0000000000..83dc41e65b --- /dev/null +++ b/internal-packages/run-engine/src/engine/index.ts @@ -0,0 +1,3160 @@ +import { Worker } from "@internal/redis-worker"; +import { Attributes, Span, SpanKind, trace, Tracer } from "@opentelemetry/api"; +import { assertExhaustive } from "@trigger.dev/core"; +import { Logger } from "@trigger.dev/core/logger"; +import { + CompleteRunAttemptResult, + DequeuedMessage, + ExecutionResult, + parsePacket, + RunExecutionData, + sanitizeError, + shouldRetryError, + StartRunAttemptResult, + TaskRunError, + taskRunErrorEnhancer, + TaskRunExecution, + TaskRunExecutionResult, + TaskRunFailedExecutionResult, + TaskRunInternalError, + TaskRunSuccessfulExecutionResult, + WaitForDurationResult, +} from "@trigger.dev/core/v3"; +import { + getMaxDuration, + parseNaturalLanguageDuration, + QueueId, + RunId, + sanitizeQueueName, + SnapshotId, + WaitpointId, +} from "@trigger.dev/core/v3/apps"; +import { + $transaction, + Prisma, + PrismaClient, + PrismaClientOrTransaction, + TaskRun, + TaskRunExecutionSnapshot, + TaskRunExecutionStatus, + TaskRunStatus, + Waitpoint, +} from "@trigger.dev/database"; +import assertNever from "assert-never"; +import { Redis } from "ioredis"; +import { nanoid } from "nanoid"; +import { EventEmitter } from "node:events"; +import { z } from "zod"; +import { RunQueue } from "../run-queue"; +import { SimpleWeightedChoiceStrategy } from "../run-queue/simpleWeightedPriorityStrategy"; +import { MinimalAuthenticatedEnvironment } from "../shared"; +import { MAX_TASK_RUN_ATTEMPTS } from "./consts"; +import { getRunWithBackgroundWorkerTasks } from "./db/worker"; +import { runStatusFromError } from "./errors"; +import { EventBusEvents } from "./eventBus"; +import { executionResultFromSnapshot, getLatestExecutionSnapshot } from "./executionSnapshots"; +import { RunLocker } from "./locking"; +import { machinePresetFromConfig } from "./machinePresets"; +import { + isCheckpointable, + isDequeueableExecutionStatus, + isExecuting, + isFinalRunStatus, +} from "./statuses"; +import { HeartbeatTimeouts, MachineResources, RunEngineOptions, TriggerParams } from "./types"; + +const workerCatalog = { + finishWaitpoint: { + schema: z.object({ + waitpointId: z.string(), + error: z.string().optional(), + }), + visibilityTimeoutMs: 5000, + }, + heartbeatSnapshot: { + schema: z.object({ + runId: z.string(), + snapshotId: z.string(), + }), + visibilityTimeoutMs: 5000, + }, + expireRun: { + schema: z.object({ + runId: z.string(), + }), + visibilityTimeoutMs: 5000, + }, + cancelRun: { + schema: z.object({ + runId: z.string(), + completedAt: z.coerce.date(), + reason: z.string().optional(), + }), + visibilityTimeoutMs: 5000, + }, + queueRunsWaitingForWorker: { + schema: z.object({ + backgroundWorkerId: z.string(), + }), + visibilityTimeoutMs: 5000, + }, + tryCompleteBatch: { + schema: z.object({ + batchId: z.string(), + }), + visibilityTimeoutMs: 10_000, + }, + continueRunIfUnblocked: { + schema: z.object({ + runId: z.string(), + }), + visibilityTimeoutMs: 10_000, + }, +}; + +type EngineWorker = Worker; + +export class RunEngine { + private redis: Redis; + private prisma: PrismaClient; + private runLock: RunLocker; + runQueue: RunQueue; + private worker: EngineWorker; + private logger = new Logger("RunEngine", "debug"); + private tracer: Tracer; + private heartbeatTimeouts: HeartbeatTimeouts; + eventBus = new EventEmitter(); + + constructor(private readonly options: RunEngineOptions) { + this.prisma = options.prisma; + this.redis = new Redis(options.redis); + this.runLock = new RunLocker({ redis: this.redis }); + + this.runQueue = new RunQueue({ + name: "rq", + tracer: trace.getTracer("rq"), + queuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 36 }), + envQueuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 12 }), + defaultEnvConcurrency: options.queue?.defaultEnvConcurrency ?? 10, + logger: new Logger("RunQueue", "warn"), + redis: options.redis, + retryOptions: options.queue?.retryOptions, + }); + + this.worker = new Worker({ + name: "runengineworker", + redisOptions: options.redis, + catalog: workerCatalog, + concurrency: options.worker, + pollIntervalMs: options.worker.pollIntervalMs, + logger: new Logger("RunEngineWorker", "debug"), + jobs: { + finishWaitpoint: async ({ payload }) => { + await this.completeWaitpoint({ + id: payload.waitpointId, + output: payload.error + ? { + value: payload.error, + isError: true, + } + : undefined, + }); + }, + heartbeatSnapshot: async ({ payload }) => { + await this.#handleStalledSnapshot(payload); + }, + expireRun: async ({ payload }) => { + await this.#expireRun({ runId: payload.runId }); + }, + cancelRun: async ({ payload }) => { + await this.cancelRun({ + runId: payload.runId, + completedAt: payload.completedAt, + reason: payload.reason, + }); + }, + queueRunsWaitingForWorker: async ({ payload }) => { + await this.#queueRunsWaitingForWorker({ backgroundWorkerId: payload.backgroundWorkerId }); + }, + tryCompleteBatch: async ({ payload }) => { + await this.#tryCompleteBatch({ batchId: payload.batchId }); + }, + continueRunIfUnblocked: async ({ payload }) => { + await this.#continueRunIfUnblocked({ + runId: payload.runId, + }); + }, + }, + }); + + this.tracer = options.tracer; + + const defaultHeartbeatTimeouts: HeartbeatTimeouts = { + PENDING_EXECUTING: 60_000, + PENDING_CANCEL: 60_000, + EXECUTING: 60_000, + EXECUTING_WITH_WAITPOINTS: 60_000, + }; + this.heartbeatTimeouts = { + ...defaultHeartbeatTimeouts, + ...(options.heartbeatTimeoutsMs ?? {}), + }; + } + + //MARK: - Run functions + + /** "Triggers" one run. */ + async trigger( + { + friendlyId, + number, + environment, + idempotencyKey, + idempotencyKeyExpiresAt, + taskIdentifier, + payload, + payloadType, + context, + traceContext, + traceId, + spanId, + parentSpanId, + lockedToVersionId, + taskVersion, + sdkVersion, + cliVersion, + concurrencyKey, + masterQueue, + queueName, + queue, + isTest, + delayUntil, + queuedAt, + maxAttempts, + priorityMs, + ttl, + tags, + parentTaskRunId, + rootTaskRunId, + batchId, + resumeParentOnCompletion, + depth, + metadata, + metadataType, + seedMetadata, + seedMetadataType, + oneTimeUseToken, + maxDurationInSeconds, + }: TriggerParams, + tx?: PrismaClientOrTransaction + ): Promise { + const prisma = tx ?? this.prisma; + + return this.#trace( + "createRunAttempt", + { + friendlyId, + environmentId: environment.id, + projectId: environment.project.id, + taskIdentifier, + }, + async (span) => { + const status = delayUntil ? "DELAYED" : "PENDING"; + + let secondaryMasterQueue = this.#environmentMasterQueueKey(environment.id); + if (lockedToVersionId) { + secondaryMasterQueue = this.#backgroundWorkerQueueKey(lockedToVersionId); + } + + //create run + const taskRun = await prisma.taskRun.create({ + data: { + id: RunId.fromFriendlyId(friendlyId), + engine: "V2", + status, + number, + friendlyId, + runtimeEnvironmentId: environment.id, + projectId: environment.project.id, + idempotencyKey, + idempotencyKeyExpiresAt, + taskIdentifier, + payload, + payloadType, + context, + traceContext, + traceId, + spanId, + parentSpanId, + lockedToVersionId, + taskVersion, + sdkVersion, + cliVersion, + concurrencyKey, + queue: queueName, + masterQueue, + secondaryMasterQueue, + isTest, + delayUntil, + queuedAt, + maxAttempts, + priorityMs, + ttl, + tags: + tags.length === 0 + ? undefined + : { + connect: tags, + }, + runTags: tags.length === 0 ? undefined : tags.map((tag) => tag.name), + oneTimeUseToken, + parentTaskRunId, + rootTaskRunId, + batchId, + resumeParentOnCompletion, + depth, + metadata, + metadataType, + seedMetadata, + seedMetadataType, + maxDurationInSeconds, + executionSnapshots: { + create: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: status, + }, + }, + }, + }); + + span.setAttribute("runId", taskRun.id); + + await this.runLock.lock([taskRun.id], 5000, async (signal) => { + //create associated waitpoint (this completes when the run completes) + const associatedWaitpoint = await this.#createRunAssociatedWaitpoint(prisma, { + projectId: environment.project.id, + environmentId: environment.id, + completedByTaskRunId: taskRun.id, + }); + + //triggerAndWait or batchTriggerAndWait + if (resumeParentOnCompletion && parentTaskRunId) { + //this will block the parent run from continuing until this waitpoint is completed (and removed) + await this.blockRunWithWaitpoint({ + runId: parentTaskRunId, + waitpointId: associatedWaitpoint.id, + environmentId: associatedWaitpoint.environmentId, + projectId: associatedWaitpoint.projectId, + tx: prisma, + }); + + //release the concurrency + //if the queue is the same then it's recursive and we need to release that too otherwise we could have a deadlock + const parentRun = await prisma.taskRun.findUnique({ + select: { + queue: true, + }, + where: { + id: parentTaskRunId, + }, + }); + const releaseRunConcurrency = parentRun?.queue === taskRun.queue; + await this.runQueue.releaseConcurrency( + environment.organization.id, + parentTaskRunId, + releaseRunConcurrency + ); + } + + //Make sure lock extension succeeded + signal.throwIfAborted(); + + if (queue) { + const concurrencyLimit = + typeof queue.concurrencyLimit === "number" + ? Math.max(Math.min(queue.concurrencyLimit, environment.maximumConcurrencyLimit), 0) + : null; + + let taskQueue = await prisma.taskQueue.findFirst({ + where: { + runtimeEnvironmentId: environment.id, + name: queueName, + }, + }); + + const existingConcurrencyLimit = + typeof taskQueue?.concurrencyLimit === "number" + ? taskQueue.concurrencyLimit + : undefined; + + if (taskQueue) { + if (existingConcurrencyLimit !== concurrencyLimit) { + taskQueue = await prisma.taskQueue.update({ + where: { + id: taskQueue.id, + }, + data: { + concurrencyLimit: + typeof concurrencyLimit === "number" ? concurrencyLimit : null, + }, + }); + + if (typeof taskQueue.concurrencyLimit === "number") { + await this.runQueue.updateQueueConcurrencyLimits( + environment, + taskQueue.name, + taskQueue.concurrencyLimit + ); + } else { + await this.runQueue.removeQueueConcurrencyLimits(environment, taskQueue.name); + } + } + } else { + taskQueue = await prisma.taskQueue.create({ + data: { + ...QueueId.generate(), + name: queueName, + concurrencyLimit, + runtimeEnvironmentId: environment.id, + projectId: environment.project.id, + type: "NAMED", + }, + }); + + if (typeof taskQueue.concurrencyLimit === "number") { + await this.runQueue.updateQueueConcurrencyLimits( + environment, + taskQueue.name, + taskQueue.concurrencyLimit + ); + } + } + } + + if (taskRun.delayUntil) { + const delayWaitpoint = await this.#createDateTimeWaitpoint(prisma, { + projectId: environment.project.id, + environmentId: environment.id, + completedAfter: taskRun.delayUntil, + }); + + await prisma.taskRunWaitpoint.create({ + data: { + taskRunId: taskRun.id, + waitpointId: delayWaitpoint.id, + projectId: delayWaitpoint.projectId, + }, + }); + } + + if (!taskRun.delayUntil && taskRun.ttl) { + const expireAt = parseNaturalLanguageDuration(taskRun.ttl); + + if (expireAt) { + await this.worker.enqueue({ + id: `expireRun:${taskRun.id}`, + job: "expireRun", + payload: { runId: taskRun.id }, + }); + } + } + + //Make sure lock extension succeeded + signal.throwIfAborted(); + + //enqueue the run if it's not delayed + if (!taskRun.delayUntil) { + await this.#enqueueRun({ + run: taskRun, + env: environment, + timestamp: Date.now() - taskRun.priorityMs, + tx: prisma, + }); + } + }); + + return taskRun; + } + ); + } + + /** + * Gets a fairly selected run from the specified master queue, returning the information required to run it. + * @param consumerId: The consumer that is pulling, allows multiple consumers to pull from the same queue + * @param masterQueue: The shared queue to pull from, can be an individual environment (for dev) + * @returns + */ + async dequeueFromMasterQueue({ + consumerId, + masterQueue, + maxRunCount, + maxResources, + backgroundWorkerId, + tx, + }: { + consumerId: string; + masterQueue: string; + maxRunCount: number; + maxResources?: MachineResources; + backgroundWorkerId?: string; + tx?: PrismaClientOrTransaction; + }): Promise { + const prisma = tx ?? this.prisma; + return this.#trace("dequeueFromMasterQueue", { consumerId, masterQueue }, async (span) => { + //gets multiple runs from the queue + const messages = await this.runQueue.dequeueMessageFromMasterQueue( + consumerId, + masterQueue, + maxRunCount + ); + if (messages.length === 0) { + return []; + } + + //we can't send more than the max resources + const consumedResources: MachineResources = { + cpu: 0, + memory: 0, + }; + + const dequeuedRuns: DequeuedMessage[] = []; + + for (const message of messages) { + const orgId = message.message.orgId; + const runId = message.messageId; + + span.setAttribute("runId", runId); + + //lock the run so nothing else can modify it + try { + const dequeuedRun = await this.runLock.lock([runId], 5000, async (signal) => { + const snapshot = await getLatestExecutionSnapshot(prisma, runId); + + if (!isDequeueableExecutionStatus(snapshot.executionStatus)) { + //create a failed snapshot + await this.#createExecutionSnapshot(prisma, { + run: { + id: snapshot.runId, + status: snapshot.runStatus, + }, + snapshot: { + executionStatus: snapshot.executionStatus, + description: + "Tried to dequeue a run that is not in a valid state to be dequeued.", + }, + checkpointId: snapshot.checkpointId ?? undefined, + completedWaitpointIds: snapshot.completedWaitpoints.map((wp) => wp.id), + error: `Tried to dequeue a run that is not in a valid state to be dequeued.`, + }); + + //todo is there a way to recover this, so the run can be retried? + //for example should we update the status to a dequeuable status and nack it? + //then at least it has a chance of succeeding and we have the error log above + await this.#systemFailure({ + runId, + error: { + type: "INTERNAL_ERROR", + code: "TASK_DEQUEUED_INVALID_STATE", + message: `Task was in the ${snapshot.executionStatus} state when it was dequeued for execution.`, + }, + tx: prisma, + }); + this.logger.error( + `RunEngine.dequeueFromMasterQueue(): Run is not in a valid state to be dequeued: ${runId}\n ${snapshot.id}:${snapshot.executionStatus}` + ); + return null; + } + + const result = await getRunWithBackgroundWorkerTasks(prisma, runId, backgroundWorkerId); + + if (!result.success) { + switch (result.code) { + case "NO_RUN": { + //this should not happen, the run is unrecoverable so we'll ack it + this.logger.error("RunEngine.dequeueFromMasterQueue(): No run found", { + runId, + latestSnapshot: snapshot.id, + }); + await this.runQueue.acknowledgeMessage(orgId, runId); + return null; + } + case "NO_WORKER": + case "TASK_NEVER_REGISTERED": + case "TASK_NOT_IN_LATEST": { + this.logger.warn(`RunEngine.dequeueFromMasterQueue(): ${result.code}`, { + runId, + latestSnapshot: snapshot.id, + result, + }); + + //not deployed yet, so we'll wait for the deploy + await this.#waitingForDeploy({ + orgId, + runId, + tx: prisma, + }); + return null; + } + case "BACKGROUND_WORKER_MISMATCH": { + this.logger.warn( + "RunEngine.dequeueFromMasterQueue(): Background worker mismatch", + { + runId, + latestSnapshot: snapshot.id, + result, + } + ); + + //worker mismatch so put it back in the queue + await this.runQueue.nackMessage({ orgId, messageId: runId }); + + return null; + } + default: { + assertExhaustive(result); + } + } + } + + //check for a valid deployment if it's not a development environment + if (result.run.runtimeEnvironment.type !== "DEVELOPMENT") { + if (!result.deployment || !result.deployment.imageReference) { + this.logger.warn("RunEngine.dequeueFromMasterQueue(): No deployment found", { + runId, + latestSnapshot: snapshot.id, + result, + }); + //not deployed yet, so we'll wait for the deploy + await this.#waitingForDeploy({ + orgId, + runId, + tx: prisma, + }); + + return null; + } + } + + const machinePreset = machinePresetFromConfig({ + machines: this.options.machines.machines, + defaultMachine: this.options.machines.defaultMachine, + config: result.task.machineConfig ?? {}, + }); + + //increment the consumed resources + consumedResources.cpu += machinePreset.cpu; + consumedResources.memory += machinePreset.memory; + + //are we under the limit? + if (maxResources) { + if ( + consumedResources.cpu > maxResources.cpu || + consumedResources.memory > maxResources.memory + ) { + this.logger.debug( + "RunEngine.dequeueFromMasterQueue(): Consumed resources over limit, nacking", + { + runId, + consumedResources, + maxResources, + } + ); + + //put it back in the queue where it was + await this.runQueue.nackMessage({ + orgId, + messageId: runId, + incrementAttemptCount: false, + retryAt: result.run.createdAt.getTime() - result.run.priorityMs, + }); + return null; + } + } + + //update the run + const lockedTaskRun = await prisma.taskRun.update({ + where: { + id: runId, + }, + data: { + lockedAt: new Date(), + lockedById: result.task.id, + lockedToVersionId: result.worker.id, + startedAt: result.run.startedAt ?? new Date(), + baseCostInCents: this.options.machines.baseCostInCents, + machinePreset: machinePreset.name, + taskVersion: result.worker.version, + sdkVersion: result.worker.sdkVersion, + cliVersion: result.worker.cliVersion, + maxDurationInSeconds: getMaxDuration( + result.run.maxDurationInSeconds, + result.task.maxDurationInSeconds + ), + }, + include: { + runtimeEnvironment: true, + attempts: { + take: 1, + orderBy: { number: "desc" }, + }, + tags: true, + }, + }); + + if (!lockedTaskRun) { + this.logger.error("RunEngine.dequeueFromMasterQueue(): Failed to lock task run", { + taskRun: result.run.id, + taskIdentifier: result.run.taskIdentifier, + deployment: result.deployment?.id, + worker: result.worker.id, + task: result.task.id, + runId, + }); + + await this.runQueue.acknowledgeMessage(orgId, runId); + return null; + } + + const queue = await prisma.taskQueue.findUnique({ + where: { + runtimeEnvironmentId_name: { + runtimeEnvironmentId: lockedTaskRun.runtimeEnvironmentId, + name: sanitizeQueueName(lockedTaskRun.queue), + }, + }, + }); + + if (!queue) { + this.logger.debug( + "RunEngine.dequeueFromMasterQueue(): queue not found, so nacking message", + { + queueMessage: message, + taskRunQueue: lockedTaskRun.queue, + runtimeEnvironmentId: lockedTaskRun.runtimeEnvironmentId, + } + ); + + //will auto-retry + const gotRequeued = await this.runQueue.nackMessage({ orgId, messageId: runId }); + if (!gotRequeued) { + await this.#systemFailure({ + runId, + error: { + type: "INTERNAL_ERROR", + code: "TASK_DEQUEUED_QUEUE_NOT_FOUND", + message: `Tried to dequeue the run but the queue doesn't exist: ${lockedTaskRun.queue}`, + }, + tx: prisma, + }); + } + + return null; + } + + const currentAttemptNumber = lockedTaskRun.attempts.at(0)?.number ?? 0; + const nextAttemptNumber = currentAttemptNumber + 1; + + const newSnapshot = await this.#createExecutionSnapshot(prisma, { + run: { + id: runId, + status: snapshot.runStatus, + }, + snapshot: { + executionStatus: "PENDING_EXECUTING", + description: "Run was dequeued for execution", + }, + checkpointId: snapshot.checkpointId ?? undefined, + completedWaitpointIds: snapshot.completedWaitpoints.map((wp) => wp.id), + }); + + return { + version: "1" as const, + snapshot: { + id: newSnapshot.id, + friendlyId: newSnapshot.friendlyId, + executionStatus: newSnapshot.executionStatus, + description: newSnapshot.description, + }, + image: result.deployment?.imageReference ?? undefined, + checkpoint: newSnapshot.checkpoint ?? undefined, + completedWaitpoints: snapshot.completedWaitpoints, + backgroundWorker: { + id: result.worker.id, + friendlyId: result.worker.friendlyId, + version: result.worker.version, + }, + deployment: { + id: result.deployment?.id, + friendlyId: result.deployment?.friendlyId, + }, + run: { + id: lockedTaskRun.id, + friendlyId: lockedTaskRun.friendlyId, + isTest: lockedTaskRun.isTest, + machine: machinePreset, + attemptNumber: nextAttemptNumber, + masterQueue: lockedTaskRun.masterQueue, + traceContext: lockedTaskRun.traceContext as Record, + }, + environment: { + id: lockedTaskRun.runtimeEnvironment.id, + type: lockedTaskRun.runtimeEnvironment.type, + }, + organization: { + id: orgId, + }, + project: { + id: lockedTaskRun.projectId, + }, + } satisfies DequeuedMessage; + }); + + if (dequeuedRun !== null) { + dequeuedRuns.push(dequeuedRun); + } + } catch (error) { + this.logger.error( + "RunEngine.dequeueFromMasterQueue(): Thrown error while preparing run to be run", + { + error, + runId, + } + ); + + const run = await prisma.taskRun.findFirst({ where: { id: runId } }); + + if (!run) { + //this isn't ideal because we're not creating a snapshot… but we can't do much else + this.logger.error( + "RunEngine.dequeueFromMasterQueue(): Thrown error, then run not found. Nacking.", + { + runId, + orgId, + } + ); + await this.runQueue.nackMessage({ orgId, messageId: runId }); + continue; + } + + //this is an unknown error, we'll reattempt (with auto-backoff and eventually DLQ) + const gotRequeued = await this.#tryNackAndRequeue({ + run, + orgId, + error: { + type: "INTERNAL_ERROR", + code: "TASK_RUN_DEQUEUED_MAX_RETRIES", + message: `We tried to dequeue the run the maximum number of times but it wouldn't start executing`, + }, + tx: prisma, + }); + //we don't need this, but it makes it clear we're in a loop here + continue; + } + } + + return dequeuedRuns; + }); + } + + async dequeueFromEnvironmentMasterQueue({ + consumerId, + environmentId, + maxRunCount, + maxResources, + backgroundWorkerId, + tx, + }: { + consumerId: string; + environmentId: string; + maxRunCount: number; + maxResources?: MachineResources; + backgroundWorkerId: string; + tx?: PrismaClientOrTransaction; + }): Promise { + return this.dequeueFromMasterQueue({ + consumerId, + masterQueue: this.#environmentMasterQueueKey(environmentId), + maxRunCount, + maxResources, + backgroundWorkerId, + tx, + }); + } + + async dequeueFromBackgroundWorkerMasterQueue({ + consumerId, + backgroundWorkerId, + maxRunCount, + maxResources, + tx, + }: { + consumerId: string; + backgroundWorkerId: string; + maxRunCount: number; + maxResources?: MachineResources; + tx?: PrismaClientOrTransaction; + }): Promise { + return this.dequeueFromMasterQueue({ + consumerId, + masterQueue: this.#backgroundWorkerQueueKey(backgroundWorkerId), + maxRunCount, + maxResources, + backgroundWorkerId, + tx, + }); + } + + async startRunAttempt({ + runId, + snapshotId, + isWarmStart, + tx, + }: { + runId: string; + snapshotId: string; + isWarmStart?: boolean; + tx?: PrismaClientOrTransaction; + }): Promise { + const prisma = tx ?? this.prisma; + + return this.#trace("startRunAttempt", { runId, snapshotId }, async (span) => { + return this.runLock.lock([runId], 5000, async (signal) => { + const latestSnapshot = await getLatestExecutionSnapshot(prisma, runId); + + if (latestSnapshot.id !== snapshotId) { + //if there is a big delay between the snapshot and the attempt, the snapshot might have changed + //we just want to log because elsewhere it should have been put back into a state where it can be attempted + this.logger.warn( + "RunEngine.createRunAttempt(): snapshot has changed since the attempt was created, ignoring." + ); + throw new ServiceValidationError("Snapshot changed", 409); + } + + const environment = await this.#getAuthenticatedEnvironmentFromRun(runId, prisma); + if (!environment) { + throw new ServiceValidationError("Environment not found", 404); + } + + const taskRun = await prisma.taskRun.findFirst({ + where: { + id: runId, + }, + include: { + tags: true, + lockedBy: { + include: { + worker: { + select: { + id: true, + version: true, + sdkVersion: true, + cliVersion: true, + supportsLazyAttempts: true, + }, + }, + }, + }, + batchItems: { + include: { + batchTaskRun: true, + }, + }, + }, + }); + + this.logger.debug("Creating a task run attempt", { taskRun }); + + if (!taskRun) { + throw new ServiceValidationError("Task run not found", 404); + } + + span.setAttribute("projectId", taskRun.projectId); + span.setAttribute("environmentId", taskRun.runtimeEnvironmentId); + span.setAttribute("taskRunId", taskRun.id); + span.setAttribute("taskRunFriendlyId", taskRun.friendlyId); + + if (taskRun.status === "CANCELED") { + throw new ServiceValidationError("Task run is cancelled", 400); + } + + if (!taskRun.lockedBy) { + throw new ServiceValidationError("Task run is not locked", 400); + } + + const queue = await prisma.taskQueue.findUnique({ + where: { + runtimeEnvironmentId_name: { + runtimeEnvironmentId: environment.id, + name: taskRun.queue, + }, + }, + }); + + if (!queue) { + throw new ServiceValidationError("Queue not found", 404); + } + + //increment the attempt number (start at 1) + const nextAttemptNumber = (taskRun.attemptNumber ?? 0) + 1; + + if (nextAttemptNumber > MAX_TASK_RUN_ATTEMPTS) { + await this.#attemptFailed({ + runId: taskRun.id, + snapshotId, + completion: { + ok: false, + id: taskRun.id, + error: { + type: "INTERNAL_ERROR", + code: "TASK_RUN_CRASHED", + message: "Max attempts reached.", + }, + }, + tx: prisma, + }); + throw new ServiceValidationError("Max attempts reached", 400); + } + + this.eventBus.emit("runAttemptStarted", { + time: new Date(), + run: { + id: taskRun.id, + attemptNumber: nextAttemptNumber, + baseCostInCents: taskRun.baseCostInCents, + }, + organization: { + id: environment.organization.id, + }, + }); + + const result = await $transaction( + prisma, + async (tx) => { + const run = await tx.taskRun.update({ + where: { + id: taskRun.id, + }, + data: { + status: "EXECUTING", + attemptNumber: nextAttemptNumber, + }, + include: { + tags: true, + lockedBy: { + include: { worker: true }, + }, + }, + }); + + const newSnapshot = await this.#createExecutionSnapshot(tx, { + run, + snapshot: { + executionStatus: "EXECUTING", + description: `Attempt created, starting execution${ + isWarmStart ? " (warm start)" : "" + }`, + }, + }); + + if (taskRun.ttl) { + //don't expire the run, it's going to execute + await this.worker.ack(`expireRun:${taskRun.id}`); + } + + return { run, snapshot: newSnapshot }; + }, + (error) => { + this.logger.error("RunEngine.createRunAttempt(): prisma.$transaction error", { + code: error.code, + meta: error.meta, + stack: error.stack, + message: error.message, + name: error.name, + }); + throw new ServiceValidationError( + "Failed to update task run and execution snapshot", + 500 + ); + } + ); + + if (!result) { + this.logger.error("RunEngine.createRunAttempt(): failed to create task run attempt", { + runId: taskRun.id, + nextAttemptNumber, + }); + throw new ServiceValidationError("Failed to create task run attempt", 500); + } + + const { run, snapshot } = result; + + const machinePreset = machinePresetFromConfig({ + machines: this.options.machines.machines, + defaultMachine: this.options.machines.defaultMachine, + config: taskRun.lockedBy.machineConfig ?? {}, + }); + + const metadata = await parsePacket({ + data: taskRun.metadata ?? undefined, + dataType: taskRun.metadataType, + }); + + const execution: TaskRunExecution = { + task: { + id: run.lockedBy!.slug, + filePath: run.lockedBy!.filePath, + exportName: run.lockedBy!.exportName, + }, + attempt: { + number: nextAttemptNumber, + startedAt: latestSnapshot.updatedAt, + /** @deprecated */ + id: "deprecated", + /** @deprecated */ + backgroundWorkerId: "deprecated", + /** @deprecated */ + backgroundWorkerTaskId: "deprecated", + /** @deprecated */ + status: "deprecated", + }, + run: { + id: run.friendlyId, + payload: run.payload, + payloadType: run.payloadType, + createdAt: run.createdAt, + tags: run.tags.map((tag) => tag.name), + isTest: run.isTest, + idempotencyKey: run.idempotencyKey ?? undefined, + startedAt: run.startedAt ?? run.createdAt, + maxAttempts: run.maxAttempts ?? undefined, + version: run.lockedBy!.worker.version, + metadata, + maxDuration: run.maxDurationInSeconds ?? undefined, + /** @deprecated */ + context: undefined, + /** @deprecated */ + durationMs: run.usageDurationMs, + /** @deprecated */ + costInCents: run.costInCents, + /** @deprecated */ + baseCostInCents: run.baseCostInCents, + traceContext: run.traceContext as Record, + }, + queue: { + id: queue.friendlyId, + name: queue.name, + }, + environment: { + id: environment.id, + slug: environment.slug, + type: environment.type, + }, + organization: { + id: environment.organization.id, + slug: environment.organization.slug, + name: environment.organization.title, + }, + project: { + id: environment.project.id, + ref: environment.project.externalRef, + slug: environment.project.slug, + name: environment.project.name, + }, + batch: + taskRun.batchItems[0] && taskRun.batchItems[0].batchTaskRun + ? { id: taskRun.batchItems[0].batchTaskRun.friendlyId } + : undefined, + machine: machinePreset, + }; + + return { run, snapshot, execution }; + }); + }); + } + + /** How a run is completed */ + async completeRunAttempt({ + runId, + snapshotId, + completion, + }: { + runId: string; + snapshotId: string; + completion: TaskRunExecutionResult; + }): Promise { + switch (completion.ok) { + case true: { + return this.#attemptSucceeded({ runId, snapshotId, completion, tx: this.prisma }); + } + case false: { + return this.#attemptFailed({ runId, snapshotId, completion, tx: this.prisma }); + } + } + } + + async waitForDuration({ + runId, + snapshotId, + date, + releaseConcurrency = true, + idempotencyKey, + tx, + }: { + runId: string; + snapshotId: string; + date: Date; + releaseConcurrency?: boolean; + idempotencyKey?: string; + tx?: PrismaClientOrTransaction; + }): Promise { + const prisma = tx ?? this.prisma; + + return await this.runLock.lock([runId], 5_000, async (signal) => { + const snapshot = await getLatestExecutionSnapshot(prisma, runId); + + if (snapshot.id !== snapshotId) { + throw new ServiceValidationError("Snapshot ID doesn't match the latest snapshot", 400); + } + + const run = await prisma.taskRun.findFirst({ + select: { + runtimeEnvironment: { + select: { + id: true, + organizationId: true, + }, + }, + projectId: true, + }, + where: { id: runId }, + }); + + if (!run) { + throw new ServiceValidationError("TaskRun not found", 404); + } + + let waitpoint = idempotencyKey + ? await prisma.waitpoint.findUnique({ + where: { + environmentId_idempotencyKey: { + environmentId: run.runtimeEnvironment.id, + idempotencyKey, + }, + }, + }) + : undefined; + + if (!waitpoint) { + waitpoint = await this.#createDateTimeWaitpoint(prisma, { + projectId: run.projectId, + environmentId: run.runtimeEnvironment.id, + completedAfter: date, + idempotencyKey, + }); + } + + //waitpoint already completed, so we don't need to wait + if (waitpoint.status === "COMPLETED") { + return { + waitUntil: waitpoint.completedAt ?? new Date(), + waitpoint: { + id: waitpoint.id, + }, + ...executionResultFromSnapshot(snapshot), + }; + } + + //block the run + const blockResult = await this.blockRunWithWaitpoint({ + runId, + waitpointId: waitpoint.id, + environmentId: waitpoint.environmentId, + projectId: waitpoint.projectId, + tx: prisma, + }); + + //release concurrency + await this.runQueue.releaseConcurrency( + run.runtimeEnvironment.organizationId, + runId, + releaseConcurrency + ); + + return { + waitUntil: date, + waitpoint: { + id: waitpoint.id, + }, + ...executionResultFromSnapshot(blockResult), + }; + }); + } + + /** + Call this to cancel a run. + If the run is in-progress it will change it's state to PENDING_CANCEL and notify the worker. + If the run is not in-progress it will finish it. + You can pass `finalizeRun` in if you know it's no longer running, e.g. the worker has messaged to say it's done. + */ + async cancelRun({ + runId, + completedAt, + reason, + finalizeRun, + tx, + }: { + runId: string; + completedAt?: Date; + reason?: string; + finalizeRun?: boolean; + tx?: PrismaClientOrTransaction; + }): Promise { + const prisma = tx ?? this.prisma; + reason = reason ?? "Cancelled by user"; + + return this.#trace("cancelRun", { runId }, async (span) => { + return this.runLock.lock([runId], 5_000, async (signal) => { + const latestSnapshot = await getLatestExecutionSnapshot(prisma, runId); + + //already finished, do nothing + if (latestSnapshot.executionStatus === "FINISHED") { + return executionResultFromSnapshot(latestSnapshot); + } + + //is pending cancellation and we're not finalizing, alert the worker again + if (latestSnapshot.executionStatus === "PENDING_CANCEL" && !finalizeRun) { + await this.#sendNotificationToWorker({ runId }); + return executionResultFromSnapshot(latestSnapshot); + } + + //set the run to cancelled immediately + const error: TaskRunError = { + type: "STRING_ERROR", + raw: reason, + }; + + const run = await prisma.taskRun.update({ + where: { id: runId }, + data: { + status: "CANCELED", + completedAt: finalizeRun ? completedAt ?? new Date() : completedAt, + error, + }, + include: { + runtimeEnvironment: true, + associatedWaitpoint: true, + childRuns: { + select: { + id: true, + }, + }, + }, + }); + + //remove it from the queue and release concurrency + await this.runQueue.acknowledgeMessage(run.runtimeEnvironment.organizationId, runId); + + //if executing, we need to message the worker to cancel the run and put it into `PENDING_CANCEL` status + if (isExecuting(latestSnapshot.executionStatus)) { + const newSnapshot = await this.#createExecutionSnapshot(prisma, { + run, + snapshot: { + executionStatus: "PENDING_CANCEL", + description: "Run was cancelled", + }, + }); + + //the worker needs to be notified so it can kill the run and complete the attempt + await this.#sendNotificationToWorker({ runId }); + return executionResultFromSnapshot(newSnapshot); + } + + //not executing, so we will actually finish the run + const newSnapshot = await this.#createExecutionSnapshot(prisma, { + run, + snapshot: { + executionStatus: "FINISHED", + description: "Run was cancelled, not finished", + }, + }); + + if (!run.associatedWaitpoint) { + throw new ServiceValidationError("No associated waitpoint found", 400); + } + + //complete the waitpoint so the parent run can continue + await this.completeWaitpoint({ + id: run.associatedWaitpoint.id, + output: { value: JSON.stringify(error), isError: true }, + }); + + this.eventBus.emit("runCancelled", { + time: new Date(), + run: { + id: run.id, + friendlyId: run.friendlyId, + spanId: run.spanId, + error, + }, + }); + + //schedule the cancellation of all the child runs + //it will call this function for each child, + //which will recursively cancel all children if they need to be + if (run.childRuns.length > 0) { + for (const childRun of run.childRuns) { + await this.worker.enqueue({ + id: `cancelRun:${childRun.id}`, + job: "cancelRun", + payload: { runId: childRun.id, completedAt: run.completedAt ?? new Date(), reason }, + }); + } + } + + await this.#finalizeRun(run); + + return executionResultFromSnapshot(newSnapshot); + }); + }); + } + + async queueRunsWaitingForWorker({ + backgroundWorkerId, + }: { + backgroundWorkerId: string; + }): Promise { + //we want this to happen in the background + await this.worker.enqueue({ + job: "queueRunsWaitingForWorker", + payload: { backgroundWorkerId }, + }); + } + + /** + * Reschedules a delayed run where the run hasn't been queued yet + */ + async rescheduleRun({ + runId, + delayUntil, + tx, + }: { + runId: string; + delayUntil: Date; + tx?: PrismaClientOrTransaction; + }): Promise { + const prisma = tx ?? this.prisma; + return this.#trace("rescheduleRun", { runId }, async (span) => { + return await this.runLock.lock([runId], 5_000, async (signal) => { + const snapshot = await getLatestExecutionSnapshot(prisma, runId); + + //if the run isn't just created then we can't reschedule it + if (snapshot.executionStatus !== "RUN_CREATED") { + throw new ServiceValidationError("Cannot reschedule a run that is not delayed"); + } + + const updatedRun = await prisma.taskRun.update({ + where: { + id: runId, + }, + data: { + delayUntil: delayUntil, + executionSnapshots: { + create: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Delayed run was rescheduled to a future date", + runStatus: "EXPIRED", + }, + }, + }, + include: { + blockedByWaitpoints: true, + }, + }); + + if (updatedRun.blockedByWaitpoints.length === 0) { + throw new ServiceValidationError( + "Cannot reschedule a run that is not blocked by a waitpoint" + ); + } + + const result = await this.#rescheduleDateTimeWaitpoint( + prisma, + updatedRun.blockedByWaitpoints[0].waitpointId, + delayUntil + ); + + if (!result.success) { + throw new ServiceValidationError("Failed to reschedule waitpoint, too late.", 400); + } + + return updatedRun; + }); + }); + } + + async lengthOfEnvQueue(environment: MinimalAuthenticatedEnvironment): Promise { + return this.runQueue.lengthOfEnvQueue(environment); + } + + /** This creates a MANUAL waitpoint, that can be explicitly completed (or failed). + * If you pass an `idempotencyKey` and it already exists, it will return the existing waitpoint. + */ + async createManualWaitpoint({ + environmentId, + projectId, + idempotencyKey, + }: { + environmentId: string; + projectId: string; + idempotencyKey?: string; + }): Promise { + const existingWaitpoint = idempotencyKey + ? await this.prisma.waitpoint.findUnique({ + where: { + environmentId_idempotencyKey: { + environmentId, + idempotencyKey, + }, + }, + }) + : undefined; + + if (existingWaitpoint) { + return existingWaitpoint; + } + + return this.prisma.waitpoint.create({ + data: { + ...WaitpointId.generate(), + type: "MANUAL", + idempotencyKey: idempotencyKey ?? nanoid(24), + userProvidedIdempotencyKey: !!idempotencyKey, + environmentId, + projectId, + }, + }); + } + + async getWaitpoint({ + waitpointId, + environmentId, + projectId, + }: { + environmentId: string; + projectId: string; + waitpointId: string; + }): Promise { + const waitpoint = await this.prisma.waitpoint.findFirst({ + where: { id: waitpointId }, + include: { + blockingTaskRuns: { + select: { + taskRun: { + select: { + id: true, + friendlyId: true, + }, + }, + }, + }, + }, + }); + + if (!waitpoint) return null; + if (waitpoint.environmentId !== environmentId) return null; + + return waitpoint; + } + + /** + * Prevents a run from continuing until the waitpoint is completed. + */ + async blockRunWithWaitpoint({ + runId, + waitpointId, + projectId, + failAfter, + tx, + }: { + runId: string; + waitpointId: string | string[]; + environmentId: string; + projectId: string; + failAfter?: Date; + tx?: PrismaClientOrTransaction; + }): Promise { + const prisma = tx ?? this.prisma; + + let waitpointIds = typeof waitpointId === "string" ? [waitpointId] : waitpointId; + + return await this.runLock.lock([runId], 5000, async (signal) => { + let snapshot: TaskRunExecutionSnapshot = await getLatestExecutionSnapshot(prisma, runId); + + //block the run with the waitpoints, returning how many waitpoints are pending + const insert = await prisma.$queryRaw<{ pending_count: BigInt }[]>` + WITH inserted AS ( + INSERT INTO "TaskRunWaitpoint" ("id", "taskRunId", "waitpointId", "projectId", "createdAt", "updatedAt") + SELECT + gen_random_uuid(), + ${runId}, + w.id, + ${projectId}, + NOW(), + NOW() + FROM "Waitpoint" w + WHERE w.id IN (${Prisma.join(waitpointIds)}) + ON CONFLICT DO NOTHING + RETURNING "waitpointId" + ) + SELECT COUNT(*) as pending_count + FROM inserted i + JOIN "Waitpoint" w ON w.id = i."waitpointId" + WHERE w.status = 'PENDING';`; + + const pendingCount = Number(insert.at(0)?.pending_count ?? 0); + + let newStatus: TaskRunExecutionStatus = "BLOCKED_BY_WAITPOINTS"; + if ( + snapshot.executionStatus === "EXECUTING" || + snapshot.executionStatus === "EXECUTING_WITH_WAITPOINTS" + ) { + newStatus = "EXECUTING_WITH_WAITPOINTS"; + } + + //if the state has changed, create a new snapshot + if (newStatus !== snapshot.executionStatus) { + snapshot = await this.#createExecutionSnapshot(prisma, { + run: { + id: snapshot.runId, + status: snapshot.runStatus, + attemptNumber: snapshot.attemptNumber, + }, + snapshot: { + executionStatus: newStatus, + description: "Run was blocked by a waitpoint.", + }, + }); + } + + if (failAfter) { + for (const waitpointId of waitpointIds) { + await this.worker.enqueue({ + id: `finishWaitpoint.${waitpointId}`, + job: "finishWaitpoint", + payload: { waitpointId, error: "Waitpoint timed out" }, + availableAt: failAfter, + }); + } + } + + //no pending waitpoint, schedule unblocking the run + //debounce if we're rapidly adding waitpoints + if (pendingCount === 0) { + await this.worker.enqueue({ + //this will debounce the call + id: `continueRunIfUnblocked:${runId}`, + job: "continueRunIfUnblocked", + payload: { runId: runId }, + //100ms in the future + availableAt: new Date(Date.now() + 100), + }); + } + + return snapshot; + }); + } + + /** This completes a waitpoint and updates all entries so the run isn't blocked, + * if they're no longer blocked. This doesn't suffer from race conditions. */ + async completeWaitpoint({ + id, + output, + }: { + id: string; + output?: { + value: string; + type?: string; + isError: boolean; + }; + }): Promise { + const waitpoint = await this.prisma.waitpoint.findUnique({ + where: { id }, + }); + + if (!waitpoint) { + throw new Error(`Waitpoint ${id} not found`); + } + + const result = await $transaction( + this.prisma, + async (tx) => { + // 1. Find the TaskRuns blocked by this waitpoint + const affectedTaskRuns = await tx.taskRunWaitpoint.findMany({ + where: { waitpointId: id }, + select: { taskRunId: true }, + }); + + if (affectedTaskRuns.length === 0) { + this.logger.warn(`No TaskRunWaitpoints found for waitpoint`, { + waitpoint, + }); + } + + // 2. Update the waitpoint to completed + const updatedWaitpoint = await tx.waitpoint.update({ + where: { id }, + data: { + status: "COMPLETED", + completedAt: new Date(), + output: output?.value, + outputType: output?.type, + outputIsError: output?.isError, + }, + }); + + return { updatedWaitpoint, affectedTaskRuns }; + }, + (error) => { + this.logger.error(`Error completing waitpoint ${id}, retrying`, { error }); + throw error; + } + ); + + if (!result) { + throw new Error(`Waitpoint couldn't be updated`); + } + + //schedule trying to continue the runs + for (const run of result.affectedTaskRuns) { + await this.worker.enqueue({ + //this will debounce the call + id: `continueRunIfUnblocked:${run.taskRunId}`, + job: "continueRunIfUnblocked", + payload: { runId: run.taskRunId }, + //50ms in the future + availableAt: new Date(Date.now() + 50), + }); + } + + return result.updatedWaitpoint; + } + + async createCheckpoint({ + runId, + snapshotId, + checkpoint, + tx, + }: { + runId: string; + snapshotId: string; + //todo + checkpoint: Record; + tx?: PrismaClientOrTransaction; + }) { + const prisma = tx ?? this.prisma; + + return await this.runLock.lock([runId], 5_000, async (signal) => { + const snapshot = await getLatestExecutionSnapshot(prisma, runId); + if (snapshot.id !== snapshotId) { + return { + ok: false as const, + error: "Not the latest snapshot", + }; + } + + //todo check the status is checkpointable + if (!isCheckpointable(snapshot.executionStatus)) { + this.logger.error("Tried to createCheckpoint on a run in an invalid state", { + snapshot, + }); + + //check if the server should already be shutting down, if so return a result saying it can shutdown but that there's no checkpoint + + //otherwise return a result saying it can't checkpoint with an error and execution status + + return; + } + + //create a new execution snapshot, with the checkpoint + + //todo return a Result, which will determine if the server is allowed to shutdown + }); + } + + /** + Send a heartbeat to signal the the run is still executing. + If a heartbeat isn't received, after a while the run is considered "stalled" + and some logic will be run to try recover it. + @returns The ExecutionResult, which could be a different snapshot. + */ + async heartbeatRun({ + runId, + snapshotId, + tx, + }: { + runId: string; + snapshotId: string; + tx?: PrismaClientOrTransaction; + }): Promise { + const prisma = tx ?? this.prisma; + + //we don't need to acquire a run lock for any of this, it's not critical if it happens on an older version + const latestSnapshot = await getLatestExecutionSnapshot(prisma, runId); + if (latestSnapshot.id !== snapshotId) { + this.logger.log("heartbeatRun no longer the latest snapshot, stopping the heartbeat.", { + runId, + snapshotId, + latestSnapshot: latestSnapshot, + }); + + await this.worker.ack(`heartbeatSnapshot.${snapshotId}`); + return executionResultFromSnapshot(latestSnapshot); + } + + //update the snapshot heartbeat time + await prisma.taskRunExecutionSnapshot.update({ + where: { id: latestSnapshot.id }, + data: { + lastHeartbeatAt: new Date(), + }, + }); + + //extending is the same as creating a new heartbeat + await this.#setHeartbeatDeadline({ runId, snapshotId, status: latestSnapshot.executionStatus }); + + return executionResultFromSnapshot(latestSnapshot); + } + + /** Get required data to execute the run */ + async getRunExecutionData({ + runId, + tx, + }: { + runId: string; + tx?: PrismaClientOrTransaction; + }): Promise { + const prisma = tx ?? this.prisma; + try { + const snapshot = await getLatestExecutionSnapshot(prisma, runId); + + const executionData: RunExecutionData = { + version: "1" as const, + snapshot: { + id: snapshot.id, + friendlyId: snapshot.friendlyId, + executionStatus: snapshot.executionStatus, + description: snapshot.description, + }, + run: { + id: snapshot.runId, + friendlyId: snapshot.runFriendlyId, + status: snapshot.runStatus, + attemptNumber: snapshot.attemptNumber ?? undefined, + }, + checkpoint: snapshot.checkpoint + ? { + id: snapshot.checkpoint.id, + friendlyId: snapshot.checkpoint.friendlyId, + type: snapshot.checkpoint.type, + location: snapshot.checkpoint.location, + imageRef: snapshot.checkpoint.imageRef, + reason: snapshot.checkpoint.reason ?? undefined, + } + : undefined, + completedWaitpoints: snapshot.completedWaitpoints, + }; + + return executionData; + } catch (e) { + this.logger.error("Failed to getRunExecutionData", { + message: e instanceof Error ? e.message : e, + }); + return null; + } + } + + async quit() { + //stop the run queue + await this.runQueue.quit(); + await this.worker.stop(); + await this.runLock.quit(); + + try { + // This is just a failsafe + await this.redis.quit(); + } catch (error) { + // And should always throw + } + } + + async #systemFailure({ + runId, + error, + tx, + }: { + runId: string; + error: TaskRunInternalError; + tx?: PrismaClientOrTransaction; + }): Promise { + const prisma = tx ?? this.prisma; + return this.#trace("#systemFailure", { runId }, async (span) => { + const latestSnapshot = await getLatestExecutionSnapshot(prisma, runId); + + //already finished + if (latestSnapshot.executionStatus === "FINISHED") { + //todo check run is in the correct state + return { + attemptStatus: "RUN_FINISHED", + snapshot: latestSnapshot, + run: { + id: runId, + friendlyId: latestSnapshot.runFriendlyId, + status: latestSnapshot.runStatus, + attemptNumber: latestSnapshot.attemptNumber, + }, + }; + } + + const result = await this.#attemptFailed({ + runId, + snapshotId: latestSnapshot.id, + completion: { + ok: false, + id: runId, + error, + }, + tx: prisma, + }); + + return result; + }); + } + + async #expireRun({ runId, tx }: { runId: string; tx?: PrismaClientOrTransaction }) { + const prisma = tx ?? this.prisma; + await this.runLock.lock([runId], 5_000, async (signal) => { + const snapshot = await getLatestExecutionSnapshot(prisma, runId); + + //if we're executing then we won't expire the run + if (isExecuting(snapshot.executionStatus)) { + return; + } + + //only expire "PENDING" runs + const run = await prisma.taskRun.findUnique({ where: { id: runId } }); + + if (!run) { + this.logger.debug("Could not find enqueued run to expire", { + runId, + }); + return; + } + + if (run.status !== "PENDING") { + this.logger.debug("Run cannot be expired because it's not in PENDING status", { + run, + }); + return; + } + + if (run.lockedAt) { + this.logger.debug("Run cannot be expired because it's locked, so will run", { + run, + }); + return; + } + + const error: TaskRunError = { + type: "STRING_ERROR", + raw: `Run expired because the TTL (${run.ttl}) was reached`, + }; + + const updatedRun = await prisma.taskRun.update({ + where: { id: runId }, + data: { + status: "EXPIRED", + completedAt: new Date(), + expiredAt: new Date(), + error, + executionSnapshots: { + create: { + engine: "V2", + executionStatus: "FINISHED", + description: "Run was expired because the TTL was reached", + runStatus: "EXPIRED", + }, + }, + }, + include: { + associatedWaitpoint: true, + }, + }); + + if (!updatedRun.associatedWaitpoint) { + throw new ServiceValidationError("No associated waitpoint found", 400); + } + + await this.completeWaitpoint({ + id: updatedRun.associatedWaitpoint.id, + output: { value: JSON.stringify(error), isError: true }, + }); + + this.eventBus.emit("runExpired", { run: updatedRun, time: new Date() }); + }); + } + + async #waitingForDeploy({ + orgId, + runId, + tx, + }: { + orgId: string; + runId: string; + tx?: PrismaClientOrTransaction; + }) { + const prisma = tx ?? this.prisma; + + return this.#trace("#waitingForDeploy", { runId }, async (span) => { + return this.runLock.lock([runId], 5_000, async (signal) => { + //mark run as waiting for deploy + const run = await prisma.taskRun.update({ + where: { id: runId }, + data: { + status: "WAITING_FOR_DEPLOY", + }, + }); + + await this.#createExecutionSnapshot(prisma, { + run, + snapshot: { + executionStatus: "RUN_CREATED", + description: + "The run doesn't have a background worker, so we're going to ack it for now.", + }, + }); + + //we ack because when it's deployed it will be requeued + await this.runQueue.acknowledgeMessage(orgId, runId); + }); + }); + } + + async #attemptSucceeded({ + runId, + snapshotId, + completion, + tx, + }: { + runId: string; + snapshotId: string; + completion: TaskRunSuccessfulExecutionResult; + tx: PrismaClientOrTransaction; + }): Promise { + const prisma = tx ?? this.prisma; + return this.#trace("#completeRunAttemptSuccess", { runId, snapshotId }, async (span) => { + return this.runLock.lock([runId], 5_000, async (signal) => { + const latestSnapshot = await getLatestExecutionSnapshot(prisma, runId); + + if (latestSnapshot.id !== snapshotId) { + throw new ServiceValidationError("Snapshot ID doesn't match the latest snapshot", 400); + } + + span.setAttribute("completionStatus", completion.ok); + + const completedAt = new Date(); + + const run = await prisma.taskRun.update({ + where: { id: runId }, + data: { + status: "COMPLETED_SUCCESSFULLY", + completedAt, + output: completion.output, + outputType: completion.outputType, + executionSnapshots: { + create: { + executionStatus: "FINISHED", + description: "Task completed successfully", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: latestSnapshot.attemptNumber, + }, + }, + }, + select: { + id: true, + friendlyId: true, + status: true, + attemptNumber: true, + spanId: true, + associatedWaitpoint: { + select: { + id: true, + }, + }, + project: { + select: { + organizationId: true, + }, + }, + batchId: true, + }, + }); + const newSnapshot = await getLatestExecutionSnapshot(prisma, runId); + await this.runQueue.acknowledgeMessage(run.project.organizationId, runId); + + // We need to manually emit this as we created the final snapshot as part of the task run update + this.eventBus.emit("executionSnapshotCreated", { + time: newSnapshot.createdAt, + run: { + id: newSnapshot.runId, + }, + snapshot: { + ...newSnapshot, + completedWaitpointIds: newSnapshot.completedWaitpoints.map((wp) => wp.id), + }, + }); + + if (!run.associatedWaitpoint) { + throw new ServiceValidationError("No associated waitpoint found", 400); + } + + await this.completeWaitpoint({ + id: run.associatedWaitpoint.id, + output: completion.output + ? { value: completion.output, type: completion.outputType, isError: false } + : undefined, + }); + + this.eventBus.emit("runSucceeded", { + time: completedAt, + run: { + id: runId, + spanId: run.spanId, + output: completion.output, + outputType: completion.outputType, + }, + }); + + await this.#finalizeRun(run); + + return { + attemptStatus: "RUN_FINISHED", + snapshot: newSnapshot, + run, + }; + }); + }); + } + + async #attemptFailed({ + runId, + snapshotId, + completion, + forceRequeue, + tx, + }: { + runId: string; + snapshotId: string; + completion: TaskRunFailedExecutionResult; + forceRequeue?: boolean; + tx: PrismaClientOrTransaction; + }): Promise { + const prisma = this.prisma; + + return this.#trace("completeRunAttemptFailure", { runId, snapshotId }, async (span) => { + return this.runLock.lock([runId], 5_000, async (signal) => { + const latestSnapshot = await getLatestExecutionSnapshot(prisma, runId); + + if (latestSnapshot.id !== snapshotId) { + throw new ServiceValidationError("Snapshot ID doesn't match the latest snapshot", 400); + } + + span.setAttribute("completionStatus", completion.ok); + + //remove waitpoints blocking the run + const deletedCount = await this.#clearBlockingWaitpoints({ runId, tx }); + if (deletedCount > 0) { + this.logger.debug("Cleared blocking waitpoints", { runId, deletedCount }); + } + + const failedAt = new Date(); + + if ( + completion.error.type === "INTERNAL_ERROR" && + completion.error.code === "TASK_RUN_CANCELLED" + ) { + // We need to cancel the task run instead of fail it + const result = await this.cancelRun({ + runId, + completedAt: failedAt, + reason: completion.error.message, + finalizeRun: true, + tx: prisma, + }); + return { + attemptStatus: + result.snapshot.executionStatus === "PENDING_CANCEL" + ? "RUN_PENDING_CANCEL" + : "RUN_FINISHED", + ...result, + }; + } + + const error = sanitizeError(completion.error); + const retriableError = shouldRetryError(taskRunErrorEnhancer(completion.error)); + + if ( + retriableError && + completion.retry !== undefined && + latestSnapshot.attemptNumber !== null && + latestSnapshot.attemptNumber < MAX_TASK_RUN_ATTEMPTS + ) { + const retryAt = new Date(completion.retry.timestamp); + + const run = await prisma.taskRun.update({ + where: { + id: runId, + }, + data: { + status: "RETRYING_AFTER_FAILURE", + }, + include: { + runtimeEnvironment: { + include: { + project: true, + organization: true, + orgMember: true, + }, + }, + }, + }); + + const nextAttemptNumber = + latestSnapshot.attemptNumber === null ? 1 : latestSnapshot.attemptNumber + 1; + + this.eventBus.emit("runRetryScheduled", { + time: failedAt, + run: { + id: run.id, + friendlyId: run.friendlyId, + attemptNumber: nextAttemptNumber, + queue: run.queue, + taskIdentifier: run.taskIdentifier, + traceContext: run.traceContext as Record, + baseCostInCents: run.baseCostInCents, + spanId: run.spanId, + }, + organization: { + id: run.runtimeEnvironment.organizationId, + }, + environment: run.runtimeEnvironment, + retryAt, + }); + + //todo anything special for DEV? Ideally not. + + //if it's a long delay and we support checkpointing, put it back in the queue + if ( + forceRequeue || + (this.options.retryWarmStartThresholdMs !== undefined && + completion.retry.delay >= this.options.retryWarmStartThresholdMs) + ) { + //we nack the message, requeuing it for later + const nackResult = await this.#tryNackAndRequeue({ + run, + orgId: run.runtimeEnvironment.organizationId, + timestamp: retryAt.getTime(), + error: { + type: "INTERNAL_ERROR", + code: "TASK_RUN_DEQUEUED_MAX_RETRIES", + message: `We tried to dequeue the run the maximum number of times but it wouldn't start executing`, + }, + tx: prisma, + }); + + if (!nackResult.wasRequeued) { + return { + attemptStatus: "RUN_FINISHED", + ...nackResult, + }; + } else { + return { attemptStatus: "RETRY_QUEUED", ...nackResult }; + } + } + + //it will continue running because the retry delay is short + const newSnapshot = await this.#createExecutionSnapshot(prisma, { + run, + snapshot: { + executionStatus: "PENDING_EXECUTING", + description: "Attempt failed wth a short delay, starting a new attempt.", + }, + }); + //the worker can fetch the latest snapshot and should create a new attempt + await this.#sendNotificationToWorker({ runId }); + + return { + attemptStatus: "RETRY_IMMEDIATELY", + ...executionResultFromSnapshot(newSnapshot), + }; + } + + const status = runStatusFromError(completion.error); + + //run permanently failed + const run = await prisma.taskRun.update({ + where: { + id: runId, + }, + data: { + status, + completedAt: failedAt, + error, + }, + include: { + runtimeEnvironment: true, + associatedWaitpoint: true, + }, + }); + + const newSnapshot = await this.#createExecutionSnapshot(prisma, { + run, + snapshot: { + executionStatus: "FINISHED", + description: "Run failed", + }, + }); + + if (!run.associatedWaitpoint) { + throw new ServiceValidationError("No associated waitpoint found", 400); + } + + await this.completeWaitpoint({ + id: run.associatedWaitpoint.id, + output: { value: JSON.stringify(error), isError: true }, + }); + + await this.runQueue.acknowledgeMessage(run.runtimeEnvironment.organizationId, runId); + + this.eventBus.emit("runFailed", { + time: failedAt, + run: { + id: runId, + status: run.status, + spanId: run.spanId, + error, + }, + }); + + await this.#finalizeRun(run); + + return { + attemptStatus: "RUN_FINISHED", + snapshot: newSnapshot, + run, + }; + }); + }); + } + + //MARK: RunQueue + + /** The run can be added to the queue. When it's pulled from the queue it will be executed. */ + async #enqueueRun({ + run, + env, + timestamp, + tx, + }: { + run: TaskRun; + env: MinimalAuthenticatedEnvironment; + timestamp: number; + tx?: PrismaClientOrTransaction; + }) { + const prisma = tx ?? this.prisma; + + await this.runLock.lock([run.id], 5000, async (signal) => { + const newSnapshot = await this.#createExecutionSnapshot(prisma, { + run: run, + snapshot: { + executionStatus: "QUEUED", + description: "Run was QUEUED", + }, + }); + + const masterQueues = [run.masterQueue]; + if (run.secondaryMasterQueue) { + masterQueues.push(run.secondaryMasterQueue); + } + + await this.runQueue.enqueueMessage({ + env, + masterQueues, + message: { + runId: run.id, + taskIdentifier: run.taskIdentifier, + orgId: env.organization.id, + projectId: env.project.id, + environmentId: env.id, + environmentType: env.type, + queue: run.queue, + concurrencyKey: run.concurrencyKey ?? undefined, + timestamp, + attempt: 0, + }, + }); + }); + } + + async #tryNackAndRequeue({ + run, + orgId, + timestamp, + error, + tx, + }: { + run: TaskRun; + orgId: string; + timestamp?: number; + error: TaskRunInternalError; + tx?: PrismaClientOrTransaction; + }): Promise<{ wasRequeued: boolean } & ExecutionResult> { + const prisma = tx ?? this.prisma; + + return await this.runLock.lock([run.id], 5000, async (signal) => { + //we nack the message, this allows another work to pick up the run + const gotRequeued = await this.runQueue.nackMessage({ + orgId, + messageId: run.id, + retryAt: timestamp, + }); + + if (!gotRequeued) { + const result = await this.#systemFailure({ + runId: run.id, + error, + tx: prisma, + }); + return { wasRequeued: false, ...result }; + } + + const newSnapshot = await this.#createExecutionSnapshot(prisma, { + run: run, + snapshot: { + executionStatus: "QUEUED", + description: "Requeued the run after a failure", + }, + }); + + return { + wasRequeued: true, + snapshot: { + id: newSnapshot.id, + friendlyId: newSnapshot.friendlyId, + executionStatus: newSnapshot.executionStatus, + description: newSnapshot.description, + }, + run: { + id: newSnapshot.runId, + friendlyId: newSnapshot.runFriendlyId, + status: newSnapshot.runStatus, + attemptNumber: newSnapshot.attemptNumber, + }, + }; + }); + } + + async #continueRunIfUnblocked({ runId }: { runId: string }) { + // 1. Get the any blocking waitpoints + const blockingWaitpoints = await this.prisma.taskRunWaitpoint.findMany({ + where: { taskRunId: runId }, + select: { + waitpoint: { + select: { id: true, status: true }, + }, + }, + }); + + // 2. There are blockers still, so do nothing + if (blockingWaitpoints.some((w) => w.waitpoint.status !== "COMPLETED")) { + return; + } + + // 3. Get the run with environment + const run = await this.prisma.taskRun.findFirst({ + where: { + id: runId, + }, + include: { + runtimeEnvironment: { + select: { + id: true, + type: true, + maximumConcurrencyLimit: true, + project: { select: { id: true } }, + organization: { select: { id: true } }, + }, + }, + }, + }); + + if (!run) { + throw new Error(`#continueRunIfUnblocked: run not found: ${runId}`); + } + + //4. Continue the run whether it's executing or not + await this.runLock.lock([runId], 5000, async (signal) => { + const snapshot = await getLatestExecutionSnapshot(this.prisma, runId); + + //run is still executing, send a message to the worker + if (isExecuting(snapshot.executionStatus)) { + const newSnapshot = await this.#createExecutionSnapshot(this.prisma, { + run: { + id: runId, + status: snapshot.runStatus, + attemptNumber: snapshot.attemptNumber, + }, + snapshot: { + executionStatus: "EXECUTING", + description: "Run was continued, whilst still executing.", + }, + completedWaitpointIds: blockingWaitpoints.map((b) => b.waitpoint.id), + }); + + //we reacquire the concurrency if it's still running because we're not going to be dequeuing (which also does this) + await this.runQueue.reacquireConcurrency(run.runtimeEnvironment.organization.id, runId); + + await this.#sendNotificationToWorker({ runId: runId }); + } else { + const newSnapshot = await this.#createExecutionSnapshot(this.prisma, { + run, + snapshot: { + executionStatus: "QUEUED", + description: "Run is QUEUED, because all waitpoints are completed.", + }, + completedWaitpointIds: blockingWaitpoints.map((b) => b.waitpoint.id), + }); + + //put it back in the queue, with the original timestamp (w/ priority) + //this prioritizes dequeuing waiting runs over new runs + await this.#enqueueRun({ + run, + env: run.runtimeEnvironment, + timestamp: run.createdAt.getTime() - run.priorityMs, + }); + } + }); + + //5. Remove the blocking waitpoints + await this.prisma.taskRunWaitpoint.deleteMany({ + where: { + taskRunId: runId, + }, + }); + } + + async #queueRunsWaitingForWorker({ backgroundWorkerId }: { backgroundWorkerId: string }) { + //It could be a lot of runs, so we will process them in a batch + //if there are still more to process we will enqueue this function again + const maxCount = this.options.queueRunsWaitingForWorkerBatchSize ?? 200; + + const backgroundWorker = await this.prisma.backgroundWorker.findFirst({ + where: { + id: backgroundWorkerId, + }, + include: { + runtimeEnvironment: { + include: { + project: true, + organization: true, + }, + }, + tasks: true, + }, + }); + + if (!backgroundWorker) { + this.logger.error("#queueRunsWaitingForWorker: background worker not found", { + id: backgroundWorkerId, + }); + return; + } + + const runsWaitingForDeploy = await this.prisma.taskRun.findMany({ + where: { + runtimeEnvironmentId: backgroundWorker.runtimeEnvironmentId, + projectId: backgroundWorker.projectId, + status: "WAITING_FOR_DEPLOY", + taskIdentifier: { + in: backgroundWorker.tasks.map((task) => task.slug), + }, + }, + orderBy: { + createdAt: "asc", + }, + take: maxCount + 1, + }); + + //none to process + if (!runsWaitingForDeploy.length) return; + + for (const run of runsWaitingForDeploy) { + await this.prisma.$transaction(async (tx) => { + const updatedRun = await tx.taskRun.update({ + where: { + id: run.id, + }, + data: { + status: "PENDING", + }, + }); + await this.#enqueueRun({ + run: updatedRun, + env: backgroundWorker.runtimeEnvironment, + //add to the queue using the original run created time + //this should ensure they're in the correct order in the queue + timestamp: updatedRun.createdAt.getTime() - updatedRun.priorityMs, + tx, + }); + }); + } + + //enqueue more if needed + if (runsWaitingForDeploy.length > maxCount) { + await this.queueRunsWaitingForWorker({ backgroundWorkerId }); + } + } + + //MARK: - Waitpoints + async #createRunAssociatedWaitpoint( + tx: PrismaClientOrTransaction, + { + projectId, + environmentId, + completedByTaskRunId, + }: { projectId: string; environmentId: string; completedByTaskRunId: string } + ) { + return tx.waitpoint.create({ + data: { + ...WaitpointId.generate(), + type: "RUN", + status: "PENDING", + idempotencyKey: nanoid(24), + userProvidedIdempotencyKey: false, + projectId, + environmentId, + completedByTaskRunId, + }, + }); + } + + async #createDateTimeWaitpoint( + tx: PrismaClientOrTransaction, + { + projectId, + environmentId, + completedAfter, + idempotencyKey, + }: { projectId: string; environmentId: string; completedAfter: Date; idempotencyKey?: string } + ) { + const waitpoint = await tx.waitpoint.create({ + data: { + ...WaitpointId.generate(), + type: "DATETIME", + status: "PENDING", + idempotencyKey: idempotencyKey ?? nanoid(24), + userProvidedIdempotencyKey: !!idempotencyKey, + projectId, + environmentId, + completedAfter, + }, + }); + + await this.worker.enqueue({ + id: `finishWaitpoint.${waitpoint.id}`, + job: "finishWaitpoint", + payload: { waitpointId: waitpoint.id }, + availableAt: completedAfter, + }); + + return waitpoint; + } + + async #rescheduleDateTimeWaitpoint( + tx: PrismaClientOrTransaction, + waitpointId: string, + completedAfter: Date + ): Promise<{ success: true } | { success: false; error: string }> { + try { + const updatedWaitpoint = await tx.waitpoint.update({ + where: { id: waitpointId, status: "PENDING" }, + data: { + completedAfter, + }, + }); + } catch (error) { + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2025") { + return { + success: false, + error: "Waitpoint doesn't exist or is already completed", + }; + } + + this.logger.error("Error rescheduling waitpoint", { error }); + + return { + success: false, + error: "An unknown error occurred", + }; + } + + //reschedule completion + await this.worker.enqueue({ + id: `finishWaitpoint.${waitpointId}`, + job: "finishWaitpoint", + payload: { waitpointId: waitpointId }, + availableAt: completedAfter, + }); + + return { + success: true, + }; + } + + async #clearBlockingWaitpoints({ runId, tx }: { runId: string; tx?: PrismaClientOrTransaction }) { + const prisma = tx ?? this.prisma; + const deleted = await prisma.taskRunWaitpoint.deleteMany({ + where: { + taskRunId: runId, + }, + }); + + return deleted.count; + } + + //#region TaskRunExecutionSnapshots + async #createExecutionSnapshot( + prisma: PrismaClientOrTransaction, + { + run, + snapshot, + checkpointId, + completedWaitpointIds, + error, + }: { + run: { id: string; status: TaskRunStatus; attemptNumber?: number | null }; + snapshot: { + executionStatus: TaskRunExecutionStatus; + description: string; + }; + checkpointId?: string; + completedWaitpointIds?: string[]; + error?: string; + } + ) { + const newSnapshot = await prisma.taskRunExecutionSnapshot.create({ + data: { + engine: "V2", + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runId: run.id, + runStatus: run.status, + attemptNumber: run.attemptNumber ?? undefined, + checkpointId: checkpointId ?? undefined, + completedWaitpoints: { + connect: completedWaitpointIds?.map((id) => ({ id })), + }, + isValid: error ? false : true, + error: error ?? undefined, + }, + include: { + checkpoint: true, + }, + }); + + if (!error) { + //set heartbeat (if relevant) + await this.#setExecutionSnapshotHeartbeat({ + status: newSnapshot.executionStatus, + runId: run.id, + snapshotId: newSnapshot.id, + }); + } + + this.eventBus.emit("executionSnapshotCreated", { + time: newSnapshot.createdAt, + run: { + id: newSnapshot.runId, + }, + snapshot: { + ...newSnapshot, + completedWaitpointIds: completedWaitpointIds ?? [], + }, + }); + + return { + ...newSnapshot, + friendlyId: SnapshotId.toFriendlyId(newSnapshot.id), + runFriendlyId: RunId.toFriendlyId(newSnapshot.runId), + }; + } + + async #setExecutionSnapshotHeartbeat({ + status, + runId, + snapshotId, + }: { + status: TaskRunExecutionStatus; + runId: string; + snapshotId: string; + }) { + await this.#setHeartbeatDeadline({ + runId, + snapshotId, + status, + }); + } + + #getHeartbeatIntervalMs(status: TaskRunExecutionStatus): number | null { + switch (status) { + case "PENDING_EXECUTING": { + return this.heartbeatTimeouts.PENDING_EXECUTING; + } + case "PENDING_CANCEL": { + return this.heartbeatTimeouts.PENDING_CANCEL; + } + case "EXECUTING": { + return this.heartbeatTimeouts.EXECUTING; + } + case "EXECUTING_WITH_WAITPOINTS": { + return this.heartbeatTimeouts.EXECUTING_WITH_WAITPOINTS; + } + default: { + return null; + } + } + } + + //#endregion + + //#region Heartbeat + async #setHeartbeatDeadline({ + runId, + snapshotId, + status, + }: { + runId: string; + snapshotId: string; + status: TaskRunExecutionStatus; + }) { + const intervalMs = this.#getHeartbeatIntervalMs(status); + + if (intervalMs === null) { + return; + } + + await this.worker.enqueue({ + id: `heartbeatSnapshot.${snapshotId}`, + job: "heartbeatSnapshot", + payload: { snapshotId, runId }, + availableAt: new Date(Date.now() + intervalMs), + }); + } + + async #handleStalledSnapshot({ + runId, + snapshotId, + tx, + }: { + runId: string; + snapshotId: string; + tx?: PrismaClientOrTransaction; + }) { + const prisma = tx ?? this.prisma; + return await this.runLock.lock([runId], 5_000, async (signal) => { + const latestSnapshot = await getLatestExecutionSnapshot(prisma, runId); + if (latestSnapshot.id !== snapshotId) { + this.logger.log( + "RunEngine.#handleStalledSnapshot() no longer the latest snapshot, stopping the heartbeat.", + { + runId, + snapshotId, + latestSnapshot: latestSnapshot, + } + ); + + await this.worker.ack(`heartbeatSnapshot.${snapshotId}`); + return; + } + + this.logger.log("RunEngine.#handleStalledSnapshot() handling stalled snapshot", { + runId, + snapshot: latestSnapshot, + }); + + switch (latestSnapshot.executionStatus) { + case "RUN_CREATED": { + throw new NotImplementedError("There shouldn't be a heartbeat for RUN_CREATED"); + } + case "QUEUED": { + throw new NotImplementedError("There shouldn't be a heartbeat for QUEUED"); + } + case "PENDING_EXECUTING": { + //the run didn't start executing, we need to requeue it + const run = await prisma.taskRun.findFirst({ + where: { id: runId }, + include: { + runtimeEnvironment: { + include: { + organization: true, + }, + }, + }, + }); + + if (!run) { + this.logger.error( + "RunEngine.#handleStalledSnapshot() PENDING_EXECUTING run not found", + { + runId, + snapshot: latestSnapshot, + } + ); + + throw new Error(`Run ${runId} not found`); + } + + //it will automatically be requeued X times depending on the queue retry settings + const gotRequeued = await this.#tryNackAndRequeue({ + run, + orgId: run.runtimeEnvironment.organizationId, + error: { + type: "INTERNAL_ERROR", + code: "TASK_RUN_DEQUEUED_MAX_RETRIES", + message: `Trying to create an attempt failed multiple times, exceeding how many times we retry.`, + }, + tx: prisma, + }); + break; + } + case "EXECUTING": + case "EXECUTING_WITH_WAITPOINTS": { + const retryDelay = 250; + + //todo call attemptFailed and force requeuing + await this.#attemptFailed({ + runId, + snapshotId: latestSnapshot.id, + completion: { + ok: false, + id: runId, + error: { + type: "INTERNAL_ERROR", + code: + latestSnapshot.executionStatus === "EXECUTING" + ? "TASK_RUN_STALLED_EXECUTING" + : "TASK_RUN_STALLED_EXECUTING_WITH_WAITPOINTS", + message: `Trying to create an attempt failed multiple times, exceeding how many times we retry.`, + }, + retry: { + //250ms in the future + timestamp: Date.now() + retryDelay, + delay: retryDelay, + }, + }, + forceRequeue: true, + tx: prisma, + }); + break; + } + case "BLOCKED_BY_WAITPOINTS": { + //todo should we do a periodic check here for whether waitpoints are actually still blocking? + //we could at least log some things out if a run has been in this state for a long time + throw new NotImplementedError("Not implemented BLOCKED_BY_WAITPOINTS"); + } + case "PENDING_CANCEL": { + //if the run is waiting to cancel but the worker hasn't confirmed that, + //we force the run to be cancelled + await this.cancelRun({ + runId: latestSnapshot.runId, + finalizeRun: true, + tx, + }); + break; + } + case "FINISHED": { + throw new NotImplementedError("There shouldn't be a heartbeat for FINISHED"); + } + default: { + assertNever(latestSnapshot.executionStatus); + } + } + }); + } + + //#endregion + + /** + * Sends a notification that a run has changed and we need to fetch the latest run state. + * The worker will call `getRunExecutionData` via the API and act accordingly. + */ + async #sendNotificationToWorker({ runId }: { runId: string }) { + this.eventBus.emit("workerNotification", { time: new Date(), run: { id: runId } }); + } + + /* + * Whether the run succeeds, fails, is cancelled… we need to run these operations + */ + async #finalizeRun({ id, batchId }: { id: string; batchId: string | null }) { + if (batchId) { + await this.worker.enqueue({ + //this will debounce the call + id: `tryCompleteBatch:${batchId}`, + job: "tryCompleteBatch", + payload: { batchId: batchId }, + //2s in the future + availableAt: new Date(Date.now() + 2_000), + }); + } + } + + /** + * Checks to see if all runs for a BatchTaskRun are completed, if they are then update the status. + * This isn't used operationally, but it's used for the Batches dashboard page. + */ + async #tryCompleteBatch({ batchId }: { batchId: string }) { + return this.#trace( + "#tryCompleteBatch", + { + batchId, + }, + async (span) => { + const batch = await this.prisma.batchTaskRun.findUnique({ + select: { + status: true, + runtimeEnvironmentId: true, + }, + where: { + id: batchId, + }, + }); + + if (!batch) { + this.logger.error("#tryCompleteBatch batch doesn't exist", { batchId }); + return; + } + + if (batch.status === "COMPLETED") { + this.logger.debug("#tryCompleteBatch: Batch already completed", { batchId }); + return; + } + + const runs = await this.prisma.taskRun.findMany({ + select: { + id: true, + status: true, + }, + where: { + batchId, + runtimeEnvironmentId: batch.runtimeEnvironmentId, + }, + }); + + if (runs.every((r) => isFinalRunStatus(r.status))) { + this.logger.debug("#tryCompleteBatch: All runs are completed", { batchId }); + await this.prisma.batchTaskRun.update({ + where: { + id: batchId, + }, + data: { + status: "COMPLETED", + }, + }); + } else { + this.logger.debug("#tryCompleteBatch: Not all runs are completed", { batchId }); + } + } + ); + } + + async #getAuthenticatedEnvironmentFromRun(runId: string, tx?: PrismaClientOrTransaction) { + const prisma = tx ?? this.prisma; + const taskRun = await prisma.taskRun.findUnique({ + where: { + id: runId, + }, + include: { + runtimeEnvironment: { + include: { + organization: true, + project: true, + }, + }, + }, + }); + + if (!taskRun) { + return; + } + + return taskRun?.runtimeEnvironment; + } + + #environmentMasterQueueKey(environmentId: string) { + return `master-env:${environmentId}`; + } + + #backgroundWorkerQueueKey(backgroundWorkerId: string) { + return `master-background-worker:${backgroundWorkerId}`; + } + + async #trace( + trace: string, + attributes: Attributes | undefined, + fn: (span: Span) => Promise + ): Promise { + return this.tracer.startActiveSpan( + `${this.constructor.name}.${trace}`, + { attributes, kind: SpanKind.SERVER }, + async (span) => { + try { + return await fn(span); + } catch (e) { + if (e instanceof ServiceValidationError) { + throw e; + } + + if (e instanceof Error) { + span.recordException(e); + } else { + span.recordException(new Error(String(e))); + } + + throw e; + } finally { + span.end(); + } + } + ); + } +} + +export class ServiceValidationError extends Error { + constructor( + message: string, + public status?: number + ) { + super(message); + this.name = "ServiceValidationError"; + } +} + +//todo temporary during development +class NotImplementedError extends Error { + constructor(message: string) { + console.error("NOT IMPLEMENTED YET", { message }); + super(message); + } +} diff --git a/internal-packages/run-engine/src/engine/locking.test.ts b/internal-packages/run-engine/src/engine/locking.test.ts new file mode 100644 index 0000000000..b09e870e27 --- /dev/null +++ b/internal-packages/run-engine/src/engine/locking.test.ts @@ -0,0 +1,37 @@ +import { redisTest } from "@internal/testcontainers"; +import { expect } from "vitest"; +import { RunLocker } from "./locking.js"; + +describe("RunLocker", () => { + redisTest("Test acquiring a lock works", { timeout: 15_000 }, async ({ redis }) => { + const runLock = new RunLocker({ redis }); + + expect(runLock.isInsideLock()).toBe(false); + + await runLock.lock(["test-1"], 5000, async (signal) => { + expect(signal).toBeDefined(); + expect(runLock.isInsideLock()).toBe(true); + }); + + expect(runLock.isInsideLock()).toBe(false); + }); + + redisTest("Test double locking works", { timeout: 15_000 }, async ({ redis }) => { + const runLock = new RunLocker({ redis }); + + expect(runLock.isInsideLock()).toBe(false); + + await runLock.lock(["test-1"], 5000, async (signal) => { + expect(signal).toBeDefined(); + expect(runLock.isInsideLock()).toBe(true); + + //should be able to "lock it again" + await runLock.lock(["test-1"], 5000, async (signal) => { + expect(signal).toBeDefined(); + expect(runLock.isInsideLock()).toBe(true); + }); + }); + + expect(runLock.isInsideLock()).toBe(false); + }); +}); diff --git a/internal-packages/run-engine/src/engine/locking.ts b/internal-packages/run-engine/src/engine/locking.ts new file mode 100644 index 0000000000..cd3aecc7c6 --- /dev/null +++ b/internal-packages/run-engine/src/engine/locking.ts @@ -0,0 +1,60 @@ +import Redis from "ioredis"; +import Redlock, { RedlockAbortSignal } from "redlock"; +import { AsyncLocalStorage } from "async_hooks"; + +interface LockContext { + resources: string; + signal: RedlockAbortSignal; +} + +export class RunLocker { + private redlock: Redlock; + private asyncLocalStorage: AsyncLocalStorage; + + constructor(options: { redis: Redis }) { + this.redlock = new Redlock([options.redis], { + driftFactor: 0.01, + retryCount: 10, + retryDelay: 200, // time in ms + retryJitter: 200, // time in ms + automaticExtensionThreshold: 500, // time in ms + }); + this.asyncLocalStorage = new AsyncLocalStorage(); + } + + /** Locks resources using RedLock. It won't lock again if we're already inside a lock with the same resources. */ + async lock( + resources: string[], + duration: number, + routine: (signal: RedlockAbortSignal) => Promise + ): Promise { + const currentContext = this.asyncLocalStorage.getStore(); + const joinedResources = resources.sort().join(","); + + if (currentContext && currentContext.resources === joinedResources) { + // We're already inside a lock with the same resources, just run the routine + return routine(currentContext.signal); + } + + // Different resources or not in a lock, proceed with new lock + return this.redlock.using(resources, duration, async (signal) => { + const newContext: LockContext = { resources: joinedResources, signal }; + + return this.asyncLocalStorage.run(newContext, async () => { + return routine(signal); + }); + }); + } + + isInsideLock(): boolean { + return !!this.asyncLocalStorage.getStore(); + } + + getCurrentResources(): string | undefined { + return this.asyncLocalStorage.getStore()?.resources; + } + + async quit() { + await this.redlock.quit(); + } +} diff --git a/internal-packages/run-engine/src/engine/machinePresets.ts b/internal-packages/run-engine/src/engine/machinePresets.ts new file mode 100644 index 0000000000..7e794fdcf1 --- /dev/null +++ b/internal-packages/run-engine/src/engine/machinePresets.ts @@ -0,0 +1,63 @@ +import { MachineConfig, MachinePreset, MachinePresetName } from "@trigger.dev/core/v3"; +import { Logger } from "@trigger.dev/core/logger"; + +const logger = new Logger("machinePresetFromConfig"); + +export function machinePresetFromConfig({ + defaultMachine, + machines, + config, +}: { + defaultMachine: MachinePresetName; + machines: Record; + config: unknown; +}): MachinePreset { + const parsedConfig = MachineConfig.safeParse(config); + + if (!parsedConfig.success) { + logger.error("Failed to parse machine config", { config }); + + return machinePresetFromName(machines, "small-1x"); + } + + if (parsedConfig.data.preset) { + return machinePresetFromName(machines, parsedConfig.data.preset); + } + + if (parsedConfig.data.cpu && parsedConfig.data.memory) { + const name = derivePresetNameFromValues( + machines, + parsedConfig.data.cpu, + parsedConfig.data.memory + ); + if (!name) { + return machinePresetFromName(machines, defaultMachine); + } + + return machinePresetFromName(machines, name); + } + + return machinePresetFromName(machines, "small-1x"); +} + +export function machinePresetFromName( + machines: Record, + name: MachinePresetName +): MachinePreset { + return { + ...machines[name], + }; +} + +// Finds the smallest machine preset name that satisfies the given CPU and memory requirements +function derivePresetNameFromValues( + machines: Record, + cpu: number, + memory: number +): MachinePresetName | undefined { + for (const [name, preset] of Object.entries(machines)) { + if (preset.cpu >= cpu && preset.memory >= memory) { + return name as MachinePresetName; + } + } +} diff --git a/internal-packages/run-engine/src/engine/statuses.ts b/internal-packages/run-engine/src/engine/statuses.ts new file mode 100644 index 0000000000..3ed80df993 --- /dev/null +++ b/internal-packages/run-engine/src/engine/statuses.ts @@ -0,0 +1,41 @@ +import { TaskRunExecutionStatus, TaskRunStatus } from "@trigger.dev/database"; + +export function isDequeueableExecutionStatus(status: TaskRunExecutionStatus): boolean { + const dequeuableExecutionStatuses: TaskRunExecutionStatus[] = ["QUEUED"]; + return dequeuableExecutionStatuses.includes(status); +} + +export function isExecuting(status: TaskRunExecutionStatus): boolean { + const executingExecutionStatuses: TaskRunExecutionStatus[] = [ + "EXECUTING", + "EXECUTING_WITH_WAITPOINTS", + ]; + return executingExecutionStatuses.includes(status); +} + +export function isCheckpointable(status: TaskRunExecutionStatus): boolean { + const checkpointableStatuses: TaskRunExecutionStatus[] = [ + //will allow checkpoint starts + "RUN_CREATED", + "QUEUED", + //executing + "EXECUTING", + "EXECUTING_WITH_WAITPOINTS", + ]; + return checkpointableStatuses.includes(status); +} + +export function isFinalRunStatus(status: TaskRunStatus): boolean { + const finalStatuses: TaskRunStatus[] = [ + "CANCELED", + "INTERRUPTED", + "COMPLETED_SUCCESSFULLY", + "COMPLETED_WITH_ERRORS", + "SYSTEM_FAILURE", + "CRASHED", + "EXPIRED", + "TIMED_OUT", + ]; + + return finalStatuses.includes(status); +} diff --git a/internal-packages/run-engine/src/engine/tests/batchTrigger.test.ts b/internal-packages/run-engine/src/engine/tests/batchTrigger.test.ts new file mode 100644 index 0000000000..f1025d4d0d --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/batchTrigger.test.ts @@ -0,0 +1,181 @@ +import { + containerTest, + setupAuthenticatedEnvironment, + setupBackgroundWorker, +} from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { generateFriendlyId } from "@trigger.dev/core/v3/apps"; +import { expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { setTimeout } from "node:timers/promises"; + +describe("RunEngine batchTrigger", () => { + containerTest( + "Batch trigger shares a batch", + { timeout: 15_000 }, + async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0005, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + const backgroundWorker = await setupBackgroundWorker( + prisma, + authenticatedEnvironment, + taskIdentifier + ); + + const batch = await prisma.batchTaskRun.create({ + data: { + friendlyId: generateFriendlyId("batch"), + runtimeEnvironmentId: authenticatedEnvironment.id, + }, + }); + + //trigger the runs + const run1 = await engine.trigger( + { + number: 1, + friendlyId: "run_1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + batchId: batch.id, + }, + prisma + ); + + const run2 = await engine.trigger( + { + number: 2, + friendlyId: "run_1235", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + batchId: batch.id, + }, + prisma + ); + + expect(run1).toBeDefined(); + expect(run1.friendlyId).toBe("run_1234"); + expect(run1.batchId).toBe(batch.id); + + expect(run2).toBeDefined(); + expect(run2.friendlyId).toBe("run_1235"); + expect(run2.batchId).toBe(batch.id); + + //check the queue length + const queueLength = await engine.runQueue.lengthOfEnvQueue(authenticatedEnvironment); + expect(queueLength).toBe(2); + + //dequeue + const [d1, d2] = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: run1.masterQueue, + maxRunCount: 10, + }); + + //attempts + const attempt1 = await engine.startRunAttempt({ + runId: d1.run.id, + snapshotId: d1.snapshot.id, + }); + const attempt2 = await engine.startRunAttempt({ + runId: d2.run.id, + snapshotId: d2.snapshot.id, + }); + + //complete the runs + const result1 = await engine.completeRunAttempt({ + runId: attempt1.run.id, + snapshotId: attempt1.snapshot.id, + completion: { + ok: true, + id: attempt1.run.id, + output: `{"foo":"bar"}`, + outputType: "application/json", + }, + }); + const result2 = await engine.completeRunAttempt({ + runId: attempt2.run.id, + snapshotId: attempt2.snapshot.id, + completion: { + ok: true, + id: attempt2.run.id, + output: `{"baz":"qux"}`, + outputType: "application/json", + }, + }); + + //the batch won't complete immediately + const batchAfter1 = await prisma.batchTaskRun.findUnique({ + where: { + id: batch.id, + }, + }); + expect(batchAfter1?.status).toBe("PENDING"); + + await setTimeout(3_000); + + //the batch should complete + const batchAfter2 = await prisma.batchTaskRun.findUnique({ + where: { + id: batch.id, + }, + }); + expect(batchAfter2?.status).toBe("COMPLETED"); + } finally { + engine.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/tests/cancelling.test.ts b/internal-packages/run-engine/src/engine/tests/cancelling.test.ts new file mode 100644 index 0000000000..8567c874f2 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/cancelling.test.ts @@ -0,0 +1,334 @@ +import { + containerTest, + setupAuthenticatedEnvironment, + setupBackgroundWorker, + assertNonNullable, +} from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { setTimeout } from "timers/promises"; +import { EventBusEventArgs } from "../eventBus.js"; + +describe("RunEngine cancelling", () => { + containerTest( + "Cancelling a run with children (that is executing)", + { timeout: 15_000 }, + async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const parentTask = "parent-task"; + const childTask = "child-task"; + + //create background worker + await setupBackgroundWorker(prisma, authenticatedEnvironment, [parentTask, childTask]); + + //trigger the run + const parentRun = await engine.trigger( + { + number: 1, + friendlyId: "run_p1234", + environment: authenticatedEnvironment, + taskIdentifier: parentTask, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: `task/${parentTask}`, + isTest: false, + tags: [], + }, + prisma + ); + + //dequeue the run + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: parentRun.masterQueue, + maxRunCount: 10, + }); + + //create an attempt + const attemptResult = await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + expect(attemptResult.snapshot.executionStatus).toBe("EXECUTING"); + + //start child run + const childRun = await engine.trigger( + { + number: 1, + friendlyId: "run_c1234", + environment: authenticatedEnvironment, + taskIdentifier: childTask, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: `task/${childTask}`, + isTest: false, + tags: [], + resumeParentOnCompletion: true, + parentTaskRunId: parentRun.id, + }, + prisma + ); + + //dequeue the child run + const dequeuedChild = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: childRun.masterQueue, + maxRunCount: 10, + }); + + //start the child run + const childAttempt = await engine.startRunAttempt({ + runId: childRun.id, + snapshotId: dequeuedChild[0].snapshot.id, + }); + + let workerNotifications: EventBusEventArgs<"workerNotification">[0][] = []; + engine.eventBus.on("workerNotification", (result) => { + workerNotifications.push(result); + }); + + //cancel the parent run + const result = await engine.cancelRun({ + runId: parentRun.id, + completedAt: new Date(), + reason: "Cancelled by the user", + }); + expect(result.snapshot.executionStatus).toBe("PENDING_CANCEL"); + + //check a worker notification was sent for the running parent + expect(workerNotifications).toHaveLength(1); + expect(workerNotifications[0].run.id).toBe(parentRun.id); + + const executionData = await engine.getRunExecutionData({ runId: parentRun.id }); + expect(executionData?.snapshot.executionStatus).toBe("PENDING_CANCEL"); + expect(executionData?.run.status).toBe("CANCELED"); + + let cancelledEventData: EventBusEventArgs<"runCancelled">[0][] = []; + engine.eventBus.on("runCancelled", (result) => { + cancelledEventData.push(result); + }); + + //todo call completeAttempt (this will happen from the worker) + const completeResult = await engine.completeRunAttempt({ + runId: parentRun.id, + snapshotId: executionData!.snapshot.id, + completion: { + ok: false, + id: executionData!.run.id, + error: { + type: "INTERNAL_ERROR" as const, + code: "TASK_RUN_CANCELLED" as const, + }, + }, + }); + + //parent should now be fully cancelled + const executionDataAfter = await engine.getRunExecutionData({ runId: parentRun.id }); + expect(executionDataAfter?.snapshot.executionStatus).toBe("FINISHED"); + expect(executionDataAfter?.run.status).toBe("CANCELED"); + + //check emitted event + expect(cancelledEventData.length).toBe(1); + const parentEvent = cancelledEventData.find((r) => r.run.id === parentRun.id); + assertNonNullable(parentEvent); + expect(parentEvent.run.spanId).toBe(parentRun.spanId); + + //cancelling children is async, so we need to wait a brief moment + await setTimeout(200); + + //check a worker notification was sent for the running parent + expect(workerNotifications).toHaveLength(2); + expect(workerNotifications[1].run.id).toBe(childRun.id); + + //child should now be pending cancel + const childExecutionDataAfter = await engine.getRunExecutionData({ runId: childRun.id }); + expect(childExecutionDataAfter?.snapshot.executionStatus).toBe("PENDING_CANCEL"); + expect(childExecutionDataAfter?.run.status).toBe("CANCELED"); + + //cancel the child (this will come from the worker) + const completeChildResult = await engine.completeRunAttempt({ + runId: childRun.id, + snapshotId: childExecutionDataAfter!.snapshot.id, + completion: { + ok: false, + id: childRun.id, + error: { + type: "INTERNAL_ERROR" as const, + code: "TASK_RUN_CANCELLED" as const, + }, + }, + }); + expect(completeChildResult.snapshot.executionStatus).toBe("FINISHED"); + expect(completeChildResult.run.status).toBe("CANCELED"); + + //child should now be pending cancel + const childExecutionDataCancelled = await engine.getRunExecutionData({ + runId: childRun.id, + }); + expect(childExecutionDataCancelled?.snapshot.executionStatus).toBe("FINISHED"); + expect(childExecutionDataCancelled?.run.status).toBe("CANCELED"); + + //check emitted event + expect(cancelledEventData.length).toBe(2); + const childEvent = cancelledEventData.find((r) => r.run.id === childRun.id); + assertNonNullable(childEvent); + expect(childEvent.run.spanId).toBe(childRun.spanId); + + //concurrency should have been released + const envConcurrencyCompleted = await engine.runQueue.currentConcurrencyOfEnvironment( + authenticatedEnvironment + ); + expect(envConcurrencyCompleted).toBe(0); + } finally { + engine.quit(); + } + } + ); + + containerTest( + "Cancelling a run (not executing)", + { timeout: 15_000 }, + async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const parentTask = "parent-task"; + + //create background worker + await setupBackgroundWorker(prisma, authenticatedEnvironment, [parentTask]); + + //trigger the run + const parentRun = await engine.trigger( + { + number: 1, + friendlyId: "run_p1234", + environment: authenticatedEnvironment, + taskIdentifier: parentTask, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: `task/${parentTask}`, + isTest: false, + tags: [], + }, + prisma + ); + + //dequeue the run + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: parentRun.masterQueue, + maxRunCount: 10, + }); + + let cancelledEventData: EventBusEventArgs<"runCancelled">[0][] = []; + engine.eventBus.on("runCancelled", (result) => { + cancelledEventData.push(result); + }); + + //cancel the parent run + const result = await engine.cancelRun({ + runId: parentRun.id, + completedAt: new Date(), + reason: "Cancelled by the user", + }); + expect(result.snapshot.executionStatus).toBe("FINISHED"); + + const executionData = await engine.getRunExecutionData({ runId: parentRun.id }); + expect(executionData?.snapshot.executionStatus).toBe("FINISHED"); + expect(executionData?.run.status).toBe("CANCELED"); + + //check emitted event + expect(cancelledEventData.length).toBe(1); + const parentEvent = cancelledEventData.find((r) => r.run.id === parentRun.id); + assertNonNullable(parentEvent); + expect(parentEvent.run.spanId).toBe(parentRun.spanId); + + //concurrency should have been released + const envConcurrencyCompleted = await engine.runQueue.currentConcurrencyOfEnvironment( + authenticatedEnvironment + ); + expect(envConcurrencyCompleted).toBe(0); + } finally { + engine.quit(); + } + } + ); + + //todo bulk cancelling runs +}); diff --git a/internal-packages/run-engine/src/engine/tests/checkpoints.test.ts b/internal-packages/run-engine/src/engine/tests/checkpoints.test.ts new file mode 100644 index 0000000000..a62747ca0c --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/checkpoints.test.ts @@ -0,0 +1,17 @@ +//todo checkpoint tests +import { + containerTest, + setupAuthenticatedEnvironment, + setupBackgroundWorker, + assertNonNullable, +} from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { setTimeout } from "timers/promises"; +import { EventBusEventArgs } from "../eventBus.js"; + +describe("RunEngine checkpoints", () => { + //todo checkpoint tests + test("empty test", async () => {}); +}); diff --git a/internal-packages/run-engine/src/engine/tests/delays.test.ts b/internal-packages/run-engine/src/engine/tests/delays.test.ts new file mode 100644 index 0000000000..dcfbcb470c --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/delays.test.ts @@ -0,0 +1,190 @@ +import { + containerTest, + setupAuthenticatedEnvironment, + setupBackgroundWorker, + assertNonNullable, +} from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { setTimeout } from "timers/promises"; + +describe("RunEngine delays", () => { + containerTest("Run start delayed", { timeout: 15_000 }, async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + const backgroundWorker = await setupBackgroundWorker( + prisma, + authenticatedEnvironment, + taskIdentifier + ); + + //trigger the run + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + delayUntil: new Date(Date.now() + 500), + }, + prisma + ); + + //should be created but not queued yet + const executionData = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData); + expect(executionData.snapshot.executionStatus).toBe("RUN_CREATED"); + + //wait for 1 seconds + await setTimeout(1_000); + + //should now be queued + const executionData2 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData2); + expect(executionData2.snapshot.executionStatus).toBe("QUEUED"); + } finally { + engine.quit(); + } + }); + + containerTest( + "Rescheduling a delayed run", + { timeout: 15_000 }, + async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + const backgroundWorker = await setupBackgroundWorker( + prisma, + authenticatedEnvironment, + taskIdentifier + ); + + //trigger the run + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + delayUntil: new Date(Date.now() + 200), + }, + prisma + ); + + //should be created but not queued yet + const executionData = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData); + expect(executionData.snapshot.executionStatus).toBe("RUN_CREATED"); + + const rescheduleTo = new Date(Date.now() + 1_500); + const updatedRun = await engine.rescheduleRun({ runId: run.id, delayUntil: rescheduleTo }); + expect(updatedRun.delayUntil?.toISOString()).toBe(rescheduleTo.toISOString()); + + //wait so the initial delay passes + await setTimeout(1_000); + + //should still be created + const executionData2 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData2); + expect(executionData2.snapshot.executionStatus).toBe("RUN_CREATED"); + + //wait so the updated delay passes + await setTimeout(1_750); + + //should now be queued + const executionData3 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData3); + expect(executionData3.snapshot.executionStatus).toBe("QUEUED"); + } finally { + engine.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/tests/dequeuing.test.ts b/internal-packages/run-engine/src/engine/tests/dequeuing.test.ts new file mode 100644 index 0000000000..13e91ef9f2 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/dequeuing.test.ts @@ -0,0 +1,209 @@ +import { + containerTest, + setupAuthenticatedEnvironment, + setupBackgroundWorker, +} from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { generateFriendlyId } from "@trigger.dev/core/v3/apps"; +import { expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { setTimeout } from "node:timers/promises"; +import { MinimalAuthenticatedEnvironment } from "../../shared/index.js"; +import { PrismaClientOrTransaction } from "@trigger.dev/database"; + +describe("RunEngine dequeuing", () => { + containerTest("Dequeues 5 runs", { timeout: 15_000 }, async ({ prisma, redisContainer }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0005, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + await setupBackgroundWorker(prisma, authenticatedEnvironment, taskIdentifier); + + //trigger the runs + const runs = await triggerRuns({ + engine, + environment: authenticatedEnvironment, + taskIdentifier, + prisma, + count: 10, + }); + expect(runs.length).toBe(10); + + //check the queue length + const queueLength = await engine.runQueue.lengthOfEnvQueue(authenticatedEnvironment); + expect(queueLength).toBe(10); + + //dequeue + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: "main", + maxRunCount: 5, + }); + + expect(dequeued.length).toBe(5); + } finally { + engine.quit(); + } + }); + + containerTest( + "Dequeues runs within machine constraints", + { timeout: 15_000 }, + async ({ prisma, redisContainer }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0005, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + await setupBackgroundWorker(prisma, authenticatedEnvironment, taskIdentifier, { + preset: "small-1x", + }); + + //trigger the runs + const runs = await triggerRuns({ + engine, + environment: authenticatedEnvironment, + taskIdentifier, + prisma, + count: 20, + }); + expect(runs.length).toBe(20); + + //check the queue length + const queueLength = await engine.runQueue.lengthOfEnvQueue(authenticatedEnvironment); + expect(queueLength).toBe(20); + + //dequeue + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: "main", + maxRunCount: 5, + maxResources: { + cpu: 1.1, + memory: 3.8, + }, + }); + expect(dequeued.length).toBe(2); + + //check the queue length + const queueLength2 = await engine.runQueue.lengthOfEnvQueue(authenticatedEnvironment); + expect(queueLength2).toBe(18); + + const dequeued2 = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: "main", + maxRunCount: 10, + maxResources: { + cpu: 4.7, + memory: 3.0, + }, + }); + expect(dequeued2.length).toBe(6); + + //check the queue length + const queueLength3 = await engine.runQueue.lengthOfEnvQueue(authenticatedEnvironment); + expect(queueLength3).toBe(12); + } finally { + engine.quit(); + } + } + ); +}); + +async function triggerRuns({ + engine, + environment, + taskIdentifier, + prisma, + count, +}: { + engine: RunEngine; + environment: MinimalAuthenticatedEnvironment; + taskIdentifier: string; + prisma: PrismaClientOrTransaction; + count: number; +}) { + const runs = []; + for (let i = 0; i < count; i++) { + runs[i] = await engine.trigger( + { + number: i, + friendlyId: generateFriendlyId("run"), + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: `task/${taskIdentifier}`, + isTest: false, + tags: [], + }, + prisma + ); + } + + return runs; +} diff --git a/internal-packages/run-engine/src/engine/tests/heartbeats.test.ts b/internal-packages/run-engine/src/engine/tests/heartbeats.test.ts new file mode 100644 index 0000000000..176da6025f --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/heartbeats.test.ts @@ -0,0 +1,494 @@ +import { + containerTest, + setupAuthenticatedEnvironment, + setupBackgroundWorker, + assertNonNullable, +} from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { setTimeout } from "timers/promises"; + +describe("RunEngine heartbeats", () => { + containerTest( + "Attempt timeout then successfully attempted", + { timeout: 15_000 }, + async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const pendingExecutingTimeout = 100; + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + heartbeatTimeoutsMs: { + PENDING_EXECUTING: pendingExecutingTimeout, + }, + queue: { + retryOptions: { + maxTimeoutInMs: 50, + }, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + const backgroundWorker = await setupBackgroundWorker( + prisma, + authenticatedEnvironment, + taskIdentifier + ); + + //trigger the run + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + //dequeue the run + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: run.masterQueue, + maxRunCount: 10, + }); + + //expect it to be pending with 0 consecutiveFailures + const executionData = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData); + expect(executionData.snapshot.executionStatus).toBe("PENDING_EXECUTING"); + + await setTimeout(pendingExecutingTimeout * 2); + + //expect it to be pending with 3 consecutiveFailures + const executionData2 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData2); + expect(executionData2.snapshot.executionStatus).toBe("QUEUED"); + + await setTimeout(1_000); + + //have to dequeue again + const dequeued2 = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: run.masterQueue, + maxRunCount: 10, + }); + expect(dequeued2.length).toBe(1); + + // create an attempt + const attemptResult = await engine.startRunAttempt({ + runId: dequeued2[0].run.id, + snapshotId: dequeued2[0].snapshot.id, + }); + expect(attemptResult.run.id).toBe(run.id); + expect(attemptResult.run.status).toBe("EXECUTING"); + expect(attemptResult.snapshot.executionStatus).toBe("EXECUTING"); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "All start attempts timeout", + { timeout: 15_000 }, + async ({ prisma, redisContainer }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const pendingExecutingTimeout = 100; + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + heartbeatTimeoutsMs: { + PENDING_EXECUTING: pendingExecutingTimeout, + }, + queue: { + retryOptions: { + //intentionally set the attempts to 2 and quick + maxAttempts: 2, + minTimeoutInMs: 50, + maxTimeoutInMs: 50, + }, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + const backgroundWorker = await setupBackgroundWorker( + prisma, + authenticatedEnvironment, + taskIdentifier + ); + + //trigger the run + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + //dequeue the run + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: run.masterQueue, + maxRunCount: 10, + }); + + //expect it to be pending + const executionData = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData); + expect(executionData.snapshot.executionStatus).toBe("PENDING_EXECUTING"); + + await setTimeout(500); + + //expect it to be pending with 3 consecutiveFailures + const executionData2 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData2); + expect(executionData2.snapshot.executionStatus).toBe("QUEUED"); + + //have to dequeue again + const dequeued2 = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: run.masterQueue, + maxRunCount: 10, + }); + expect(dequeued2.length).toBe(1); + + //expect it to be pending + const executionData3 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData3); + expect(executionData3.snapshot.executionStatus).toBe("PENDING_EXECUTING"); + + await setTimeout(pendingExecutingTimeout * 3); + + //expect it to be pending with 3 consecutiveFailures + const executionData4 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData4); + expect(executionData4.snapshot.executionStatus).toBe("FINISHED"); + expect(executionData4.run.status).toBe("SYSTEM_FAILURE"); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "Execution timeout (worker doesn't heartbeat)", + { timeout: 15_000 }, + async ({ prisma, redisContainer }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const executingTimeout = 100; + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + heartbeatTimeoutsMs: { + EXECUTING: executingTimeout, + }, + queue: { + retryOptions: { + //intentionally set the attempts to 2 and quick + maxAttempts: 2, + minTimeoutInMs: 50, + maxTimeoutInMs: 50, + }, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + const backgroundWorker = await setupBackgroundWorker( + prisma, + authenticatedEnvironment, + taskIdentifier + ); + + //trigger the run + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + //dequeue the run + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: run.masterQueue, + maxRunCount: 10, + }); + + //create an attempt + await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + + //should be executing + const executionData = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData); + expect(executionData.snapshot.executionStatus).toBe("EXECUTING"); + expect(executionData.run.status).toBe("EXECUTING"); + + //wait long enough for the heartbeat to timeout + await setTimeout(1_000); + + //expect it to be queued again + const executionData2 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData2); + expect(executionData2.snapshot.executionStatus).toBe("QUEUED"); + + //have to dequeue again + const dequeued2 = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: run.masterQueue, + maxRunCount: 10, + }); + expect(dequeued2.length).toBe(1); + + //create an attempt + await engine.startRunAttempt({ + runId: dequeued2[0].run.id, + snapshotId: dequeued2[0].snapshot.id, + }); + + //should be executing + const executionData3 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData3); + expect(executionData3.snapshot.executionStatus).toBe("EXECUTING"); + expect(executionData3.run.status).toBe("EXECUTING"); + + //again wait long enough that the heartbeat fails + await setTimeout(1_000); + + //expect it to be queued again + const executionData4 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData4); + expect(executionData4.snapshot.executionStatus).toBe("FINISHED"); + expect(executionData4.run.status).toBe("SYSTEM_FAILURE"); + } finally { + await engine.quit(); + } + } + ); + + containerTest("Pending cancel", { timeout: 15_000 }, async ({ prisma, redisContainer }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const heartbeatTimeout = 100; + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + heartbeatTimeoutsMs: { + PENDING_CANCEL: heartbeatTimeout, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + const backgroundWorker = await setupBackgroundWorker( + prisma, + authenticatedEnvironment, + taskIdentifier + ); + + //trigger the run + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + //dequeue the run + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: run.masterQueue, + maxRunCount: 10, + }); + + //create an attempt + await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + + //cancel run + await engine.cancelRun({ runId: dequeued[0].run.id }); + + //expect it to be pending_cancel + const executionData2 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData2); + expect(executionData2.snapshot.executionStatus).toBe("PENDING_CANCEL"); + + //wait long enough for the heartbeat to timeout + await setTimeout(1_000); + + //expect it to be queued again + const executionData3 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData3); + expect(executionData3.snapshot.executionStatus).toBe("FINISHED"); + expect(executionData3.run.status).toBe("CANCELED"); + } finally { + await engine.quit(); + } + }); +}); diff --git a/internal-packages/run-engine/src/engine/tests/notDeployed.test.ts b/internal-packages/run-engine/src/engine/tests/notDeployed.test.ts new file mode 100644 index 0000000000..b748475c51 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/notDeployed.test.ts @@ -0,0 +1,152 @@ +import { + containerTest, + setupAuthenticatedEnvironment, + setupBackgroundWorker, + assertNonNullable, +} from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { setTimeout } from "timers/promises"; + +describe("RunEngine not deployed", () => { + containerTest("Not yet deployed", { timeout: 15_000 }, async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + //set this so we have to requeue the runs in two batches + queueRunsWaitingForWorkerBatchSize: 1, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //trigger the run + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + //trigger another run + const run2 = await engine.trigger( + { + number: 2, + friendlyId: "run_1235", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12346", + spanId: "s12346", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + //should be queued + const executionDataR1 = await engine.getRunExecutionData({ runId: run.id }); + const executionDataR2 = await engine.getRunExecutionData({ runId: run2.id }); + assertNonNullable(executionDataR1); + assertNonNullable(executionDataR2); + expect(executionDataR1.snapshot.executionStatus).toBe("QUEUED"); + expect(executionDataR2.snapshot.executionStatus).toBe("QUEUED"); + + //dequeuing should fail + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: run.masterQueue, + maxRunCount: 10, + }); + expect(dequeued.length).toBe(0); + + //queue should be empty + const queueLength = await engine.runQueue.lengthOfQueue(authenticatedEnvironment, run.queue); + expect(queueLength).toBe(0); + + //check the execution data now + const executionData2R1 = await engine.getRunExecutionData({ runId: run.id }); + const executionData2R2 = await engine.getRunExecutionData({ runId: run2.id }); + assertNonNullable(executionData2R1); + assertNonNullable(executionData2R2); + expect(executionData2R1.snapshot.executionStatus).toBe("RUN_CREATED"); + expect(executionData2R2.snapshot.executionStatus).toBe("RUN_CREATED"); + expect(executionData2R1.run.status).toBe("WAITING_FOR_DEPLOY"); + expect(executionData2R2.run.status).toBe("WAITING_FOR_DEPLOY"); + + //create background worker + const backgroundWorker = await setupBackgroundWorker( + prisma, + authenticatedEnvironment, + taskIdentifier + ); + + //now we deploy the background worker + await engine.queueRunsWaitingForWorker({ backgroundWorkerId: backgroundWorker.worker.id }); + + //it's async so we wait + await setTimeout(500); + + //should now be queued + const executionData3R1 = await engine.getRunExecutionData({ runId: run.id }); + const executionData3R2 = await engine.getRunExecutionData({ runId: run2.id }); + assertNonNullable(executionData3R1); + assertNonNullable(executionData3R2); + expect(executionData3R1.snapshot.executionStatus).toBe("QUEUED"); + expect(executionData3R2.snapshot.executionStatus).toBe("QUEUED"); + expect(executionData3R1.run.status).toBe("PENDING"); + expect(executionData3R2.run.status).toBe("PENDING"); + + //queue should be empty + const queueLength2 = await engine.runQueue.lengthOfQueue(authenticatedEnvironment, run.queue); + expect(queueLength2).toBe(2); + } finally { + engine.quit(); + } + }); +}); diff --git a/internal-packages/run-engine/src/engine/tests/priority.test.ts b/internal-packages/run-engine/src/engine/tests/priority.test.ts new file mode 100644 index 0000000000..a0df381a71 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/priority.test.ts @@ -0,0 +1,144 @@ +import { + containerTest, + setupAuthenticatedEnvironment, + setupBackgroundWorker, +} from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { generateFriendlyId } from "@trigger.dev/core/v3/apps"; +import { expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { PrismaClientOrTransaction } from "@trigger.dev/database"; +import { MinimalAuthenticatedEnvironment } from "../../shared/index.js"; +import { setTimeout } from "timers/promises"; + +describe("RunEngine priority", () => { + containerTest( + "Two runs execute in the correct order", + { timeout: 15_000 }, + async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0005, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + const backgroundWorker = await setupBackgroundWorker( + prisma, + authenticatedEnvironment, + taskIdentifier + ); + + //the order should be 4,3,1,0,2 + // 0 1 2 3 4 + const priorities = [undefined, 500, -1200, 1000, 4000]; + + //trigger the runs + const runs = await triggerRuns({ + engine, + environment: authenticatedEnvironment, + taskIdentifier, + prisma, + priorities, + }); + expect(runs.length).toBe(priorities.length); + + //check the queue length + const queueLength = await engine.runQueue.lengthOfEnvQueue(authenticatedEnvironment); + expect(queueLength).toBe(priorities.length); + + //dequeue (expect 4 items because of the negative priority) + const dequeue = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: "main", + maxRunCount: 20, + }); + expect(dequeue.length).toBe(4); + expect(dequeue[0].run.friendlyId).toBe(runs[4].friendlyId); + expect(dequeue[1].run.friendlyId).toBe(runs[3].friendlyId); + expect(dequeue[2].run.friendlyId).toBe(runs[1].friendlyId); + expect(dequeue[3].run.friendlyId).toBe(runs[0].friendlyId); + + //wait 2 seconds (because of the negative priority) + await setTimeout(2_000); + const dequeue2 = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: "main", + maxRunCount: 20, + }); + expect(dequeue2.length).toBe(1); + expect(dequeue2[0].run.friendlyId).toBe(runs[2].friendlyId); + } finally { + engine.quit(); + } + } + ); +}); + +async function triggerRuns({ + engine, + environment, + taskIdentifier, + priorities, + prisma, +}: { + engine: RunEngine; + environment: MinimalAuthenticatedEnvironment; + taskIdentifier: string; + prisma: PrismaClientOrTransaction; + priorities: (number | undefined)[]; +}) { + const runs = []; + for (let i = 0; i < priorities.length; i++) { + runs[i] = await engine.trigger( + { + number: i, + friendlyId: generateFriendlyId("run"), + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: `task/${taskIdentifier}`, + isTest: false, + tags: [], + priorityMs: priorities[i], + }, + prisma + ); + } + + return runs; +} diff --git a/internal-packages/run-engine/src/engine/tests/trigger.test.ts b/internal-packages/run-engine/src/engine/tests/trigger.test.ts new file mode 100644 index 0000000000..84e5951c57 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/trigger.test.ts @@ -0,0 +1,487 @@ +import { + assertNonNullable, + containerTest, + setupAuthenticatedEnvironment, + setupBackgroundWorker, +} from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { expect } from "vitest"; +import { EventBusEventArgs } from "../eventBus.js"; +import { RunEngine } from "../index.js"; + +describe("RunEngine trigger()", () => { + containerTest("Single run (success)", { timeout: 15_000 }, async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0005, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + const backgroundWorker = await setupBackgroundWorker( + prisma, + authenticatedEnvironment, + taskIdentifier + ); + + //trigger the run + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + expect(run).toBeDefined(); + expect(run.friendlyId).toBe("run_1234"); + + //check it's actually in the db + const runFromDb = await prisma.taskRun.findUnique({ + where: { + friendlyId: "run_1234", + }, + }); + expect(runFromDb).toBeDefined(); + expect(runFromDb?.id).toBe(run.id); + + const executionData = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData); + expect(executionData.snapshot.executionStatus).toBe("QUEUED"); + + //check the waitpoint is created + const runWaitpoint = await prisma.waitpoint.findMany({ + where: { + completedByTaskRunId: run.id, + }, + }); + expect(runWaitpoint.length).toBe(1); + expect(runWaitpoint[0].type).toBe("RUN"); + + //check the queue length + const queueLength = await engine.runQueue.lengthOfQueue(authenticatedEnvironment, run.queue); + expect(queueLength).toBe(1); + + //concurrency before + const envConcurrencyBefore = await engine.runQueue.currentConcurrencyOfEnvironment( + authenticatedEnvironment + ); + expect(envConcurrencyBefore).toBe(0); + + //dequeue the run + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: run.masterQueue, + maxRunCount: 10, + }); + expect(dequeued.length).toBe(1); + expect(dequeued[0].run.id).toBe(run.id); + expect(dequeued[0].run.attemptNumber).toBe(1); + + const envConcurrencyAfter = await engine.runQueue.currentConcurrencyOfEnvironment( + authenticatedEnvironment + ); + expect(envConcurrencyAfter).toBe(1); + + let attemptEvent: EventBusEventArgs<"runAttemptStarted">[0] | undefined = undefined; + engine.eventBus.on("runAttemptStarted", (result) => { + attemptEvent = result; + }); + + //create an attempt + const attemptResult = await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + expect(attemptResult.run.id).toBe(run.id); + expect(attemptResult.run.status).toBe("EXECUTING"); + expect(attemptResult.snapshot.executionStatus).toBe("EXECUTING"); + + //attempt event + assertNonNullable(attemptEvent); + const attemptedEvent = attemptEvent as EventBusEventArgs<"runAttemptStarted">[0]; + expect(attemptedEvent.run.id).toBe(run.id); + expect(attemptedEvent.run.baseCostInCents).toBe(0.0005); + + const executionData2 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData2); + expect(executionData2.snapshot.executionStatus).toBe("EXECUTING"); + expect(executionData2.run.attemptNumber).toBe(1); + expect(executionData2.run.status).toBe("EXECUTING"); + + let successEvent: EventBusEventArgs<"runSucceeded">[0] | undefined = undefined; + engine.eventBus.on("runSucceeded", (result) => { + successEvent = result; + }); + + //complete the run + const result = await engine.completeRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: attemptResult.snapshot.id, + completion: { + ok: true, + id: dequeued[0].run.id, + output: `{"foo":"bar"}`, + outputType: "application/json", + }, + }); + expect(result.attemptStatus).toBe("RUN_FINISHED"); + expect(result.snapshot.executionStatus).toBe("FINISHED"); + expect(result.run.attemptNumber).toBe(1); + expect(result.run.status).toBe("COMPLETED_SUCCESSFULLY"); + + //state should be completed + const executionData3 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData3); + expect(executionData3.snapshot.executionStatus).toBe("FINISHED"); + expect(executionData3.run.attemptNumber).toBe(1); + expect(executionData3.run.status).toBe("COMPLETED_SUCCESSFULLY"); + + //success event + assertNonNullable(successEvent); + const completedEvent = successEvent as EventBusEventArgs<"runSucceeded">[0]; + expect(completedEvent.run.spanId).toBe(run.spanId); + expect(completedEvent.run.output).toBe('{"foo":"bar"}'); + expect(completedEvent.run.outputType).toBe("application/json"); + + //concurrency should have been released + const envConcurrencyCompleted = await engine.runQueue.currentConcurrencyOfEnvironment( + authenticatedEnvironment + ); + expect(envConcurrencyCompleted).toBe(0); + + //waitpoint should have been completed, with the output + const runWaitpointAfter = await prisma.waitpoint.findMany({ + where: { + completedByTaskRunId: run.id, + }, + }); + expect(runWaitpointAfter.length).toBe(1); + expect(runWaitpointAfter[0].type).toBe("RUN"); + expect(runWaitpointAfter[0].output).toBe(`{"foo":"bar"}`); + } finally { + engine.quit(); + } + }); + + containerTest("Single run (failed)", { timeout: 15_000 }, async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + const backgroundWorker = await setupBackgroundWorker( + prisma, + authenticatedEnvironment, + taskIdentifier + ); + + //trigger the run + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + //dequeue the run + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: run.masterQueue, + maxRunCount: 10, + }); + + //create an attempt + const attemptResult = await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + + //fail the attempt + const error = { + type: "BUILT_IN_ERROR" as const, + name: "UserError", + message: "This is a user error", + stackTrace: "Error: This is a user error\n at :1:1", + }; + const result = await engine.completeRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: attemptResult.snapshot.id, + completion: { + ok: false, + id: dequeued[0].run.id, + error, + }, + }); + expect(result.attemptStatus).toBe("RUN_FINISHED"); + expect(result.snapshot.executionStatus).toBe("FINISHED"); + expect(result.run.attemptNumber).toBe(1); + expect(result.run.status).toBe("COMPLETED_WITH_ERRORS"); + + //state should be completed + const executionData3 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData3); + expect(executionData3.snapshot.executionStatus).toBe("FINISHED"); + expect(executionData3.run.attemptNumber).toBe(1); + expect(executionData3.run.status).toBe("COMPLETED_WITH_ERRORS"); + + //concurrency should have been released + const envConcurrencyCompleted = await engine.runQueue.currentConcurrencyOfEnvironment( + authenticatedEnvironment + ); + expect(envConcurrencyCompleted).toBe(0); + + //waitpoint should have been completed, with the output + const runWaitpointAfter = await prisma.waitpoint.findMany({ + where: { + completedByTaskRunId: run.id, + }, + }); + expect(runWaitpointAfter.length).toBe(1); + expect(runWaitpointAfter[0].type).toBe("RUN"); + const output = JSON.parse(runWaitpointAfter[0].output as string); + expect(output.type).toBe(error.type); + expect(runWaitpointAfter[0].outputIsError).toBe(true); + } finally { + engine.quit(); + } + }); + + containerTest( + "Single run (retry attempt, then succeed)", + { timeout: 15_000 }, + async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + const backgroundWorker = await setupBackgroundWorker( + prisma, + authenticatedEnvironment, + taskIdentifier + ); + + //trigger the run + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + //dequeue the run + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: run.masterQueue, + maxRunCount: 10, + }); + + //create an attempt + const attemptResult = await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + + //fail the attempt + const error = { + type: "BUILT_IN_ERROR" as const, + name: "UserError", + message: "This is a user error", + stackTrace: "Error: This is a user error\n at :1:1", + }; + const result = await engine.completeRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: attemptResult.snapshot.id, + completion: { + ok: false, + id: dequeued[0].run.id, + error, + retry: { + timestamp: Date.now(), + delay: 0, + }, + }, + }); + expect(result.attemptStatus).toBe("RETRY_IMMEDIATELY"); + expect(result.snapshot.executionStatus).toBe("PENDING_EXECUTING"); + expect(result.run.status).toBe("RETRYING_AFTER_FAILURE"); + + //state should be completed + const executionData3 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData3); + expect(executionData3.snapshot.executionStatus).toBe("PENDING_EXECUTING"); + //only when the new attempt is created, should the attempt be increased + expect(executionData3.run.attemptNumber).toBe(1); + expect(executionData3.run.status).toBe("RETRYING_AFTER_FAILURE"); + + //create a second attempt + const attemptResult2 = await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: executionData3.snapshot.id, + }); + expect(attemptResult2.run.attemptNumber).toBe(2); + + //now complete it successfully + const result2 = await engine.completeRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: attemptResult2.snapshot.id, + completion: { + ok: true, + id: dequeued[0].run.id, + output: `{"foo":"bar"}`, + outputType: "application/json", + }, + }); + expect(result2.snapshot.executionStatus).toBe("FINISHED"); + expect(result2.run.attemptNumber).toBe(2); + expect(result2.run.status).toBe("COMPLETED_SUCCESSFULLY"); + + //waitpoint should have been completed, with the output + const runWaitpointAfter = await prisma.waitpoint.findMany({ + where: { + completedByTaskRunId: run.id, + }, + }); + expect(runWaitpointAfter.length).toBe(1); + expect(runWaitpointAfter[0].type).toBe("RUN"); + expect(runWaitpointAfter[0].output).toBe(`{"foo":"bar"}`); + expect(runWaitpointAfter[0].outputIsError).toBe(false); + + //state should be completed + const executionData4 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData4); + expect(executionData4.snapshot.executionStatus).toBe("FINISHED"); + expect(executionData4.run.attemptNumber).toBe(2); + expect(executionData4.run.status).toBe("COMPLETED_SUCCESSFULLY"); + } finally { + engine.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/tests/triggerAndWait.test.ts b/internal-packages/run-engine/src/engine/tests/triggerAndWait.test.ts new file mode 100644 index 0000000000..8c8bab2bd0 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/triggerAndWait.test.ts @@ -0,0 +1,453 @@ +import { + assertNonNullable, + containerTest, + setupAuthenticatedEnvironment, + setupBackgroundWorker, +} from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { setTimeout } from "node:timers/promises"; + +describe("RunEngine triggerAndWait", () => { + containerTest("triggerAndWait", { timeout: 15_000 }, async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const parentTask = "parent-task"; + const childTask = "child-task"; + + //create background worker + await setupBackgroundWorker(prisma, authenticatedEnvironment, [parentTask, childTask]); + + //trigger the run + const parentRun = await engine.trigger( + { + number: 1, + friendlyId: "run_p1234", + environment: authenticatedEnvironment, + taskIdentifier: parentTask, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: `task/${parentTask}`, + isTest: false, + tags: [], + }, + prisma + ); + + //dequeue parent + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: parentRun.masterQueue, + maxRunCount: 10, + }); + + //create an attempt + const initialExecutionData = await engine.getRunExecutionData({ runId: parentRun.id }); + assertNonNullable(initialExecutionData); + const attemptResult = await engine.startRunAttempt({ + runId: parentRun.id, + snapshotId: initialExecutionData.snapshot.id, + }); + + const childRun = await engine.trigger( + { + number: 1, + friendlyId: "run_c1234", + environment: authenticatedEnvironment, + taskIdentifier: childTask, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: `task/${childTask}`, + isTest: false, + tags: [], + resumeParentOnCompletion: true, + parentTaskRunId: parentRun.id, + }, + prisma + ); + + const childExecutionData = await engine.getRunExecutionData({ runId: childRun.id }); + assertNonNullable(childExecutionData); + expect(childExecutionData.snapshot.executionStatus).toBe("QUEUED"); + + const parentExecutionData = await engine.getRunExecutionData({ runId: parentRun.id }); + assertNonNullable(parentExecutionData); + expect(parentExecutionData.snapshot.executionStatus).toBe("EXECUTING_WITH_WAITPOINTS"); + + //check the waitpoint blocking the parent run + const runWaitpoint = await prisma.taskRunWaitpoint.findFirst({ + where: { + taskRunId: parentRun.id, + }, + include: { + waitpoint: true, + }, + }); + assertNonNullable(runWaitpoint); + expect(runWaitpoint.waitpoint.type).toBe("RUN"); + expect(runWaitpoint.waitpoint.completedByTaskRunId).toBe(childRun.id); + + //dequeue the child run + const dequeuedChild = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: childRun.masterQueue, + maxRunCount: 10, + }); + + //start the child run + const childAttempt = await engine.startRunAttempt({ + runId: childRun.id, + snapshotId: dequeuedChild[0].snapshot.id, + }); + + // complete the child run + await engine.completeRunAttempt({ + runId: childRun.id, + snapshotId: childAttempt.snapshot.id, + completion: { + id: childRun.id, + ok: true, + output: '{"foo":"bar"}', + outputType: "application/json", + }, + }); + + //child snapshot + const childExecutionDataAfter = await engine.getRunExecutionData({ runId: childRun.id }); + assertNonNullable(childExecutionDataAfter); + expect(childExecutionDataAfter.snapshot.executionStatus).toBe("FINISHED"); + + const waitpointAfter = await prisma.waitpoint.findFirst({ + where: { + id: runWaitpoint.waitpointId, + }, + }); + expect(waitpointAfter?.completedAt).not.toBeNull(); + expect(waitpointAfter?.status).toBe("COMPLETED"); + expect(waitpointAfter?.output).toBe('{"foo":"bar"}'); + + await setTimeout(500); + + const runWaitpointAfter = await prisma.taskRunWaitpoint.findFirst({ + where: { + taskRunId: parentRun.id, + }, + include: { + waitpoint: true, + }, + }); + expect(runWaitpointAfter).toBeNull(); + + //parent snapshot + const parentExecutionDataAfter = await engine.getRunExecutionData({ runId: parentRun.id }); + assertNonNullable(parentExecutionDataAfter); + expect(parentExecutionDataAfter.snapshot.executionStatus).toBe("EXECUTING"); + expect(parentExecutionDataAfter.completedWaitpoints?.length).toBe(1); + expect(parentExecutionDataAfter.completedWaitpoints![0].id).toBe(runWaitpoint.waitpointId); + expect(parentExecutionDataAfter.completedWaitpoints![0].completedByTaskRun?.id).toBe( + childRun.id + ); + expect(parentExecutionDataAfter.completedWaitpoints![0].output).toBe('{"foo":"bar"}'); + } finally { + engine.quit(); + } + }); + + /** This happens if you `triggerAndWait` with an idempotencyKey if that run is in progress */ + containerTest( + "triggerAndWait two runs with shared awaited child", + { timeout: 15_000 }, + async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const parentTask = "parent-task"; + const childTask = "child-task"; + + //create background worker + await setupBackgroundWorker(prisma, authenticatedEnvironment, [parentTask, childTask]); + + //trigger the run + const parentRun1 = await engine.trigger( + { + number: 1, + friendlyId: "run_p1234", + environment: authenticatedEnvironment, + taskIdentifier: parentTask, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: `task/${parentTask}`, + isTest: false, + tags: [], + }, + prisma + ); + + //dequeue parent and create the attempt + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: parentRun1.masterQueue, + maxRunCount: 10, + }); + const attemptResult = await engine.startRunAttempt({ + runId: parentRun1.id, + snapshotId: dequeued[0].snapshot.id, + }); + + //trigger the child + const childRun = await engine.trigger( + { + number: 1, + friendlyId: "run_c1234", + environment: authenticatedEnvironment, + taskIdentifier: childTask, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: `task/${childTask}`, + isTest: false, + tags: [], + resumeParentOnCompletion: true, + parentTaskRunId: parentRun1.id, + }, + prisma + ); + + const childExecutionData = await engine.getRunExecutionData({ runId: childRun.id }); + assertNonNullable(childExecutionData); + expect(childExecutionData.snapshot.executionStatus).toBe("QUEUED"); + + const parentExecutionData = await engine.getRunExecutionData({ runId: parentRun1.id }); + assertNonNullable(parentExecutionData); + expect(parentExecutionData.snapshot.executionStatus).toBe("EXECUTING_WITH_WAITPOINTS"); + + //check the waitpoint blocking the parent run + const runWaitpoint = await prisma.taskRunWaitpoint.findFirst({ + where: { + taskRunId: parentRun1.id, + }, + include: { + waitpoint: true, + }, + }); + assertNonNullable(runWaitpoint); + expect(runWaitpoint.waitpoint.type).toBe("RUN"); + expect(runWaitpoint.waitpoint.completedByTaskRunId).toBe(childRun.id); + + //dequeue the child run + const dequeuedChild = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: childRun.masterQueue, + maxRunCount: 10, + }); + + //start the child run + const childAttempt = await engine.startRunAttempt({ + runId: childRun.id, + snapshotId: dequeuedChild[0].snapshot.id, + }); + + //trigger a second parent run + const parentRun2 = await engine.trigger( + { + number: 2, + friendlyId: "run_p1235", + environment: authenticatedEnvironment, + taskIdentifier: parentTask, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12346", + spanId: "s12346", + masterQueue: "main", + queueName: `task/${parentTask}`, + isTest: false, + tags: [], + }, + prisma + ); + //dequeue 2nd parent + const dequeued2 = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: parentRun2.masterQueue, + maxRunCount: 10, + }); + + //create the 2nd parent attempt + const attemptResultParent2 = await engine.startRunAttempt({ + runId: parentRun2.id, + snapshotId: dequeued2[0].snapshot.id, + }); + + //block the 2nd parent run with the child + const childRunWithWaitpoint = await prisma.taskRun.findUniqueOrThrow({ + where: { id: childRun.id }, + include: { + associatedWaitpoint: true, + }, + }); + const blockedResult = await engine.blockRunWithWaitpoint({ + runId: parentRun2.id, + waitpointId: childRunWithWaitpoint.associatedWaitpoint!.id, + environmentId: authenticatedEnvironment.id, + projectId: authenticatedEnvironment.project.id, + tx: prisma, + }); + expect(blockedResult.executionStatus).toBe("EXECUTING_WITH_WAITPOINTS"); + const parent2ExecutionData = await engine.getRunExecutionData({ runId: parentRun2.id }); + assertNonNullable(parent2ExecutionData); + expect(parent2ExecutionData.snapshot.executionStatus).toBe("EXECUTING_WITH_WAITPOINTS"); + + // complete the child run + await engine.completeRunAttempt({ + runId: childRun.id, + snapshotId: childAttempt.snapshot.id, + completion: { + id: childRun.id, + ok: true, + output: '{"foo":"bar"}', + outputType: "application/json", + }, + }); + + //child snapshot + const childExecutionDataAfter = await engine.getRunExecutionData({ runId: childRun.id }); + assertNonNullable(childExecutionDataAfter); + expect(childExecutionDataAfter.snapshot.executionStatus).toBe("FINISHED"); + + const waitpointAfter = await prisma.waitpoint.findFirst({ + where: { + id: runWaitpoint.waitpointId, + }, + }); + expect(waitpointAfter?.completedAt).not.toBeNull(); + expect(waitpointAfter?.status).toBe("COMPLETED"); + expect(waitpointAfter?.output).toBe('{"foo":"bar"}'); + + await setTimeout(500); + + const parent1RunWaitpointAfter = await prisma.taskRunWaitpoint.findFirst({ + where: { + taskRunId: parentRun1.id, + }, + }); + expect(parent1RunWaitpointAfter).toBeNull(); + + const parent2RunWaitpointAfter = await prisma.taskRunWaitpoint.findFirst({ + where: { + taskRunId: parentRun2.id, + }, + }); + expect(parent2RunWaitpointAfter).toBeNull(); + + //parent snapshot + const parentExecutionDataAfter = await engine.getRunExecutionData({ runId: parentRun1.id }); + assertNonNullable(parentExecutionDataAfter); + expect(parentExecutionDataAfter.snapshot.executionStatus).toBe("EXECUTING"); + expect(parentExecutionDataAfter.completedWaitpoints?.length).toBe(1); + expect(parentExecutionDataAfter.completedWaitpoints![0].id).toBe(runWaitpoint.waitpointId); + expect(parentExecutionDataAfter.completedWaitpoints![0].completedByTaskRun?.id).toBe( + childRun.id + ); + expect(parentExecutionDataAfter.completedWaitpoints![0].output).toBe('{"foo":"bar"}'); + + //parent 2 snapshot + const parent2ExecutionDataAfter = await engine.getRunExecutionData({ + runId: parentRun2.id, + }); + assertNonNullable(parent2ExecutionDataAfter); + expect(parent2ExecutionDataAfter.snapshot.executionStatus).toBe("EXECUTING"); + expect(parent2ExecutionDataAfter.completedWaitpoints?.length).toBe(1); + expect(parent2ExecutionDataAfter.completedWaitpoints![0].id).toBe( + childRunWithWaitpoint.associatedWaitpoint!.id + ); + expect(parentExecutionDataAfter.completedWaitpoints![0].completedByTaskRun?.id).toBe( + childRun.id + ); + expect(parent2ExecutionDataAfter.completedWaitpoints![0].output).toBe('{"foo":"bar"}'); + } finally { + engine.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/tests/ttl.test.ts b/internal-packages/run-engine/src/engine/tests/ttl.test.ts new file mode 100644 index 0000000000..7f0d836672 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/ttl.test.ts @@ -0,0 +1,109 @@ +import { + containerTest, + setupAuthenticatedEnvironment, + setupBackgroundWorker, + assertNonNullable, +} from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { setTimeout } from "timers/promises"; +import { EventBusEventArgs } from "../eventBus.js"; + +describe("RunEngine ttl", () => { + containerTest("Run expiring (ttl)", { timeout: 15_000 }, async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + const backgroundWorker = await setupBackgroundWorker( + prisma, + authenticatedEnvironment, + taskIdentifier + ); + + //trigger the run + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + ttl: "1s", + }, + prisma + ); + + const executionData = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData); + expect(executionData.snapshot.executionStatus).toBe("QUEUED"); + + let expiredEventData: EventBusEventArgs<"runExpired">[0] | undefined = undefined; + engine.eventBus.on("runExpired", (result) => { + expiredEventData = result; + }); + + //wait for 1 seconds + await setTimeout(1_000); + + assertNonNullable(expiredEventData); + const assertedExpiredEventData = expiredEventData as EventBusEventArgs<"runExpired">[0]; + expect(assertedExpiredEventData.run.spanId).toBe(run.spanId); + + const executionData2 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData2); + expect(executionData2.snapshot.executionStatus).toBe("FINISHED"); + expect(executionData2.run.attemptNumber).toBe(undefined); + expect(executionData2.run.status).toBe("EXPIRED"); + + //concurrency should have been released + const envConcurrencyCompleted = await engine.runQueue.currentConcurrencyOfEnvironment( + authenticatedEnvironment + ); + expect(envConcurrencyCompleted).toBe(0); + } finally { + engine.quit(); + } + }); +}); diff --git a/internal-packages/run-engine/src/engine/tests/waitpoints.test.ts b/internal-packages/run-engine/src/engine/tests/waitpoints.test.ts new file mode 100644 index 0000000000..419efacb85 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/waitpoints.test.ts @@ -0,0 +1,663 @@ +import { + assertNonNullable, + containerTest, + setupAuthenticatedEnvironment, + setupBackgroundWorker, +} from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { setTimeout } from "timers/promises"; +import { EventBusEventArgs } from "../eventBus.js"; + +describe("RunEngine Waitpoints", () => { + containerTest("waitForDuration", { timeout: 15_000 }, async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + await setupBackgroundWorker(prisma, authenticatedEnvironment, taskIdentifier); + + //trigger the run + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_p1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + //dequeue the run + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: run.masterQueue, + maxRunCount: 10, + }); + + //create an attempt + const attemptResult = await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + expect(attemptResult.snapshot.executionStatus).toBe("EXECUTING"); + + const durationMs = 1_000; + + //waitForDuration + const date = new Date(Date.now() + durationMs); + const result = await engine.waitForDuration({ + runId: run.id, + snapshotId: attemptResult.snapshot.id, + date, + releaseConcurrency: false, + }); + expect(result.waitUntil.toISOString()).toBe(date.toISOString()); + expect(result.snapshot.executionStatus).toBe("EXECUTING_WITH_WAITPOINTS"); + expect(result.run.status).toBe("EXECUTING"); + + const executionData = await engine.getRunExecutionData({ runId: run.id }); + expect(executionData?.snapshot.executionStatus).toBe("EXECUTING_WITH_WAITPOINTS"); + + await setTimeout(1_500); + + const waitpoint = await prisma.waitpoint.findFirst({ + where: { + id: result.waitpoint.id, + }, + }); + expect(waitpoint?.status).toBe("COMPLETED"); + expect(waitpoint?.completedAt?.getTime()).toBeLessThanOrEqual(date.getTime() + 200); + + const executionDataAfter = await engine.getRunExecutionData({ runId: run.id }); + expect(executionDataAfter?.snapshot.executionStatus).toBe("EXECUTING"); + } finally { + engine.quit(); + } + }); + + containerTest( + "Waitpoints cleared if attempt fails", + { timeout: 15_000 }, + async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + await setupBackgroundWorker(prisma, authenticatedEnvironment, taskIdentifier); + + //trigger the run + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_p1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + //dequeue the run + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: run.masterQueue, + maxRunCount: 10, + }); + + //create an attempt + const attemptResult = await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + expect(attemptResult.snapshot.executionStatus).toBe("EXECUTING"); + + //waitForDuration + const date = new Date(Date.now() + 60_000); + const result = await engine.waitForDuration({ + runId: run.id, + snapshotId: attemptResult.snapshot.id, + date, + releaseConcurrency: false, + }); + + const executionData = await engine.getRunExecutionData({ runId: run.id }); + expect(executionData?.snapshot.executionStatus).toBe("EXECUTING_WITH_WAITPOINTS"); + + //fail the attempt (user error) + const error = { + type: "BUILT_IN_ERROR" as const, + name: "UserError", + message: "This is a user error", + stackTrace: "Error: This is a user error\n at :1:1", + }; + const failResult = await engine.completeRunAttempt({ + runId: executionData!.run.id, + snapshotId: executionData!.snapshot.id, + completion: { + ok: false, + id: executionData!.run.id, + error, + retry: { + timestamp: Date.now(), + delay: 0, + }, + }, + }); + expect(failResult.attemptStatus).toBe("RETRY_IMMEDIATELY"); + expect(failResult.snapshot.executionStatus).toBe("PENDING_EXECUTING"); + expect(failResult.run.attemptNumber).toBe(1); + expect(failResult.run.status).toBe("RETRYING_AFTER_FAILURE"); + + const executionData2 = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData2); + expect(executionData2.snapshot.executionStatus).toBe("PENDING_EXECUTING"); + expect(executionData2.run.attemptNumber).toBe(1); + expect(executionData2.run.status).toBe("RETRYING_AFTER_FAILURE"); + expect(executionData2.completedWaitpoints.length).toBe(0); + + //check there are no waitpoints blocking the parent run + const runWaitpoint = await prisma.taskRunWaitpoint.findFirst({ + where: { + taskRunId: run.id, + }, + include: { + waitpoint: true, + }, + }); + expect(runWaitpoint).toBeNull(); + } finally { + engine.quit(); + } + } + ); + + containerTest( + "Create, block, and complete a Manual waitpoint", + { timeout: 15_000 }, + async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + await setupBackgroundWorker(prisma, authenticatedEnvironment, taskIdentifier); + + //trigger the run + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_p1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + //dequeue the run + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: run.masterQueue, + maxRunCount: 10, + }); + + //create an attempt + const attemptResult = await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + expect(attemptResult.snapshot.executionStatus).toBe("EXECUTING"); + + //create a manual waitpoint + const waitpoint = await engine.createManualWaitpoint({ + environmentId: authenticatedEnvironment.id, + projectId: authenticatedEnvironment.projectId, + }); + expect(waitpoint.status).toBe("PENDING"); + + //block the run + await engine.blockRunWithWaitpoint({ + runId: run.id, + waitpointId: waitpoint.id, + environmentId: authenticatedEnvironment.id, + projectId: authenticatedEnvironment.projectId, + }); + + const executionData = await engine.getRunExecutionData({ runId: run.id }); + expect(executionData?.snapshot.executionStatus).toBe("EXECUTING_WITH_WAITPOINTS"); + + //check there is a waitpoint blocking the parent run + const runWaitpointBefore = await prisma.taskRunWaitpoint.findFirst({ + where: { + taskRunId: run.id, + }, + include: { + waitpoint: true, + }, + }); + expect(runWaitpointBefore?.waitpointId).toBe(waitpoint.id); + + let event: EventBusEventArgs<"workerNotification">[0] | undefined = undefined; + engine.eventBus.on("workerNotification", (result) => { + event = result; + }); + + //complete the waitpoint + await engine.completeWaitpoint({ + id: waitpoint.id, + }); + + await setTimeout(200); + + assertNonNullable(event); + const notificationEvent = event as EventBusEventArgs<"workerNotification">[0]; + expect(notificationEvent.run.id).toBe(run.id); + + const executionData2 = await engine.getRunExecutionData({ runId: run.id }); + expect(executionData2?.snapshot.executionStatus).toBe("EXECUTING"); + + //check there are no waitpoints blocking the parent run + const runWaitpoint = await prisma.taskRunWaitpoint.findFirst({ + where: { + taskRunId: run.id, + }, + include: { + waitpoint: true, + }, + }); + expect(runWaitpoint).toBeNull(); + } finally { + engine.quit(); + } + } + ); + + containerTest( + "Manual waitpoint timeout", + { timeout: 15_000 }, + async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + await setupBackgroundWorker(prisma, authenticatedEnvironment, taskIdentifier); + + //trigger the run + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_p1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + //dequeue the run + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: run.masterQueue, + maxRunCount: 10, + }); + + //create an attempt + const attemptResult = await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + expect(attemptResult.snapshot.executionStatus).toBe("EXECUTING"); + + //create a manual waitpoint + const waitpoint = await engine.createManualWaitpoint({ + environmentId: authenticatedEnvironment.id, + projectId: authenticatedEnvironment.projectId, + }); + + //block the run + await engine.blockRunWithWaitpoint({ + runId: run.id, + waitpointId: waitpoint.id, + environmentId: authenticatedEnvironment.id, + projectId: authenticatedEnvironment.projectId, + //fail after 200ms + failAfter: new Date(Date.now() + 200), + }); + + const executionData = await engine.getRunExecutionData({ runId: run.id }); + expect(executionData?.snapshot.executionStatus).toBe("EXECUTING_WITH_WAITPOINTS"); + + await setTimeout(750); + + const executionData2 = await engine.getRunExecutionData({ runId: run.id }); + expect(executionData2?.snapshot.executionStatus).toBe("EXECUTING"); + expect(executionData2?.completedWaitpoints.length).toBe(1); + expect(executionData2?.completedWaitpoints[0].outputIsError).toBe(true); + + //check there are no waitpoints blocking the parent run + const runWaitpoint = await prisma.taskRunWaitpoint.findFirst({ + where: { + taskRunId: run.id, + }, + include: { + waitpoint: true, + }, + }); + expect(runWaitpoint).toBeNull(); + } finally { + engine.quit(); + } + } + ); + + containerTest( + "Race condition with multiple waitpoints completing simultaneously", + { timeout: 60_000 }, + async ({ prisma, redisContainer }) => { + //create environment + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + worker: { + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + //create background worker + await setupBackgroundWorker(prisma, authenticatedEnvironment, taskIdentifier); + + //trigger the run + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_p1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + //dequeue the run + const dequeued = await engine.dequeueFromMasterQueue({ + consumerId: "test_12345", + masterQueue: run.masterQueue, + maxRunCount: 10, + }); + + //create an attempt + const attemptResult = await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + expect(attemptResult.snapshot.executionStatus).toBe("EXECUTING"); + + const iterationCount = 10; + + for (let i = 0; i < iterationCount; i++) { + const waitpointCount = 5; + + //create waitpoints + const waitpoints = await Promise.all( + Array.from({ length: waitpointCount }).map(() => + engine.createManualWaitpoint({ + environmentId: authenticatedEnvironment.id, + projectId: authenticatedEnvironment.projectId, + }) + ) + ); + + //block the run with them + await Promise.all( + waitpoints.map((waitpoint) => + engine.blockRunWithWaitpoint({ + runId: run.id, + waitpointId: waitpoint.id, + environmentId: authenticatedEnvironment.id, + projectId: authenticatedEnvironment.projectId, + }) + ) + ); + + const executionData = await engine.getRunExecutionData({ runId: run.id }); + expect(executionData?.snapshot.executionStatus).toBe("EXECUTING_WITH_WAITPOINTS"); + + //check there is a waitpoint blocking the parent run + const runWaitpointsBefore = await prisma.taskRunWaitpoint.findMany({ + where: { + taskRunId: run.id, + }, + include: { + waitpoint: true, + }, + }); + expect(runWaitpointsBefore.length).toBe(waitpointCount); + + //complete the waitpoints + await Promise.all( + waitpoints.map((waitpoint) => + engine.completeWaitpoint({ + id: waitpoint.id, + }) + ) + ); + + await setTimeout(500); + + //expect the run to be executing again + const executionData2 = await engine.getRunExecutionData({ runId: run.id }); + expect(executionData2?.snapshot.executionStatus).toBe("EXECUTING"); + + //check there are no waitpoints blocking the parent run + const runWaitpoints = await prisma.taskRunWaitpoint.findMany({ + where: { + taskRunId: run.id, + }, + include: { + waitpoint: true, + }, + }); + expect(runWaitpoints.length).toBe(0); + } + } finally { + engine.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/types.ts b/internal-packages/run-engine/src/engine/types.ts new file mode 100644 index 0000000000..1d3efb222c --- /dev/null +++ b/internal-packages/run-engine/src/engine/types.ts @@ -0,0 +1,82 @@ +import { type WorkerConcurrencyOptions } from "@internal/redis-worker"; +import { Tracer } from "@opentelemetry/api"; +import { MachinePreset, MachinePresetName, QueueOptions, RetryOptions } from "@trigger.dev/core/v3"; +import { PrismaClient } from "@trigger.dev/database"; +import { type RedisOptions } from "ioredis"; +import { MinimalAuthenticatedEnvironment } from "../shared"; + +export type RunEngineOptions = { + redis: RedisOptions; + prisma: PrismaClient; + worker: WorkerConcurrencyOptions & { + pollIntervalMs?: number; + }; + machines: { + defaultMachine: MachinePresetName; + machines: Record; + baseCostInCents: number; + }; + queue?: { + retryOptions?: RetryOptions; + defaultEnvConcurrency?: number; + }; + /** If not set then checkpoints won't ever be used */ + retryWarmStartThresholdMs?: number; + heartbeatTimeoutsMs?: Partial; + queueRunsWaitingForWorkerBatchSize?: number; + tracer: Tracer; +}; + +export type HeartbeatTimeouts = { + PENDING_EXECUTING: number; + PENDING_CANCEL: number; + EXECUTING: number; + EXECUTING_WITH_WAITPOINTS: number; +}; + +export type MachineResources = { + cpu: number; + memory: number; +}; + +export type TriggerParams = { + friendlyId: string; + number: number; + environment: MinimalAuthenticatedEnvironment; + idempotencyKey?: string; + idempotencyKeyExpiresAt?: Date; + taskIdentifier: string; + payload: string; + payloadType: string; + context: any; + traceContext: Record; + traceId: string; + spanId: string; + parentSpanId?: string; + lockedToVersionId?: string; + taskVersion?: string; + sdkVersion?: string; + cliVersion?: string; + concurrencyKey?: string; + masterQueue: string; + queueName: string; + queue?: QueueOptions; + isTest: boolean; + delayUntil?: Date; + queuedAt?: Date; + maxAttempts?: number; + priorityMs?: number; + ttl?: string; + tags: { id: string; name: string }[]; + parentTaskRunId?: string; + rootTaskRunId?: string; + batchId?: string; + resumeParentOnCompletion?: boolean; + depth?: number; + metadata?: string; + metadataType?: string; + seedMetadata?: string; + seedMetadataType?: string; + oneTimeUseToken?: string; + maxDurationInSeconds?: number; +}; diff --git a/internal-packages/run-engine/src/index.ts b/internal-packages/run-engine/src/index.ts new file mode 100644 index 0000000000..b71175be2a --- /dev/null +++ b/internal-packages/run-engine/src/index.ts @@ -0,0 +1 @@ +export { RunEngine } from "./engine/index"; diff --git a/internal-packages/run-engine/src/run-queue/index.test.ts b/internal-packages/run-engine/src/run-queue/index.test.ts new file mode 100644 index 0000000000..c2eee409f1 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/index.test.ts @@ -0,0 +1,825 @@ +import { redisTest } from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { Logger } from "@trigger.dev/core/logger"; +import Redis from "ioredis"; +import { describe } from "node:test"; +import { setTimeout } from "node:timers/promises"; +import { RunQueue } from "./index.js"; +import { SimpleWeightedChoiceStrategy } from "./simpleWeightedPriorityStrategy.js"; +import { InputPayload } from "./types.js"; + +const testOptions = { + name: "rq", + tracer: trace.getTracer("rq"), + queuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 36 }), + envQueuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 12 }), + workers: 1, + defaultEnvConcurrency: 25, + enableRebalancing: false, + logger: new Logger("RunQueue", "warn"), + retryOptions: { + maxAttempts: 5, + factor: 1.1, + minTimeoutInMs: 100, + maxTimeoutInMs: 1_000, + randomize: true, + }, +}; + +const authenticatedEnvProd = { + id: "e1234", + type: "PRODUCTION" as const, + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, +}; + +const authenticatedEnvDev = { + id: "e1234", + type: "DEVELOPMENT" as const, + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, +}; + +const messageProd: InputPayload = { + runId: "r1234", + taskIdentifier: "task/my-task", + orgId: "o1234", + projectId: "p1234", + environmentId: "e1234", + environmentType: "PRODUCTION", + queue: "task/my-task", + timestamp: Date.now(), + attempt: 0, +}; + +const messageDev: InputPayload = { + runId: "r4321", + taskIdentifier: "task/my-task", + orgId: "o1234", + projectId: "p1234", + environmentId: "e4321", + environmentType: "DEVELOPMENT", + queue: "task/my-task", + timestamp: Date.now(), + attempt: 0, +}; + +describe("RunQueue", () => { + redisTest( + "Get/set Queue concurrency limit", + { timeout: 15_000 }, + async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + //initial value + const initial = await queue.getQueueConcurrencyLimit(authenticatedEnvProd, "task/my-task"); + expect(initial).toBe(undefined); + + //set 20 + const result = await queue.updateQueueConcurrencyLimits( + authenticatedEnvProd, + "task/my-task", + 20 + ); + expect(result).toBe("OK"); + + //get 20 + const updated = await queue.getQueueConcurrencyLimit(authenticatedEnvProd, "task/my-task"); + expect(updated).toBe(20); + + //remove + const result2 = await queue.removeQueueConcurrencyLimits( + authenticatedEnvProd, + "task/my-task" + ); + expect(result2).toBe(1); + + //get undefined + const removed = await queue.getQueueConcurrencyLimit(authenticatedEnvProd, "task/my-task"); + expect(removed).toBe(undefined); + } finally { + await queue.quit(); + } + } + ); + + redisTest( + "Update env concurrency limits", + { timeout: 5_000 }, + async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + //initial value + const initial = await queue.getEnvConcurrencyLimit(authenticatedEnvProd); + expect(initial).toBe(25); + + //set 20 + await queue.updateEnvConcurrencyLimits({ + ...authenticatedEnvProd, + maximumConcurrencyLimit: 20, + }); + + //get 20 + const updated = await queue.getEnvConcurrencyLimit(authenticatedEnvProd); + expect(updated).toBe(20); + } finally { + await queue.quit(); + } + } + ); + + redisTest( + "Enqueue/Dequeue a message in env (DEV run, no concurrency key)", + { timeout: 5_000 }, + async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + //initial queue length + const result = await queue.lengthOfQueue(authenticatedEnvDev, messageDev.queue); + expect(result).toBe(0); + const envQueueLength = await queue.lengthOfEnvQueue(authenticatedEnvDev); + expect(envQueueLength).toBe(0); + + //initial oldest message + const oldestScore = await queue.oldestMessageInQueue(authenticatedEnvDev, messageDev.queue); + expect(oldestScore).toBe(undefined); + + const envMasterQueue = `env:${authenticatedEnvDev.id}`; + + //enqueue message + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: messageDev, + masterQueues: ["main", envMasterQueue], + }); + + //queue length + const result2 = await queue.lengthOfQueue(authenticatedEnvDev, messageDev.queue); + expect(result2).toBe(1); + const envQueueLength2 = await queue.lengthOfEnvQueue(authenticatedEnvDev); + expect(envQueueLength2).toBe(1); + + //oldest message + const oldestScore2 = await queue.oldestMessageInQueue( + authenticatedEnvDev, + messageDev.queue + ); + expect(oldestScore2).toBe(messageDev.timestamp); + + //concurrencies + const queueConcurrency = await queue.currentConcurrencyOfQueue( + authenticatedEnvDev, + messageDev.queue + ); + expect(queueConcurrency).toBe(0); + const envConcurrency = await queue.currentConcurrencyOfEnvironment(authenticatedEnvDev); + expect(envConcurrency).toBe(0); + const projectConcurrency = await queue.currentConcurrencyOfProject(authenticatedEnvDev); + expect(projectConcurrency).toBe(0); + const taskConcurrency = await queue.currentConcurrencyOfTask( + authenticatedEnvDev, + messageDev.taskIdentifier + ); + expect(taskConcurrency).toBe(0); + + const dequeued = await queue.dequeueMessageFromMasterQueue( + "test_12345", + envMasterQueue, + 10 + ); + expect(dequeued.length).toBe(1); + expect(dequeued[0].messageId).toEqual(messageDev.runId); + expect(dequeued[0].message.orgId).toEqual(messageDev.orgId); + expect(dequeued[0].message.version).toEqual("1"); + expect(dequeued[0].message.masterQueues).toEqual(["main", envMasterQueue]); + + //concurrencies + const queueConcurrency2 = await queue.currentConcurrencyOfQueue( + authenticatedEnvDev, + messageDev.queue + ); + expect(queueConcurrency2).toBe(1); + const envConcurrency2 = await queue.currentConcurrencyOfEnvironment(authenticatedEnvDev); + expect(envConcurrency2).toBe(1); + const projectConcurrency2 = await queue.currentConcurrencyOfProject(authenticatedEnvDev); + expect(projectConcurrency2).toBe(1); + const taskConcurrency2 = await queue.currentConcurrencyOfTask( + authenticatedEnvDev, + messageDev.taskIdentifier + ); + expect(taskConcurrency2).toBe(1); + + //queue lengths + const result3 = await queue.lengthOfQueue(authenticatedEnvDev, messageDev.queue); + expect(result3).toBe(0); + const envQueueLength3 = await queue.lengthOfEnvQueue(authenticatedEnvDev); + expect(envQueueLength3).toBe(0); + + const dequeued2 = await queue.dequeueMessageFromMasterQueue( + "test_12345", + envMasterQueue, + 10 + ); + expect(dequeued2.length).toBe(0); + + const dequeued3 = await queue.dequeueMessageFromMasterQueue("test_12345", "main", 10); + expect(dequeued3.length).toBe(0); + } finally { + await queue.quit(); + } + } + ); + + redisTest( + "Enqueue/Dequeue a message from the main queue (PROD run, no concurrency key)", + { timeout: 5_000 }, + async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + //initial queue length + const result = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); + expect(result).toBe(0); + const envQueueLength = await queue.lengthOfEnvQueue(authenticatedEnvProd); + expect(envQueueLength).toBe(0); + + //initial oldest message + const oldestScore = await queue.oldestMessageInQueue( + authenticatedEnvProd, + messageProd.queue + ); + expect(oldestScore).toBe(undefined); + + const envMasterQueue = `env:${authenticatedEnvDev.id}`; + + //enqueue message + await queue.enqueueMessage({ + env: authenticatedEnvProd, + message: messageProd, + masterQueues: ["main", envMasterQueue], + }); + + //queue length + const queueLength = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); + expect(queueLength).toBe(1); + const envLength = await queue.lengthOfEnvQueue(authenticatedEnvProd); + expect(envLength).toBe(1); + + //oldest message + const oldestScore2 = await queue.oldestMessageInQueue( + authenticatedEnvProd, + messageProd.queue + ); + expect(oldestScore2).toBe(messageProd.timestamp); + + //concurrencies + const queueConcurrency = await queue.currentConcurrencyOfQueue( + authenticatedEnvProd, + messageProd.queue + ); + expect(queueConcurrency).toBe(0); + const envConcurrency = await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd); + expect(envConcurrency).toBe(0); + const projectConcurrency = await queue.currentConcurrencyOfProject(authenticatedEnvProd); + expect(projectConcurrency).toBe(0); + const taskConcurrency = await queue.currentConcurrencyOfTask( + authenticatedEnvProd, + messageProd.taskIdentifier + ); + expect(taskConcurrency).toBe(0); + + //dequeue + const dequeued = await queue.dequeueMessageFromMasterQueue("test_12345", "main", 10); + expect(dequeued.length).toBe(1); + expect(dequeued[0].messageId).toEqual(messageProd.runId); + expect(dequeued[0].message.orgId).toEqual(messageProd.orgId); + expect(dequeued[0].message.version).toEqual("1"); + expect(dequeued[0].message.masterQueues).toEqual(["main", envMasterQueue]); + + //concurrencies + const queueConcurrency2 = await queue.currentConcurrencyOfQueue( + authenticatedEnvProd, + messageProd.queue + ); + expect(queueConcurrency2).toBe(1); + const envConcurrency2 = await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd); + expect(envConcurrency2).toBe(1); + const projectConcurrency2 = await queue.currentConcurrencyOfProject(authenticatedEnvProd); + expect(projectConcurrency2).toBe(1); + const taskConcurrency2 = await queue.currentConcurrencyOfTask( + authenticatedEnvProd, + messageProd.taskIdentifier + ); + expect(taskConcurrency2).toBe(1); + + //queue length + const length2 = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); + expect(length2).toBe(0); + const envLength2 = await queue.lengthOfEnvQueue(authenticatedEnvProd); + expect(envLength2).toBe(0); + + const dequeued2 = await queue.dequeueMessageFromMasterQueue("test_12345", "main", 10); + expect(dequeued2.length).toBe(0); + } finally { + await queue.quit(); + } + } + ); + + redisTest( + "Dequeue multiple messages from the queue", + { timeout: 5_000 }, + async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + // Create 20 messages with different runIds and some with different queues + const messages = Array.from({ length: 20 }, (_, i) => ({ + ...messageProd, + runId: `r${i + 1}`, + queue: i < 15 ? "task/my-task" : "task/other-task", // Mix up the queues + })); + + // Enqueue all messages + for (const message of messages) { + await queue.enqueueMessage({ + env: authenticatedEnvProd, + message, + masterQueues: "main", + }); + } + + // Check initial queue lengths + const initialLength1 = await queue.lengthOfQueue(authenticatedEnvProd, "task/my-task"); + const initialLength2 = await queue.lengthOfQueue(authenticatedEnvProd, "task/other-task"); + expect(initialLength1).toBe(15); + expect(initialLength2).toBe(5); + const envQueueLength = await queue.lengthOfEnvQueue(authenticatedEnvProd); + expect(envQueueLength).toBe(20); + + // Dequeue first batch of 10 messages + const dequeued1 = await queue.dequeueMessageFromMasterQueue("test_12345", "main", 10); + expect(dequeued1.length).toBe(10); + + // Dequeue second batch of 10 messages + const dequeued2 = await queue.dequeueMessageFromMasterQueue("test_12345", "main", 10); + expect(dequeued2.length).toBe(10); + + // Combine all dequeued message IDs + const dequeuedIds = [...dequeued1, ...dequeued2].map((m) => m.messageId); + + // Check that all original messages were dequeued + const allOriginalIds = messages.map((m) => m.runId); + expect(dequeuedIds.sort()).toEqual(allOriginalIds.sort()); + + // Try to dequeue more - should get none + const dequeued3 = await queue.dequeueMessageFromMasterQueue("test_12345", "main", 10); + expect(dequeued3.length).toBe(0); + + // Check final queue lengths + const finalLength1 = await queue.lengthOfQueue(authenticatedEnvProd, "task/my-task"); + const finalLength2 = await queue.lengthOfQueue(authenticatedEnvProd, "task/other-task"); + expect(finalLength1).toBe(0); + expect(finalLength2).toBe(0); + const finalEnvQueueLength = await queue.lengthOfEnvQueue(authenticatedEnvProd); + expect(finalEnvQueueLength).toBe(0); + } finally { + await queue.quit(); + } + } + ); + + redisTest("Get shared queue details", { timeout: 5_000 }, async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + const result = await queue.getSharedQueueDetails("main", 10); + expect(result.selectionId).toBe("getSharedQueueDetails"); + expect(result.queueCount).toBe(0); + expect(result.queueChoice.choices).toStrictEqual({ abort: true }); + + await queue.enqueueMessage({ + env: authenticatedEnvProd, + message: messageProd, + masterQueues: "main", + }); + + const result2 = await queue.getSharedQueueDetails("main", 10); + expect(result2.selectionId).toBe("getSharedQueueDetails"); + expect(result2.queueCount).toBe(1); + expect(result2.queues[0].score).toBe(messageProd.timestamp); + if (!Array.isArray(result2.queueChoice.choices)) { + throw new Error("Expected queueChoice.choices to be an array"); + } + expect(result2.queueChoice.choices[0]).toBe( + "{org:o1234}:proj:p1234:env:e1234:queue:task/my-task" + ); + } finally { + await queue.quit(); + } + }); + + redisTest("Acking", { timeout: 5_000 }, async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + await queue.enqueueMessage({ + env: authenticatedEnvProd, + message: messageProd, + masterQueues: "main", + }); + + const queueLength = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); + expect(queueLength).toBe(1); + const envQueueLength = await queue.lengthOfEnvQueue(authenticatedEnvProd); + expect(envQueueLength).toBe(1); + + const messages = await queue.dequeueMessageFromMasterQueue("test_12345", "main", 10); + expect(messages.length).toBe(1); + + const queueLength2 = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); + expect(queueLength2).toBe(0); + const envQueueLength2 = await queue.lengthOfEnvQueue(authenticatedEnvProd); + expect(envQueueLength2).toBe(0); + + //check the message is gone + const key = queue.keys.messageKey(messages[0].message.orgId, messages[0].messageId); + const exists = await redis.exists(key); + expect(exists).toBe(1); + + await queue.acknowledgeMessage(messages[0].message.orgId, messages[0].messageId); + + //concurrencies + const queueConcurrency = await queue.currentConcurrencyOfQueue( + authenticatedEnvProd, + messageProd.queue + ); + expect(queueConcurrency).toBe(0); + const envConcurrency = await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd); + expect(envConcurrency).toBe(0); + const projectConcurrency = await queue.currentConcurrencyOfProject(authenticatedEnvProd); + expect(projectConcurrency).toBe(0); + const taskConcurrency = await queue.currentConcurrencyOfTask( + authenticatedEnvProd, + messageProd.taskIdentifier + ); + expect(taskConcurrency).toBe(0); + + //queue lengths + const queueLength3 = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); + expect(queueLength3).toBe(0); + const envQueueLength3 = await queue.lengthOfEnvQueue(authenticatedEnvProd); + expect(envQueueLength3).toBe(0); + + //check the message is gone + const exists2 = await redis.exists(key); + expect(exists2).toBe(0); + + //dequeue + const messages2 = await queue.dequeueMessageFromMasterQueue("test_12345", "main", 10); + expect(messages2.length).toBe(0); + } finally { + await queue.quit(); + } + }); + + redisTest("Ack (before dequeue)", { timeout: 5_000 }, async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + await queue.enqueueMessage({ + env: authenticatedEnvProd, + message: messageProd, + masterQueues: "main", + }); + + const queueLength = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); + expect(queueLength).toBe(1); + const envQueueLength = await queue.lengthOfEnvQueue(authenticatedEnvProd); + expect(envQueueLength).toBe(1); + + await queue.acknowledgeMessage(messageProd.orgId, messageProd.runId); + + //concurrencies + const queueConcurrency = await queue.currentConcurrencyOfQueue( + authenticatedEnvProd, + messageProd.queue + ); + expect(queueConcurrency).toBe(0); + const envConcurrency = await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd); + expect(envConcurrency).toBe(0); + const projectConcurrency = await queue.currentConcurrencyOfProject(authenticatedEnvProd); + expect(projectConcurrency).toBe(0); + const taskConcurrency = await queue.currentConcurrencyOfTask( + authenticatedEnvProd, + messageProd.taskIdentifier + ); + expect(taskConcurrency).toBe(0); + + //queue lengths + const queueLength3 = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); + expect(queueLength3).toBe(0); + const envQueueLength3 = await queue.lengthOfEnvQueue(authenticatedEnvProd); + expect(envQueueLength3).toBe(0); + + //dequeue + const messages2 = await queue.dequeueMessageFromMasterQueue("test_12345", "main", 10); + expect(messages2.length).toBe(0); + } finally { + await queue.quit(); + } + }); + + redisTest("Nacking", { timeout: 15_000 }, async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + await queue.enqueueMessage({ + env: authenticatedEnvProd, + message: messageProd, + masterQueues: "main2", + }); + + const messages = await queue.dequeueMessageFromMasterQueue("test_12345", "main2", 10); + expect(messages.length).toBe(1); + + //check the message is there + const key = queue.keys.messageKey(messages[0].message.orgId, messages[0].messageId); + const exists = await redis.exists(key); + expect(exists).toBe(1); + + //concurrencies + const queueConcurrency = await queue.currentConcurrencyOfQueue( + authenticatedEnvProd, + messageProd.queue + ); + expect(queueConcurrency).toBe(1); + const envConcurrency = await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd); + expect(envConcurrency).toBe(1); + const projectConcurrency = await queue.currentConcurrencyOfProject(authenticatedEnvProd); + expect(projectConcurrency).toBe(1); + const taskConcurrency = await queue.currentConcurrencyOfTask( + authenticatedEnvProd, + messageProd.taskIdentifier + ); + expect(taskConcurrency).toBe(1); + + await queue.nackMessage({ + orgId: messages[0].message.orgId, + messageId: messages[0].messageId, + }); + + //we need to wait because the default wait is 1 second + await setTimeout(300); + + //concurrencies + const queueConcurrency2 = await queue.currentConcurrencyOfQueue( + authenticatedEnvProd, + messageProd.queue + ); + expect(queueConcurrency2).toBe(0); + const envConcurrency2 = await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd); + expect(envConcurrency2).toBe(0); + const projectConcurrency2 = await queue.currentConcurrencyOfProject(authenticatedEnvProd); + expect(projectConcurrency2).toBe(0); + const taskConcurrency2 = await queue.currentConcurrencyOfTask( + authenticatedEnvProd, + messageProd.taskIdentifier + ); + expect(taskConcurrency2).toBe(0); + + //queue lengths + const queueLength = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); + expect(queueLength).toBe(1); + const envQueueLength = await queue.lengthOfEnvQueue(authenticatedEnvProd); + expect(envQueueLength).toBe(1); + + //check the message is there + const exists2 = await redis.exists(key); + expect(exists2).toBe(1); + + //dequeue + const messages2 = await queue.dequeueMessageFromMasterQueue("test_12345", "main2", 10); + expect(messages2[0].messageId).toBe(messageProd.runId); + } finally { + await queue.quit(); + } + }); + + redisTest("Releasing concurrency", { timeout: 5_000 }, async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + await queue.enqueueMessage({ + env: authenticatedEnvProd, + message: messageProd, + masterQueues: "main", + }); + + const messages = await queue.dequeueMessageFromMasterQueue("test_12345", "main", 10); + expect(messages.length).toBe(1); + + //check the message is gone + const key = queue.keys.messageKey(messages[0].message.orgId, messages[0].messageId); + const exists = await redis.exists(key); + expect(exists).toBe(1); + + //concurrencies + expect(await queue.currentConcurrencyOfQueue(authenticatedEnvProd, messageProd.queue)).toBe( + 1 + ); + expect(await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd)).toBe(1); + expect(await queue.currentConcurrencyOfProject(authenticatedEnvProd)).toBe(1); + expect( + await queue.currentConcurrencyOfTask(authenticatedEnvProd, messageProd.taskIdentifier) + ).toBe(1); + + //release the concurrency (not the queue) + await queue.releaseConcurrency( + authenticatedEnvProd.organization.id, + messages[0].messageId, + false + ); + + //concurrencies + expect(await queue.currentConcurrencyOfQueue(authenticatedEnvProd, messageProd.queue)).toBe( + 1 + ); + expect(await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd)).toBe(0); + expect(await queue.currentConcurrencyOfProject(authenticatedEnvProd)).toBe(0); + expect( + await queue.currentConcurrencyOfTask(authenticatedEnvProd, messageProd.taskIdentifier) + ).toBe(0); + + //reacquire the concurrency + await queue.reacquireConcurrency(authenticatedEnvProd.organization.id, messages[0].messageId); + + //check concurrencies are back to what they were before + expect(await queue.currentConcurrencyOfQueue(authenticatedEnvProd, messageProd.queue)).toBe( + 1 + ); + expect(await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd)).toBe(1); + expect(await queue.currentConcurrencyOfProject(authenticatedEnvProd)).toBe(1); + expect( + await queue.currentConcurrencyOfTask(authenticatedEnvProd, messageProd.taskIdentifier) + ).toBe(1); + + //release the concurrency (with the queue this time) + await queue.releaseConcurrency( + authenticatedEnvProd.organization.id, + messages[0].messageId, + true + ); + + //concurrencies + expect(await queue.currentConcurrencyOfQueue(authenticatedEnvProd, messageProd.queue)).toBe( + 0 + ); + expect(await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd)).toBe(0); + expect(await queue.currentConcurrencyOfProject(authenticatedEnvProd)).toBe(0); + expect( + await queue.currentConcurrencyOfTask(authenticatedEnvProd, messageProd.taskIdentifier) + ).toBe(0); + + //reacquire the concurrency + await queue.reacquireConcurrency(authenticatedEnvProd.organization.id, messages[0].messageId); + + //check concurrencies are back to what they were before + expect(await queue.currentConcurrencyOfQueue(authenticatedEnvProd, messageProd.queue)).toBe( + 1 + ); + expect(await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd)).toBe(1); + expect(await queue.currentConcurrencyOfProject(authenticatedEnvProd)).toBe(1); + expect( + await queue.currentConcurrencyOfTask(authenticatedEnvProd, messageProd.taskIdentifier) + ).toBe(1); + } finally { + await queue.quit(); + } + }); + + redisTest("Dead Letter Queue", { timeout: 8_000 }, async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + retryOptions: { + maxAttempts: 1, + }, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + await queue.enqueueMessage({ + env: authenticatedEnvProd, + message: messageProd, + masterQueues: "main", + }); + + const messages = await queue.dequeueMessageFromMasterQueue("test_12345", "main", 10); + expect(messages.length).toBe(1); + + //check the message is there + const key = queue.keys.messageKey(messages[0].message.orgId, messages[0].messageId); + const exists = await redis.exists(key); + expect(exists).toBe(1); + + //nack (we only have attempts set to 1) + await queue.nackMessage({ + orgId: messages[0].message.orgId, + messageId: messages[0].messageId, + }); + + //dequeue + const messages2 = await queue.dequeueMessageFromMasterQueue("test_12345", "main", 10); + expect(messages2.length).toBe(0); + + //concurrencies + const queueConcurrency2 = await queue.currentConcurrencyOfQueue( + authenticatedEnvProd, + messageProd.queue + ); + expect(queueConcurrency2).toBe(0); + const envConcurrency2 = await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd); + expect(envConcurrency2).toBe(0); + const projectConcurrency2 = await queue.currentConcurrencyOfProject(authenticatedEnvProd); + expect(projectConcurrency2).toBe(0); + const taskConcurrency2 = await queue.currentConcurrencyOfTask( + authenticatedEnvProd, + messageProd.taskIdentifier + ); + expect(taskConcurrency2).toBe(0); + + //check the message is still there + const exists2 = await redis.exists(key); + expect(exists2).toBe(1); + + //check it's in the dlq + const dlqKey = "dlq"; + const dlqExists = await redis.exists(dlqKey); + expect(dlqExists).toBe(1); + const dlqMembers = await redis.zrange(dlqKey, 0, -1); + expect(dlqMembers).toContain(messageProd.runId); + + //redrive + const redisClient = new Redis({ + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + }); + + // Publish redrive message + await redisClient.publish( + "rq:redrive", + JSON.stringify({ runId: messageProd.runId, orgId: messageProd.orgId }) + ); + + // Wait for the item to be redrived and processed + await setTimeout(5_000); + await redisClient.quit(); + + //shouldn't be in the dlq now + const dlqMembersAfter = await redis.zrange(dlqKey, 0, -1); + expect(dlqMembersAfter).not.toContain(messageProd.runId); + + //dequeue + const messages3 = await queue.dequeueMessageFromMasterQueue("test_12345", "main", 10); + expect(messages3[0].messageId).toBe(messageProd.runId); + } finally { + await queue.quit(); + } + }); +}); diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts new file mode 100644 index 0000000000..b5cb643355 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -0,0 +1,1658 @@ +import { context, propagation, Span, SpanKind, SpanOptions, Tracer } from "@opentelemetry/api"; +import { + SEMATTRS_MESSAGE_ID, + SEMATTRS_MESSAGING_OPERATION, + SEMATTRS_MESSAGING_SYSTEM, +} from "@opentelemetry/semantic-conventions"; +import { Logger } from "@trigger.dev/core/logger"; +import { calculateNextRetryDelay, flattenAttributes } from "@trigger.dev/core/v3"; +import { type RetryOptions } from "@trigger.dev/core/v3/schemas"; +import { Redis, type Callback, type RedisOptions, type Result } from "ioredis"; +import { + attributesFromAuthenticatedEnv, + MinimalAuthenticatedEnvironment, +} from "../shared/index.js"; +import { RunQueueShortKeyProducer } from "./keyProducer.js"; +import { + InputPayload, + OutputPayload, + QueueCapacities, + QueueRange, + RunQueueKeyProducer, + RunQueuePriorityStrategy, +} from "./types.js"; + +const SemanticAttributes = { + QUEUE: "runqueue.queue", + MASTER_QUEUES: "runqueue.masterQueues", + RUN_ID: "runqueue.runId", + RESULT_COUNT: "runqueue.resultCount", + CONCURRENCY_KEY: "runqueue.concurrencyKey", + ORG_ID: "runqueue.orgId", +}; + +export type RunQueueOptions = { + name: string; + tracer: Tracer; + redis: RedisOptions; + defaultEnvConcurrency: number; + windowSize?: number; + queuePriorityStrategy: RunQueuePriorityStrategy; + envQueuePriorityStrategy: RunQueuePriorityStrategy; + verbose?: boolean; + logger: Logger; + retryOptions?: RetryOptions; +}; + +type DequeuedMessage = { + messageId: string; + messageScore: string; + message: OutputPayload; +}; + +const defaultRetrySettings = { + maxAttempts: 12, + factor: 2, + minTimeoutInMs: 1_000, + maxTimeoutInMs: 3_600_000, + randomize: true, +}; + +/** + * RunQueue – the queue that's used to process runs + */ +export class RunQueue { + private retryOptions: RetryOptions; + private subscriber: Redis; + private logger: Logger; + private redis: Redis; + public keys: RunQueueKeyProducer; + private queuePriorityStrategy: RunQueuePriorityStrategy; + + constructor(private readonly options: RunQueueOptions) { + this.retryOptions = options.retryOptions ?? defaultRetrySettings; + this.redis = new Redis(options.redis); + this.logger = options.logger; + + this.keys = new RunQueueShortKeyProducer("rq:"); + this.queuePriorityStrategy = options.queuePriorityStrategy; + + this.subscriber = new Redis(options.redis); + this.#setupSubscriber(); + + this.#registerCommands(); + } + + get name() { + return this.options.name; + } + + get tracer() { + return this.options.tracer; + } + + public async updateQueueConcurrencyLimits( + env: MinimalAuthenticatedEnvironment, + queue: string, + concurrency: number + ) { + return this.redis.set(this.keys.queueConcurrencyLimitKey(env, queue), concurrency); + } + + public async removeQueueConcurrencyLimits(env: MinimalAuthenticatedEnvironment, queue: string) { + return this.redis.del(this.keys.queueConcurrencyLimitKey(env, queue)); + } + + public async getQueueConcurrencyLimit(env: MinimalAuthenticatedEnvironment, queue: string) { + const result = await this.redis.get(this.keys.queueConcurrencyLimitKey(env, queue)); + + return result ? Number(result) : undefined; + } + + public async updateEnvConcurrencyLimits(env: MinimalAuthenticatedEnvironment) { + await this.#callUpdateGlobalConcurrencyLimits({ + envConcurrencyLimitKey: this.keys.envConcurrencyLimitKey(env), + envConcurrencyLimit: env.maximumConcurrencyLimit, + }); + } + + public async getEnvConcurrencyLimit(env: MinimalAuthenticatedEnvironment) { + const result = await this.redis.get(this.keys.envConcurrencyLimitKey(env)); + + return result ? Number(result) : this.options.defaultEnvConcurrency; + } + + public async lengthOfQueue( + env: MinimalAuthenticatedEnvironment, + queue: string, + concurrencyKey?: string + ) { + return this.redis.zcard(this.keys.queueKey(env, queue, concurrencyKey)); + } + + public async lengthOfEnvQueue(env: MinimalAuthenticatedEnvironment) { + return this.redis.zcard(this.keys.envQueueKey(env)); + } + + public async oldestMessageInQueue( + env: MinimalAuthenticatedEnvironment, + queue: string, + concurrencyKey?: string + ) { + // Get the "score" of the sorted set to get the oldest message score + const result = await this.redis.zrange( + this.keys.queueKey(env, queue, concurrencyKey), + 0, + 0, + "WITHSCORES" + ); + + if (result.length === 0) { + return; + } + + return Number(result[1]); + } + + public async currentConcurrencyOfQueue( + env: MinimalAuthenticatedEnvironment, + queue: string, + concurrencyKey?: string + ) { + return this.redis.scard(this.keys.currentConcurrencyKey(env, queue, concurrencyKey)); + } + + public async currentConcurrencyOfEnvironment(env: MinimalAuthenticatedEnvironment) { + return this.redis.scard(this.keys.envCurrentConcurrencyKey(env)); + } + + public async currentConcurrencyOfProject(env: MinimalAuthenticatedEnvironment) { + return this.redis.scard(this.keys.projectCurrentConcurrencyKey(env)); + } + + public async currentConcurrencyOfTask( + env: MinimalAuthenticatedEnvironment, + taskIdentifier: string + ) { + return this.redis.scard(this.keys.taskIdentifierCurrentConcurrencyKey(env, taskIdentifier)); + } + + public async enqueueMessage({ + env, + message, + masterQueues, + }: { + env: MinimalAuthenticatedEnvironment; + message: InputPayload; + masterQueues: string | string[]; + }) { + return await this.#trace( + "enqueueMessage", + async (span) => { + const { runId, concurrencyKey } = message; + + const queue = this.keys.queueKey(env, message.queue, concurrencyKey); + + propagation.inject(context.active(), message); + + const parentQueues = typeof masterQueues === "string" ? [masterQueues] : masterQueues; + + span.setAttributes({ + [SemanticAttributes.QUEUE]: queue, + [SemanticAttributes.RUN_ID]: runId, + [SemanticAttributes.CONCURRENCY_KEY]: concurrencyKey, + [SemanticAttributes.MASTER_QUEUES]: parentQueues.join(","), + }); + + const messagePayload: OutputPayload = { + ...message, + version: "1", + queue, + masterQueues: parentQueues, + attempt: 0, + }; + + await this.#callEnqueueMessage(messagePayload, parentQueues); + }, + { + kind: SpanKind.PRODUCER, + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "publish", + [SEMATTRS_MESSAGE_ID]: message.runId, + [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", + ...attributesFromAuthenticatedEnv(env), + }, + } + ); + } + + public async getSharedQueueDetails(masterQueue: string, maxCount: number) { + const { range } = await this.queuePriorityStrategy.nextCandidateSelection( + masterQueue, + "getSharedQueueDetails" + ); + const queues = await this.#getChildQueuesWithScores(masterQueue, range); + + const queuesWithScores = await this.#calculateQueueScores(queues, (queue) => + this.#calculateMessageQueueCapacities(queue) + ); + + // We need to priority shuffle here to ensure all workers aren't just working on the highest priority queue + const result = this.queuePriorityStrategy.chooseQueues( + queuesWithScores, + masterQueue, + "getSharedQueueDetails", + range, + maxCount + ); + + return { + selectionId: "getSharedQueueDetails", + queues, + queuesWithScores, + nextRange: range, + queueCount: queues.length, + queueChoice: result, + }; + } + + /** + * Dequeue messages from the master queue + */ + public async dequeueMessageFromMasterQueue( + consumerId: string, + masterQueue: string, + maxCount: number + ): Promise { + return this.#trace( + "dequeueMessageInSharedQueue", + async (span) => { + // Read the parent queue for matching queues + const selectedQueues = await this.#getRandomQueueFromParentQueue( + masterQueue, + this.options.queuePriorityStrategy, + (queue) => this.#calculateMessageQueueCapacities(queue, { checkForDisabled: true }), + consumerId, + maxCount + ); + + if (!selectedQueues || selectedQueues.length === 0) { + return []; + } + + const messages: DequeuedMessage[] = []; + const remainingMessages = selectedQueues.map((q) => q.size); + let currentQueueIndex = 0; + + while (messages.length < maxCount) { + let foundMessage = false; + + // Try each queue once in this round + for (let i = 0; i < selectedQueues.length; i++) { + currentQueueIndex = (currentQueueIndex + i) % selectedQueues.length; + + // Skip if this queue is empty + if (remainingMessages[currentQueueIndex] <= 0) continue; + + const selectedQueue = selectedQueues[currentQueueIndex]; + const queue = selectedQueue.queue; + + const message = await this.#callDequeueMessage({ + messageQueue: queue, + concurrencyLimitKey: this.keys.concurrencyLimitKeyFromQueue(queue), + currentConcurrencyKey: this.keys.currentConcurrencyKeyFromQueue(queue), + envConcurrencyLimitKey: this.keys.envConcurrencyLimitKeyFromQueue(queue), + envCurrentConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(queue), + projectCurrentConcurrencyKey: this.keys.projectCurrentConcurrencyKeyFromQueue(queue), + messageKeyPrefix: this.keys.messageKeyPrefixFromQueue(queue), + envQueueKey: this.keys.envQueueKeyFromQueue(queue), + taskCurrentConcurrentKeyPrefix: + this.keys.taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queue), + }); + + if (message) { + messages.push(message); + remainingMessages[currentQueueIndex]--; + foundMessage = true; + break; + } else { + // If we failed to get a message, mark this queue as empty + remainingMessages[currentQueueIndex] = 0; + } + } + + // If we couldn't get a message from any queue, break + if (!foundMessage) break; + } + + span.setAttributes({ + [SemanticAttributes.RESULT_COUNT]: messages.length, + [SemanticAttributes.MASTER_QUEUES]: masterQueue, + }); + + return messages; + }, + { + kind: SpanKind.CONSUMER, + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "receive", + [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", + }, + } + ); + } + + /** + * Acknowledge a message, which will: + * - remove all data from the queue + * - release all concurrency + * This is done when the run is in a final state. + * @param messageId + */ + public async acknowledgeMessage(orgId: string, messageId: string) { + return this.#trace( + "acknowledgeMessage", + async (span) => { + const message = await this.#readMessage(orgId, messageId); + + if (!message) { + this.logger.log(`[${this.name}].acknowledgeMessage() message not found`, { + messageId, + service: this.name, + }); + return; + } + + span.setAttributes({ + [SemanticAttributes.QUEUE]: message.queue, + [SemanticAttributes.ORG_ID]: message.orgId, + [SemanticAttributes.RUN_ID]: messageId, + [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, + }); + + await this.#callAcknowledgeMessage({ + messageId, + messageQueue: message.queue, + masterQueues: message.masterQueues, + messageKey: this.keys.messageKey(orgId, messageId), + concurrencyKey: this.keys.currentConcurrencyKeyFromQueue(message.queue), + envConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(message.queue), + taskConcurrencyKey: this.keys.taskIdentifierCurrentConcurrencyKeyFromQueue( + message.queue, + message.taskIdentifier + ), + envQueueKey: this.keys.envQueueKeyFromQueue(message.queue), + projectConcurrencyKey: this.keys.projectCurrentConcurrencyKeyFromQueue(message.queue), + }); + }, + { + kind: SpanKind.CONSUMER, + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "ack", + [SEMATTRS_MESSAGE_ID]: messageId, + [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", + }, + } + ); + } + + /** + * Negative acknowledge a message, which will requeue the message (with an optional future date). + If you pass no date it will get reattempted with exponential backoff. + */ + public async nackMessage({ + orgId, + messageId, + retryAt, + incrementAttemptCount = true, + }: { + orgId: string; + messageId: string; + retryAt?: number; + incrementAttemptCount?: boolean; + }) { + return this.#trace( + "nackMessage", + async (span) => { + const maxAttempts = this.retryOptions.maxAttempts ?? defaultRetrySettings.maxAttempts; + + const message = await this.#readMessage(orgId, messageId); + if (!message) { + this.logger.log(`[${this.name}].nackMessage() message not found`, { + orgId, + messageId, + maxAttempts, + retryAt, + service: this.name, + }); + return; + } + + span.setAttributes({ + [SemanticAttributes.QUEUE]: message.queue, + [SemanticAttributes.RUN_ID]: messageId, + [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, + [SemanticAttributes.MASTER_QUEUES]: message.masterQueues.join(","), + }); + + const messageKey = this.keys.messageKey(orgId, messageId); + const messageQueue = message.queue; + const concurrencyKey = this.keys.currentConcurrencyKeyFromQueue(message.queue); + const envConcurrencyKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); + const taskConcurrencyKey = this.keys.taskIdentifierCurrentConcurrencyKeyFromQueue( + message.queue, + message.taskIdentifier + ); + const projectConcurrencyKey = this.keys.projectCurrentConcurrencyKeyFromQueue( + message.queue + ); + const envQueueKey = this.keys.envQueueKeyFromQueue(message.queue); + + if (incrementAttemptCount) { + message.attempt = message.attempt + 1; + if (message.attempt >= maxAttempts) { + await this.redis.moveToDeadLetterQueue( + messageKey, + messageQueue, + concurrencyKey, + envConcurrencyKey, + projectConcurrencyKey, + envQueueKey, + taskConcurrencyKey, + "dlq", + messageId, + JSON.stringify(message.masterQueues) + ); + return false; + } + } + + const nextRetryDelay = calculateNextRetryDelay(this.retryOptions, message.attempt); + const messageScore = retryAt ?? (nextRetryDelay ? Date.now() + nextRetryDelay : Date.now()); + + this.logger.debug("Calling nackMessage", { + messageKey, + messageQueue, + masterQueues: message.masterQueues, + concurrencyKey, + envConcurrencyKey, + projectConcurrencyKey, + envQueueKey, + taskConcurrencyKey, + messageId, + messageScore, + attempt: message.attempt, + service: this.name, + }); + + await this.redis.nackMessage( + //keys + messageKey, + messageQueue, + concurrencyKey, + envConcurrencyKey, + projectConcurrencyKey, + envQueueKey, + taskConcurrencyKey, + //args + messageId, + JSON.stringify(message), + String(messageScore), + JSON.stringify(message.masterQueues) + ); + return true; + }, + { + kind: SpanKind.CONSUMER, + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "nack", + [SEMATTRS_MESSAGE_ID]: messageId, + [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", + }, + } + ); + } + + public async releaseConcurrency( + orgId: string, + messageId: string, + releaseForRun: boolean = false + ) { + return this.#trace( + "releaseConcurrency", + async (span) => { + const message = await this.#readMessage(orgId, messageId); + + if (!message) { + this.logger.log(`[${this.name}].acknowledgeMessage() message not found`, { + messageId, + service: this.name, + }); + return; + } + + span.setAttributes({ + [SemanticAttributes.QUEUE]: message.queue, + [SemanticAttributes.ORG_ID]: message.orgId, + [SemanticAttributes.RUN_ID]: messageId, + [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, + }); + + return this.redis.releaseConcurrency( + this.keys.messageKey(orgId, messageId), + message.queue, + releaseForRun ? this.keys.currentConcurrencyKeyFromQueue(message.queue) : "", + this.keys.envCurrentConcurrencyKeyFromQueue(message.queue), + this.keys.projectCurrentConcurrencyKeyFromQueue(message.queue), + this.keys.taskIdentifierCurrentConcurrencyKeyFromQueue( + message.queue, + message.taskIdentifier + ), + messageId, + JSON.stringify(message.masterQueues) + ); + }, + { + kind: SpanKind.CONSUMER, + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "releaseConcurrency", + [SEMATTRS_MESSAGE_ID]: messageId, + [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", + }, + } + ); + } + + public async reacquireConcurrency(orgId: string, messageId: string) { + return this.#trace( + "reacquireConcurrency", + async (span) => { + const message = await this.#readMessage(orgId, messageId); + + if (!message) { + this.logger.log(`[${this.name}].acknowledgeMessage() message not found`, { + messageId, + service: this.name, + }); + return; + } + + span.setAttributes({ + [SemanticAttributes.QUEUE]: message.queue, + [SemanticAttributes.ORG_ID]: message.orgId, + [SemanticAttributes.RUN_ID]: messageId, + [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, + }); + + return this.redis.reacquireConcurrency( + this.keys.messageKey(orgId, messageId), + message.queue, + this.keys.currentConcurrencyKeyFromQueue(message.queue), + this.keys.envCurrentConcurrencyKeyFromQueue(message.queue), + this.keys.projectCurrentConcurrencyKeyFromQueue(message.queue), + this.keys.taskIdentifierCurrentConcurrencyKeyFromQueue( + message.queue, + message.taskIdentifier + ), + messageId, + JSON.stringify(message.masterQueues) + ); + }, + { + kind: SpanKind.CONSUMER, + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "releaseConcurrency", + [SEMATTRS_MESSAGE_ID]: messageId, + [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", + }, + } + ); + } + + queueConcurrencyScanStream( + count: number = 100, + onEndCallback?: () => void, + onErrorCallback?: (error: Error) => void + ) { + const pattern = this.keys.queueCurrentConcurrencyScanPattern(); + + this.logger.debug("Starting queue concurrency scan stream", { + pattern, + component: "runqueue", + operation: "queueConcurrencyScanStream", + service: this.name, + count, + }); + + const redis = this.redis.duplicate(); + + const stream = redis.scanStream({ + match: pattern, + type: "set", + count, + }); + + stream.on("end", () => { + onEndCallback?.(); + redis.quit(); + }); + + stream.on("error", (error) => { + onErrorCallback?.(error); + redis.quit(); + }); + + return { stream, redis }; + } + + async quit() { + await this.subscriber.unsubscribe(); + await this.subscriber.quit(); + await this.redis.quit(); + } + + private async handleRedriveMessage(channel: string, message: string) { + try { + const { runId, orgId } = JSON.parse(message) as any; + if (typeof orgId !== "string" || typeof runId !== "string") { + this.logger.error( + "handleRedriveMessage: invalid message format: runId and orgId must be strings", + { message, channel } + ); + return; + } + + const data = await this.#readMessage(orgId, runId); + + if (!data) { + this.logger.error(`handleRedriveMessage: couldn't read message`, { orgId, runId, channel }); + return; + } + + await this.enqueueMessage({ + env: { + id: data.environmentId, + type: data.environmentType, + //this isn't used in enqueueMessage + maximumConcurrencyLimit: -1, + project: { + id: data.projectId, + }, + organization: { + id: data.orgId, + }, + }, + message: { + ...data, + attempt: 0, + }, + masterQueues: data.masterQueues, + }); + + //remove from the dlq + const result = await this.redis.zrem("dlq", runId); + + if (result === 0) { + this.logger.error(`handleRedriveMessage: couldn't remove message from dlq`, { + orgId, + runId, + channel, + }); + return; + } + + this.logger.log(`handleRedriveMessage: redrived item ${runId} from Dead Letter Queue`); + } catch (error) { + this.logger.error("Error processing redrive message", { error, message }); + } + } + + async #trace( + name: string, + fn: (span: Span) => Promise, + options?: SpanOptions & { sampleRate?: number } + ): Promise { + return this.tracer.startActiveSpan( + name, + { + ...options, + attributes: { + ...options?.attributes, + }, + }, + async (span) => { + try { + return await fn(span); + } catch (e) { + if (e instanceof Error) { + span.recordException(e); + } else { + span.recordException(new Error(String(e))); + } + + throw e; + } finally { + span.end(); + } + } + ); + } + + async #setupSubscriber() { + const channel = `${this.options.name}:redrive`; + this.subscriber.subscribe(channel, (err) => { + if (err) { + this.logger.error(`Failed to subscribe to ${channel}`, { error: err }); + } else { + this.logger.log(`Subscribed to ${channel}`); + } + }); + + this.subscriber.on("message", this.handleRedriveMessage.bind(this)); + } + + async #readMessage(orgId: string, messageId: string) { + return this.#trace( + "readMessage", + async (span) => { + const rawMessage = await this.redis.get(this.keys.messageKey(orgId, messageId)); + + if (!rawMessage) { + return; + } + + const message = OutputPayload.safeParse(JSON.parse(rawMessage)); + + if (!message.success) { + this.logger.error(`[${this.name}] Failed to parse message`, { + messageId, + error: message.error, + service: this.name, + }); + + return; + } + + return message.data; + }, + { + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "receive", + [SEMATTRS_MESSAGE_ID]: messageId, + [SEMATTRS_MESSAGING_SYSTEM]: "marqs", + [SemanticAttributes.RUN_ID]: messageId, + }, + } + ); + } + + async #getRandomQueueFromParentQueue( + parentQueue: string, + queuePriorityStrategy: RunQueuePriorityStrategy, + calculateCapacities: (queue: string) => Promise, + consumerId: string, + maxCount: number + ): Promise< + | { + queue: string; + capacities: QueueCapacities; + age: number; + size: number; + }[] + | undefined + > { + return this.#trace( + "getRandomQueueFromParentQueue", + async (span) => { + span.setAttribute("consumerId", consumerId); + + const { range } = await queuePriorityStrategy.nextCandidateSelection( + parentQueue, + consumerId + ); + + const queues = await this.#getChildQueuesWithScores(parentQueue, range, span); + span.setAttribute("queueCount", queues.length); + + const queuesWithScores = await this.#calculateQueueScores(queues, calculateCapacities); + span.setAttribute("queuesWithScoresCount", queuesWithScores.length); + + // We need to priority shuffle here to ensure all workers aren't just working on the highest priority queue + const { choices, nextRange } = queuePriorityStrategy.chooseQueues( + queuesWithScores, + parentQueue, + consumerId, + range, + maxCount + ); + + span.setAttributes({ + ...flattenAttributes(queues, "runqueue.queues"), + }); + span.setAttributes({ + ...flattenAttributes(queuesWithScores, "runqueue.queuesWithScores"), + }); + span.setAttribute("range.offset", range.offset); + span.setAttribute("range.count", range.count); + span.setAttribute("nextRange.offset", nextRange.offset); + span.setAttribute("nextRange.count", nextRange.count); + + if (this.options.verbose || nextRange.offset > 0) { + if (Array.isArray(choices)) { + this.logger.debug(`[${this.name}] getRandomQueueFromParentQueue`, { + queues, + queuesWithScores, + range, + nextRange, + queueCount: queues.length, + queuesWithScoresCount: queuesWithScores.length, + queueChoices: choices, + consumerId, + }); + } else { + this.logger.debug(`[${this.name}] getRandomQueueFromParentQueue`, { + queues, + queuesWithScores, + range, + nextRange, + queueCount: queues.length, + queuesWithScoresCount: queuesWithScores.length, + noQueueChoice: true, + consumerId, + }); + } + } + + if (Array.isArray(choices)) { + span.setAttribute("queueChoices", choices); + return queuesWithScores.filter((queue) => choices.includes(queue.queue)); + } else { + span.setAttribute("noQueueChoice", true); + return; + } + }, + { + kind: SpanKind.CONSUMER, + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "receive", + [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", + [SemanticAttributes.MASTER_QUEUES]: parentQueue, + }, + } + ); + } + + // Calculate the weights of the queues based on the age and the capacity + async #calculateQueueScores( + queues: Array<{ value: string; score: number }>, + calculateCapacities: (queue: string) => Promise + ) { + const now = Date.now(); + + const queueScores = await Promise.all( + queues.map(async (queue) => { + return { + queue: queue.value, + capacities: await calculateCapacities(queue.value), + age: now - queue.score, + size: await this.redis.zcard(queue.value), + }; + }) + ); + + return queueScores; + } + + async #calculateMessageQueueCapacities(queue: string, options?: { checkForDisabled?: boolean }) { + return await this.#callCalculateMessageCapacities({ + currentConcurrencyKey: this.keys.currentConcurrencyKeyFromQueue(queue), + currentEnvConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(queue), + concurrencyLimitKey: this.keys.concurrencyLimitKeyFromQueue(queue), + envConcurrencyLimitKey: this.keys.envConcurrencyLimitKeyFromQueue(queue), + disabledConcurrencyLimitKey: options?.checkForDisabled + ? this.keys.disabledConcurrencyLimitKeyFromQueue(queue) + : undefined, + }); + } + + async #getChildQueuesWithScores( + key: string, + range: QueueRange, + span?: Span + ): Promise> { + const valuesWithScores = await this.redis.zrangebyscore( + key, + "-inf", + Date.now(), + "WITHSCORES", + "LIMIT", + range.offset, + range.count + ); + + span?.setAttribute("zrangebyscore.valuesWithScores.rawLength", valuesWithScores.length); + span?.setAttributes({ + ...flattenAttributes(valuesWithScores, "zrangebyscore.valuesWithScores.rawValues"), + }); + + const result: Array<{ value: string; score: number }> = []; + + for (let i = 0; i < valuesWithScores.length; i += 2) { + result.push({ + value: valuesWithScores[i], + score: Number(valuesWithScores[i + 1]), + }); + } + + return result; + } + + async #callEnqueueMessage(message: OutputPayload, masterQueues: string[]) { + const concurrencyKey = this.keys.currentConcurrencyKeyFromQueue(message.queue); + const envConcurrencyKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); + const taskConcurrencyKey = this.keys.taskIdentifierCurrentConcurrencyKeyFromQueue( + message.queue, + message.taskIdentifier + ); + const projectConcurrencyKey = this.keys.projectCurrentConcurrencyKeyFromQueue(message.queue); + + this.logger.debug("Calling enqueueMessage", { + messagePayload: message, + concurrencyKey, + envConcurrencyKey, + masterQueues, + service: this.name, + }); + + return this.redis.enqueueMessage( + message.queue, + this.keys.messageKey(message.orgId, message.runId), + concurrencyKey, + envConcurrencyKey, + taskConcurrencyKey, + projectConcurrencyKey, + this.keys.envQueueKeyFromQueue(message.queue), + message.queue, + message.runId, + JSON.stringify(message), + String(message.timestamp), + JSON.stringify(masterQueues) + ); + } + + async #callDequeueMessage({ + messageQueue, + concurrencyLimitKey, + envConcurrencyLimitKey, + currentConcurrencyKey, + envCurrentConcurrencyKey, + projectCurrentConcurrencyKey, + messageKeyPrefix, + envQueueKey, + taskCurrentConcurrentKeyPrefix, + }: { + messageQueue: string; + concurrencyLimitKey: string; + envConcurrencyLimitKey: string; + currentConcurrencyKey: string; + envCurrentConcurrencyKey: string; + projectCurrentConcurrencyKey: string; + messageKeyPrefix: string; + envQueueKey: string; + taskCurrentConcurrentKeyPrefix: string; + }): Promise { + const result = await this.redis.dequeueMessage( + //keys + messageQueue, + concurrencyLimitKey, + envConcurrencyLimitKey, + currentConcurrencyKey, + envCurrentConcurrencyKey, + projectCurrentConcurrencyKey, + messageKeyPrefix, + envQueueKey, + taskCurrentConcurrentKeyPrefix, + //args + messageQueue, + String(Date.now()), + String(this.options.defaultEnvConcurrency) + ); + + if (!result) { + return; + } + + this.logger.debug("Dequeue message result", { + result, + service: this.name, + }); + + if (result.length !== 3) { + this.logger.error("Invalid dequeue message result", { + result, + service: this.name, + }); + return; + } + + const [messageId, messageScore, rawMessage] = result; + + //read message + const parsedMessage = OutputPayload.safeParse(JSON.parse(rawMessage)); + if (!parsedMessage.success) { + this.logger.error(`[${this.name}] Failed to parse message`, { + messageId, + error: parsedMessage.error, + service: this.name, + }); + + return; + } + + const message = parsedMessage.data; + + return { + messageId, + messageScore, + message, + }; + } + + async #callAcknowledgeMessage({ + messageId, + masterQueues, + messageKey, + messageQueue, + concurrencyKey, + envConcurrencyKey, + taskConcurrencyKey, + envQueueKey, + projectConcurrencyKey, + }: { + masterQueues: string[]; + messageKey: string; + messageQueue: string; + concurrencyKey: string; + envConcurrencyKey: string; + taskConcurrencyKey: string; + envQueueKey: string; + projectConcurrencyKey: string; + messageId: string; + }) { + this.logger.debug("Calling acknowledgeMessage", { + messageKey, + messageQueue, + concurrencyKey, + envConcurrencyKey, + projectConcurrencyKey, + envQueueKey, + taskConcurrencyKey, + messageId, + masterQueues, + service: this.name, + }); + + return this.redis.acknowledgeMessage( + messageKey, + messageQueue, + concurrencyKey, + envConcurrencyKey, + projectConcurrencyKey, + envQueueKey, + taskConcurrencyKey, + messageId, + JSON.stringify(masterQueues) + ); + } + + async #callCalculateMessageCapacities({ + currentConcurrencyKey, + currentEnvConcurrencyKey, + concurrencyLimitKey, + envConcurrencyLimitKey, + disabledConcurrencyLimitKey, + }: { + currentConcurrencyKey: string; + currentEnvConcurrencyKey: string; + concurrencyLimitKey: string; + envConcurrencyLimitKey: string; + disabledConcurrencyLimitKey: string | undefined; + }): Promise { + const capacities = disabledConcurrencyLimitKey + ? await this.redis.calculateMessageQueueCapacitiesWithDisabling( + currentConcurrencyKey, + currentEnvConcurrencyKey, + concurrencyLimitKey, + envConcurrencyLimitKey, + disabledConcurrencyLimitKey, + String(this.options.defaultEnvConcurrency) + ) + : await this.redis.calculateMessageQueueCapacities( + currentConcurrencyKey, + currentEnvConcurrencyKey, + concurrencyLimitKey, + envConcurrencyLimitKey, + String(this.options.defaultEnvConcurrency) + ); + + const queueCurrent = Number(capacities[0]); + const envLimit = Number(capacities[3]); + const isOrgEnabled = Boolean(capacities[4]); + const queueLimit = capacities[1] + ? Number(capacities[1]) + : Math.min(envLimit, isOrgEnabled ? Infinity : 0); + const envCurrent = Number(capacities[2]); + + return { + queue: { current: queueCurrent, limit: queueLimit }, + env: { current: envCurrent, limit: envLimit }, + }; + } + + #callUpdateGlobalConcurrencyLimits({ + envConcurrencyLimitKey, + envConcurrencyLimit, + }: { + envConcurrencyLimitKey: string; + envConcurrencyLimit: number; + }) { + return this.redis.updateGlobalConcurrencyLimits( + envConcurrencyLimitKey, + String(envConcurrencyLimit) + ); + } + + #registerCommands() { + this.redis.defineCommand("enqueueMessage", { + numberOfKeys: 7, + lua: ` +local queue = KEYS[1] +local messageKey = KEYS[2] +local concurrencyKey = KEYS[3] +local envConcurrencyKey = KEYS[4] +local taskConcurrencyKey = KEYS[5] +local projectConcurrencyKey = KEYS[6] +local envQueueKey = KEYS[7] + +local queueName = ARGV[1] +local messageId = ARGV[2] +local messageData = ARGV[3] +local messageScore = ARGV[4] +local parentQueues = cjson.decode(ARGV[5]) + +-- Write the message to the message key +redis.call('SET', messageKey, messageData) + +-- Add the message to the queue +redis.call('ZADD', queue, messageScore, messageId) + +-- Add the message to the env queue +redis.call('ZADD', envQueueKey, messageScore, messageId) + +-- Rebalance the parent queues +for _, parentQueue in ipairs(parentQueues) do + local earliestMessage = redis.call('ZRANGE', queue, 0, 0, 'WITHSCORES') + if #earliestMessage == 0 then + redis.call('ZREM', parentQueue, queueName) + else + redis.call('ZADD', parentQueue, earliestMessage[2], queueName) + end +end + +-- Update the concurrency keys +redis.call('SREM', concurrencyKey, messageId) +redis.call('SREM', envConcurrencyKey, messageId) +redis.call('SREM', taskConcurrencyKey, messageId) +redis.call('SREM', projectConcurrencyKey, messageId) + `, + }); + + this.redis.defineCommand("dequeueMessage", { + numberOfKeys: 9, + lua: ` +local childQueue = KEYS[1] +local concurrencyLimitKey = KEYS[2] +local envConcurrencyLimitKey = KEYS[3] +local currentConcurrencyKey = KEYS[4] +local envCurrentConcurrencyKey = KEYS[5] +local projectConcurrencyKey = KEYS[6] +local messageKeyPrefix = KEYS[7] +local envQueueKey = KEYS[8] +local taskCurrentConcurrentKeyPrefix = KEYS[9] + +local childQueueName = ARGV[1] +local currentTime = tonumber(ARGV[2]) +local defaultEnvConcurrencyLimit = ARGV[3] + +-- Check current env concurrency against the limit +local envCurrentConcurrency = tonumber(redis.call('SCARD', envCurrentConcurrencyKey) or '0') +local envConcurrencyLimit = tonumber(redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit) + +if envCurrentConcurrency >= envConcurrencyLimit then + return nil +end + +-- Check current queue concurrency against the limit +local currentConcurrency = tonumber(redis.call('SCARD', currentConcurrencyKey) or '0') +local concurrencyLimit = tonumber(redis.call('GET', concurrencyLimitKey) or '1000000') + +-- Check condition only if concurrencyLimit exists +if currentConcurrency >= concurrencyLimit then + return nil +end + +-- Attempt to dequeue the next message +local messages = redis.call('ZRANGEBYSCORE', childQueue, '-inf', currentTime, 'WITHSCORES', 'LIMIT', 0, 1) + +if #messages == 0 then + return nil +end + +local messageId = messages[1] +local messageScore = tonumber(messages[2]) + +-- Get the message payload +local messageKey = messageKeyPrefix .. messageId +local messagePayload = redis.call('GET', messageKey) +local decodedPayload = cjson.decode(messagePayload); + +-- Extract taskIdentifier +local taskIdentifier = decodedPayload.taskIdentifier + +-- Perform SADD with taskIdentifier and messageId +local taskConcurrencyKey = taskCurrentConcurrentKeyPrefix .. taskIdentifier + +-- Update concurrency +redis.call('ZREM', childQueue, messageId) +redis.call('ZREM', envQueueKey, messageId) +redis.call('SADD', currentConcurrencyKey, messageId) +redis.call('SADD', envCurrentConcurrencyKey, messageId) +redis.call('SADD', projectConcurrencyKey, messageId) +redis.call('SADD', taskConcurrencyKey, messageId) + +-- Rebalance the parent queues +for _, parentQueue in ipairs(decodedPayload.masterQueues) do + local earliestMessage = redis.call('ZRANGE', childQueue, 0, 0, 'WITHSCORES') + if #earliestMessage == 0 then + redis.call('ZREM', parentQueue, childQueue) + else + redis.call('ZADD', parentQueue, earliestMessage[2], childQueue) + end +end + +return {messageId, messageScore, messagePayload} -- Return message details + `, + }); + + this.redis.defineCommand("acknowledgeMessage", { + numberOfKeys: 7, + lua: ` +-- Keys: +local messageKey = KEYS[1] +local messageQueue = KEYS[2] +local concurrencyKey = KEYS[3] +local envCurrentConcurrencyKey = KEYS[4] +local projectCurrentConcurrencyKey = KEYS[5] +local envQueueKey = KEYS[6] +local taskCurrentConcurrencyKey = KEYS[7] + +-- Args: +local messageId = ARGV[1] +local parentQueues = cjson.decode(ARGV[2]) + +-- Remove the message from the message key +redis.call('DEL', messageKey) + +-- Remove the message from the queue +redis.call('ZREM', messageQueue, messageId) +redis.call('ZREM', envQueueKey, messageId) + +-- Rebalance the parent queues +for _, parentQueue in ipairs(parentQueues) do + local earliestMessage = redis.call('ZRANGE', messageQueue, 0, 0, 'WITHSCORES') + if #earliestMessage == 0 then + redis.call('ZREM', parentQueue, messageQueue) + else + redis.call('ZADD', parentQueue, earliestMessage[2], messageQueue) + end +end + +-- Update the concurrency keys +redis.call('SREM', concurrencyKey, messageId) +redis.call('SREM', envCurrentConcurrencyKey, messageId) +redis.call('SREM', projectCurrentConcurrencyKey, messageId) +redis.call('SREM', taskCurrentConcurrencyKey, messageId) +`, + }); + + this.redis.defineCommand("nackMessage", { + numberOfKeys: 7, + lua: ` +-- Keys: +local messageKey = KEYS[1] +local messageQueueKey = KEYS[2] +local concurrencyKey = KEYS[3] +local envConcurrencyKey = KEYS[4] +local projectConcurrencyKey = KEYS[5] +local envQueueKey = KEYS[6] +local taskConcurrencyKey = KEYS[7] + +-- Args: +local messageId = ARGV[1] +local messageData = ARGV[2] +local messageScore = tonumber(ARGV[3]) +local parentQueues = cjson.decode(ARGV[4]) + +-- Update the message data +redis.call('SET', messageKey, messageData) + +-- Update the concurrency keys +redis.call('SREM', concurrencyKey, messageId) +redis.call('SREM', envConcurrencyKey, messageId) +redis.call('SREM', projectConcurrencyKey, messageId) +redis.call('SREM', taskConcurrencyKey, messageId) + +-- Enqueue the message into the queue +redis.call('ZADD', messageQueueKey, messageScore, messageId) +redis.call('ZADD', envQueueKey, messageScore, messageId) + +-- Rebalance the parent queues +for _, parentQueue in ipairs(parentQueues) do + local earliestMessage = redis.call('ZRANGE', messageQueueKey, 0, 0, 'WITHSCORES') + if #earliestMessage == 0 then + redis.call('ZREM', parentQueue, messageQueueKey) + else + redis.call('ZADD', parentQueue, earliestMessage[2], messageQueueKey) + end +end +`, + }); + + this.redis.defineCommand("moveToDeadLetterQueue", { + numberOfKeys: 8, + lua: ` +-- Keys: +local messageKey = KEYS[1] +local messageQueue = KEYS[2] +local concurrencyKey = KEYS[3] +local envCurrentConcurrencyKey = KEYS[4] +local projectCurrentConcurrencyKey = KEYS[5] +local envQueueKey = KEYS[6] +local taskCurrentConcurrencyKey = KEYS[7] +local deadLetterQueueKey = KEYS[8] + +-- Args: +local messageId = ARGV[1] +local parentQueues = cjson.decode(ARGV[2]) + +-- Remove the message from the queue +redis.call('ZREM', messageQueue, messageId) +redis.call('ZREM', envQueueKey, messageId) + +-- Rebalance the parent queues +for _, parentQueue in ipairs(parentQueues) do + local earliestMessage = redis.call('ZRANGE', messageQueue, 0, 0, 'WITHSCORES') + if #earliestMessage == 0 then + redis.call('ZREM', parentQueue, messageQueue) + else + redis.call('ZADD', parentQueue, earliestMessage[2], messageQueue) + end +end + +-- Add the message to the dead letter queue +redis.call('ZADD', deadLetterQueueKey, tonumber(redis.call('TIME')[1]), messageId) + +-- Update the concurrency keys +redis.call('SREM', concurrencyKey, messageId) +redis.call('SREM', envCurrentConcurrencyKey, messageId) +redis.call('SREM', projectCurrentConcurrencyKey, messageId) +redis.call('SREM', taskCurrentConcurrencyKey, messageId) +`, + }); + + this.redis.defineCommand("releaseConcurrency", { + numberOfKeys: 6, + lua: ` +-- Keys: +local messageKey = KEYS[1] +local messageQueue = KEYS[2] +local concurrencyKey = KEYS[3] +local envCurrentConcurrencyKey = KEYS[4] +local projectCurrentConcurrencyKey = KEYS[5] +local taskCurrentConcurrencyKey = KEYS[6] + +-- Args: +local messageId = ARGV[1] + +-- Update the concurrency keys +if concurrencyKey ~= "" then + redis.call('SREM', concurrencyKey, messageId) +end +redis.call('SREM', envCurrentConcurrencyKey, messageId) +redis.call('SREM', projectCurrentConcurrencyKey, messageId) +redis.call('SREM', taskCurrentConcurrencyKey, messageId) +`, + }); + + this.redis.defineCommand("reacquireConcurrency", { + numberOfKeys: 6, + lua: ` +-- Keys: +local messageKey = KEYS[1] +local messageQueue = KEYS[2] +local concurrencyKey = KEYS[3] +local envCurrentConcurrencyKey = KEYS[4] +local projectCurrentConcurrencyKey = KEYS[5] +local taskCurrentConcurrencyKey = KEYS[6] + +-- Args: +local messageId = ARGV[1] + +-- Update the concurrency keys +redis.call('SADD', concurrencyKey, messageId) +redis.call('SADD', envCurrentConcurrencyKey, messageId) +redis.call('SADD', projectCurrentConcurrencyKey, messageId) +redis.call('SADD', taskCurrentConcurrencyKey, messageId) +`, + }); + + this.redis.defineCommand("calculateMessageQueueCapacitiesWithDisabling", { + numberOfKeys: 5, + lua: ` +-- Keys +local currentConcurrencyKey = KEYS[1] +local currentEnvConcurrencyKey = KEYS[2] +local concurrencyLimitKey = KEYS[3] +local envConcurrencyLimitKey = KEYS[4] +local disabledConcurrencyLimitKey = KEYS[5] + +-- Args +local defaultEnvConcurrencyLimit = tonumber(ARGV[1]) + +-- Check if disabledConcurrencyLimitKey exists +local orgIsEnabled +if redis.call('EXISTS', disabledConcurrencyLimitKey) == 1 then + orgIsEnabled = false +else + orgIsEnabled = true +end + +local currentEnvConcurrency = tonumber(redis.call('SCARD', currentEnvConcurrencyKey) or '0') +local envConcurrencyLimit = tonumber(redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit) + +local currentConcurrency = tonumber(redis.call('SCARD', currentConcurrencyKey) or '0') +local concurrencyLimit = redis.call('GET', concurrencyLimitKey) + +-- Return current capacity and concurrency limits for the queue, env, org +return { currentConcurrency, concurrencyLimit, currentEnvConcurrency, envConcurrencyLimit, orgIsEnabled } + `, + }); + + this.redis.defineCommand("calculateMessageQueueCapacities", { + numberOfKeys: 4, + lua: ` +-- Keys: +local currentConcurrencyKey = KEYS[1] +local currentEnvConcurrencyKey = KEYS[2] +local concurrencyLimitKey = KEYS[3] +local envConcurrencyLimitKey = KEYS[4] + +-- Args +local defaultEnvConcurrencyLimit = tonumber(ARGV[1]) + +local currentEnvConcurrency = tonumber(redis.call('SCARD', currentEnvConcurrencyKey) or '0') +local envConcurrencyLimit = tonumber(redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit) + +local currentConcurrency = tonumber(redis.call('SCARD', currentConcurrencyKey) or '0') +local concurrencyLimit = redis.call('GET', concurrencyLimitKey) + +-- Return current capacity and concurrency limits for the queue, env, org +return { currentConcurrency, concurrencyLimit, currentEnvConcurrency, envConcurrencyLimit, true } + `, + }); + + this.redis.defineCommand("updateGlobalConcurrencyLimits", { + numberOfKeys: 1, + lua: ` +-- Keys: envConcurrencyLimitKey +local envConcurrencyLimitKey = KEYS[1] + +-- Args: envConcurrencyLimit +local envConcurrencyLimit = ARGV[1] + +redis.call('SET', envConcurrencyLimitKey, envConcurrencyLimit) + `, + }); + } +} + +declare module "ioredis" { + interface RedisCommander { + enqueueMessage( + //keys + queue: string, + messageKey: string, + concurrencyKey: string, + envConcurrencyKey: string, + taskConcurrencyKey: string, + projectConcurrencyKey: string, + envQueueKey: string, + //args + queueName: string, + messageId: string, + messageData: string, + messageScore: string, + parentQueues: string, + callback?: Callback + ): Result; + + dequeueMessage( + //keys + childQueue: string, + concurrencyLimitKey: string, + envConcurrencyLimitKey: string, + currentConcurrencyKey: string, + envConcurrencyKey: string, + projectConcurrencyKey: string, + messageKeyPrefix: string, + envQueueKey: string, + taskCurrentConcurrentKeyPrefix: string, + //args + childQueueName: string, + currentTime: string, + defaultEnvConcurrencyLimit: string, + callback?: Callback<[string, string]> + ): Result<[string, string, string] | null, Context>; + + acknowledgeMessage( + messageKey: string, + messageQueue: string, + concurrencyKey: string, + envConcurrencyKey: string, + projectConcurrencyKey: string, + envQueueKey: string, + taskConcurrencyKey: string, + messageId: string, + masterQueues: string, + callback?: Callback + ): Result; + + nackMessage( + messageKey: string, + messageQueue: string, + concurrencyKey: string, + envConcurrencyKey: string, + projectConcurrencyKey: string, + envQueueKey: string, + taskConcurrencyKey: string, + messageId: string, + messageData: string, + messageScore: string, + masterQueues: string, + callback?: Callback + ): Result; + + moveToDeadLetterQueue( + messageKey: string, + messageQueue: string, + concurrencyKey: string, + envConcurrencyKey: string, + projectConcurrencyKey: string, + envQueueKey: string, + taskConcurrencyKey: string, + deadLetterQueueKey: string, + messageId: string, + masterQueues: string, + callback?: Callback + ): Result; + + releaseConcurrency( + messageKey: string, + messageQueue: string, + concurrencyKey: string, + envConcurrencyKey: string, + projectConcurrencyKey: string, + taskConcurrencyKey: string, + messageId: string, + masterQueues: string, + callback?: Callback + ): Result; + + reacquireConcurrency( + messageKey: string, + messageQueue: string, + concurrencyKey: string, + envConcurrencyKey: string, + projectConcurrencyKey: string, + taskConcurrencyKey: string, + messageId: string, + masterQueues: string, + callback?: Callback + ): Result; + + calculateMessageQueueCapacities( + currentConcurrencyKey: string, + currentEnvConcurrencyKey: string, + concurrencyLimitKey: string, + envConcurrencyLimitKey: string, + defaultEnvConcurrencyLimit: string, + callback?: Callback + ): Result<[number, number, number, number, boolean], Context>; + + calculateMessageQueueCapacitiesWithDisabling( + currentConcurrencyKey: string, + currentEnvConcurrencyKey: string, + concurrencyLimitKey: string, + envConcurrencyLimitKey: string, + disabledConcurrencyLimitKey: string, + defaultEnvConcurrencyLimit: string, + callback?: Callback + ): Result<[number, number, number, number, boolean], Context>; + + updateGlobalConcurrencyLimits( + envConcurrencyLimitKey: string, + envConcurrencyLimit: string, + callback?: Callback + ): Result; + } +} diff --git a/internal-packages/run-engine/src/run-queue/keyProducer.test.ts b/internal-packages/run-engine/src/run-queue/keyProducer.test.ts new file mode 100644 index 0000000000..886d695f59 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/keyProducer.test.ts @@ -0,0 +1,361 @@ +import { describe } from "node:test"; +import { expect, it } from "vitest"; +import { RunQueueShortKeyProducer } from "./keyProducer.js"; + +describe("KeyProducer", () => { + it("sharedQueueScanPattern", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const pattern = keyProducer.masterQueueScanPattern("main"); + expect(pattern).toBe("test:*main"); + }); + + it("queueCurrentConcurrencyScanPattern", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const pattern = keyProducer.queueCurrentConcurrencyScanPattern(); + expect(pattern).toBe("test:{org:*}:proj:*:env:*:queue:*:currentConcurrency"); + }); + + it("stripKeyPrefix", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.stripKeyPrefix("test:abc"); + expect(key).toBe("abc"); + }); + + it("queueConcurrencyLimitKey", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.queueConcurrencyLimitKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:concurrency"); + }); + + it("envConcurrencyLimitKey", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.envConcurrencyLimitKey({ + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }); + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:concurrency"); + }); + + it("queueKey (no concurrency)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name"); + }); + + it("queueKey (w concurrency)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name", + "c1234" + ); + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:ck:c1234"); + }); + + it("concurrencyLimitKeyFromQueue (w concurrency)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name", + "c1234" + ); + const key = keyProducer.concurrencyLimitKeyFromQueue(queueKey); + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:concurrency"); + }); + + it("concurrencyLimitKeyFromQueue (no concurrency)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + const key = keyProducer.concurrencyLimitKeyFromQueue(queueKey); + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:concurrency"); + }); + + it("currentConcurrencyKeyFromQueue (w concurrency)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name", + "c1234" + ); + const key = keyProducer.currentConcurrencyKeyFromQueue(queueKey); + expect(key).toBe( + "{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:ck:c1234:currentConcurrency" + ); + }); + + it("currentConcurrencyKeyFromQueue (no concurrency)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + const key = keyProducer.currentConcurrencyKeyFromQueue(queueKey); + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:currentConcurrency"); + }); + + it("currentConcurrencyKey (w concurrency)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.currentConcurrencyKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name", + "c1234" + ); + expect(key).toBe( + "{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:ck:c1234:currentConcurrency" + ); + }); + + it("currentConcurrencyKey (no concurrency)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.currentConcurrencyKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:currentConcurrency"); + }); + + it("taskIdentifierCurrentConcurrencyKeyPrefixFromQueue", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + const key = keyProducer.taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queueKey); + expect(key).toBe("{org:o1234}:proj:p1234:task:"); + }); + + it("taskIdentifierCurrentConcurrencyKeyFromQueue", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + const key = keyProducer.taskIdentifierCurrentConcurrencyKeyFromQueue(queueKey, "task-name"); + expect(key).toBe("{org:o1234}:proj:p1234:task:task-name"); + }); + + it("taskIdentifierCurrentConcurrencyKey", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.taskIdentifierCurrentConcurrencyKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task-name" + ); + expect(key).toBe("{org:o1234}:proj:p1234:task:task-name"); + }); + + it("projectCurrentConcurrencyKey", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.projectCurrentConcurrencyKey({ + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }); + expect(key).toBe("{org:o1234}:proj:p1234:currentConcurrency"); + }); + + it("projectCurrentConcurrencyKeyFromQueue", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.projectCurrentConcurrencyKeyFromQueue( + "{org:o1234}:proj:p1234:currentConcurrency" + ); + expect(key).toBe("{org:o1234}:proj:p1234:currentConcurrency"); + }); + + it("disabledConcurrencyLimitKeyFromQueue", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + const key = keyProducer.disabledConcurrencyLimitKeyFromQueue(queueKey); + expect(key).toBe("{org:o1234}:disabledConcurrency"); + }); + + it("envConcurrencyLimitKeyFromQueue", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + const key = keyProducer.envConcurrencyLimitKeyFromQueue(queueKey); + expect(key).toBe("{org:o1234}:env:e1234:concurrency"); + }); + + it("envCurrentConcurrencyKeyFromQueue", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + const key = keyProducer.envCurrentConcurrencyKeyFromQueue(queueKey); + expect(key).toBe("{org:o1234}:env:e1234:currentConcurrency"); + }); + + it("envCurrentConcurrencyKey", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.envCurrentConcurrencyKey({ + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }); + expect(key).toBe("{org:o1234}:env:e1234:currentConcurrency"); + }); + + it("messageKey", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.messageKey("o1234", "m1234"); + expect(key).toBe("{org:o1234}:message:m1234"); + }); + + it("extractComponentsFromQueue (no concurrencyKey)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + const components = keyProducer.extractComponentsFromQueue(queueKey); + expect(components).toEqual({ + orgId: "o1234", + projectId: "p1234", + envId: "e1234", + queue: "task/task-name", + concurrencyKey: undefined, + }); + }); + + it("extractComponentsFromQueue (w concurrencyKey)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name", + "c1234" + ); + const components = keyProducer.extractComponentsFromQueue(queueKey); + expect(components).toEqual({ + orgId: "o1234", + projectId: "p1234", + envId: "e1234", + queue: "task/task-name", + concurrencyKey: "c1234", + }); + }); +}); diff --git a/internal-packages/run-engine/src/run-queue/keyProducer.ts b/internal-packages/run-engine/src/run-queue/keyProducer.ts new file mode 100644 index 0000000000..1ba42f7f0f --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/keyProducer.ts @@ -0,0 +1,204 @@ +import { MinimalAuthenticatedEnvironment } from "../shared/index.js"; +import { RunQueueKeyProducer } from "./types.js"; + +const constants = { + CURRENT_CONCURRENCY_PART: "currentConcurrency", + CONCURRENCY_LIMIT_PART: "concurrency", + DISABLED_CONCURRENCY_LIMIT_PART: "disabledConcurrency", + ENV_PART: "env", + ORG_PART: "org", + PROJECT_PART: "proj", + QUEUE_PART: "queue", + CONCURRENCY_KEY_PART: "ck", + TASK_PART: "task", + MESSAGE_PART: "message", +} as const; + +export class RunQueueShortKeyProducer implements RunQueueKeyProducer { + constructor(private _prefix: string) {} + + masterQueueScanPattern(masterQueue: string) { + return `${this._prefix}*${masterQueue}`; + } + + queueCurrentConcurrencyScanPattern() { + return `${this._prefix}{${constants.ORG_PART}:*}:${constants.PROJECT_PART}:*:${constants.ENV_PART}:*:${constants.QUEUE_PART}:*:${constants.CURRENT_CONCURRENCY_PART}`; + } + + stripKeyPrefix(key: string): string { + if (key.startsWith(this._prefix)) { + return key.slice(this._prefix.length); + } + + return key; + } + + queueConcurrencyLimitKey(env: MinimalAuthenticatedEnvironment, queue: string) { + return [this.queueKey(env, queue), constants.CONCURRENCY_LIMIT_PART].join(":"); + } + + envConcurrencyLimitKey(env: MinimalAuthenticatedEnvironment) { + return [ + this.orgKeySection(env.organization.id), + this.projKeySection(env.project.id), + this.envKeySection(env.id), + constants.CONCURRENCY_LIMIT_PART, + ].join(":"); + } + + queueKey(env: MinimalAuthenticatedEnvironment, queue: string, concurrencyKey?: string) { + return [ + this.orgKeySection(env.organization.id), + this.projKeySection(env.project.id), + this.envKeySection(env.id), + this.queueSection(queue), + ] + .concat(concurrencyKey ? this.concurrencyKeySection(concurrencyKey) : []) + .join(":"); + } + + envQueueKey(env: MinimalAuthenticatedEnvironment) { + return [this.orgKeySection(env.organization.id), this.envKeySection(env.id)].join(":"); + } + + envQueueKeyFromQueue(queue: string) { + const { orgId, envId } = this.extractComponentsFromQueue(queue); + return [this.orgKeySection(orgId), this.envKeySection(envId)].join(":"); + } + + concurrencyLimitKeyFromQueue(queue: string) { + const concurrencyQueueName = queue.replace(/:ck:.+$/, ""); + return `${concurrencyQueueName}:${constants.CONCURRENCY_LIMIT_PART}`; + } + + currentConcurrencyKeyFromQueue(queue: string) { + return `${queue}:${constants.CURRENT_CONCURRENCY_PART}`; + } + + currentConcurrencyKey( + env: MinimalAuthenticatedEnvironment, + queue: string, + concurrencyKey?: string + ): string { + return [this.queueKey(env, queue, concurrencyKey), constants.CURRENT_CONCURRENCY_PART].join( + ":" + ); + } + + disabledConcurrencyLimitKeyFromQueue(queue: string) { + const { orgId } = this.extractComponentsFromQueue(queue); + return `{${constants.ORG_PART}:${orgId}}:${constants.DISABLED_CONCURRENCY_LIMIT_PART}`; + } + + envConcurrencyLimitKeyFromQueue(queue: string) { + const { orgId, envId } = this.extractComponentsFromQueue(queue); + return `{${constants.ORG_PART}:${orgId}}:${constants.ENV_PART}:${envId}:${constants.CONCURRENCY_LIMIT_PART}`; + } + + envCurrentConcurrencyKeyFromQueue(queue: string) { + const { orgId, envId } = this.extractComponentsFromQueue(queue); + return `{${constants.ORG_PART}:${orgId}}:${constants.ENV_PART}:${envId}:${constants.CURRENT_CONCURRENCY_PART}`; + } + + envCurrentConcurrencyKey(env: MinimalAuthenticatedEnvironment): string { + return [ + this.orgKeySection(env.organization.id), + this.envKeySection(env.id), + constants.CURRENT_CONCURRENCY_PART, + ].join(":"); + } + + taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queue: string) { + const { orgId, projectId } = this.extractComponentsFromQueue(queue); + + return `${[this.orgKeySection(orgId), this.projKeySection(projectId), constants.TASK_PART] + .filter(Boolean) + .join(":")}:`; + } + + taskIdentifierCurrentConcurrencyKeyFromQueue(queue: string, taskIdentifier: string) { + return `${this.taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queue)}${taskIdentifier}`; + } + + taskIdentifierCurrentConcurrencyKey( + env: MinimalAuthenticatedEnvironment, + taskIdentifier: string + ): string { + return [ + this.orgKeySection(env.organization.id), + this.projKeySection(env.project.id), + constants.TASK_PART, + taskIdentifier, + ].join(":"); + } + + projectCurrentConcurrencyKey(env: MinimalAuthenticatedEnvironment): string { + return [ + this.orgKeySection(env.organization.id), + this.projKeySection(env.project.id), + constants.CURRENT_CONCURRENCY_PART, + ].join(":"); + } + + projectCurrentConcurrencyKeyFromQueue(queue: string): string { + const { orgId, projectId } = this.extractComponentsFromQueue(queue); + return `${this.orgKeySection(orgId)}:${this.projKeySection(projectId)}:${ + constants.CURRENT_CONCURRENCY_PART + }`; + } + + messageKeyPrefixFromQueue(queue: string) { + const { orgId } = this.extractComponentsFromQueue(queue); + return `${this.orgKeySection(orgId)}:${constants.MESSAGE_PART}:`; + } + + messageKey(orgId: string, messageId: string) { + return [this.orgKeySection(orgId), `${constants.MESSAGE_PART}:${messageId}`] + .filter(Boolean) + .join(":"); + } + + extractComponentsFromQueue(queue: string) { + const parts = this.normalizeQueue(queue).split(":"); + return { + orgId: parts[1].replace("{", "").replace("}", ""), + projectId: parts[3], + envId: parts[5], + queue: parts[7], + concurrencyKey: parts.at(9), + }; + } + + private envKeySection(envId: string) { + return `${constants.ENV_PART}:${envId}`; + } + + private projKeySection(projId: string) { + return `${constants.PROJECT_PART}:${projId}`; + } + + private orgKeySection(orgId: string) { + return `{${constants.ORG_PART}:${orgId}}`; + } + + private queueSection(queue: string) { + return `${constants.QUEUE_PART}:${queue}`; + } + + private concurrencyKeySection(concurrencyKey: string) { + return `${constants.CONCURRENCY_KEY_PART}:${concurrencyKey}`; + } + + private taskIdentifierSection(taskIdentifier: string) { + return `${constants.TASK_PART}:${taskIdentifier}`; + } + + // This removes the leading prefix from the queue name if it exists + private normalizeQueue(queue: string) { + if (queue.startsWith(this._prefix)) { + return queue.slice(this._prefix.length); + } + + return queue; + } +} diff --git a/internal-packages/run-engine/src/run-queue/simpleWeightedPriorityStrategy.ts b/internal-packages/run-engine/src/run-queue/simpleWeightedPriorityStrategy.ts new file mode 100644 index 0000000000..04eb68c7d7 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/simpleWeightedPriorityStrategy.ts @@ -0,0 +1,130 @@ +import { + RunQueuePriorityStrategy, + PriorityStrategyChoice, + QueueRange, + QueueWithScores, +} from "./types.js"; + +export type SimpleWeightedChoiceStrategyOptions = { + queueSelectionCount: number; + randomSeed?: string; + excludeEnvCapacity?: boolean; +}; + +export class SimpleWeightedChoiceStrategy implements RunQueuePriorityStrategy { + private _nextRangesByParentQueue: Map = new Map(); + + constructor(private options: SimpleWeightedChoiceStrategyOptions) {} + + private nextRangeForParentQueue(parentQueue: string, consumerId: string): QueueRange { + return ( + this._nextRangesByParentQueue.get(`${consumerId}:${parentQueue}`) ?? { + offset: 0, + count: this.options.queueSelectionCount, + } + ); + } + + chooseQueues( + queues: QueueWithScores[], + parentQueue: string, + consumerId: string, + previousRange: QueueRange, + maxCount: number + ): { choices: PriorityStrategyChoice; nextRange: QueueRange } { + const filteredQueues = filterQueuesAtCapacity(queues); + + if (queues.length === this.options.queueSelectionCount) { + const nextRange: QueueRange = { + offset: previousRange.offset + this.options.queueSelectionCount, + count: this.options.queueSelectionCount, + }; + + // If all queues are at capacity, and we were passed the max number of queues, then we will slide the window "to the right" + this._nextRangesByParentQueue.set(`${consumerId}:${parentQueue}`, nextRange); + } else { + this._nextRangesByParentQueue.delete(`${consumerId}:${parentQueue}`); + } + + if (filteredQueues.length === 0) { + return { + choices: { abort: true }, + nextRange: this.nextRangeForParentQueue(parentQueue, consumerId), + }; + } + + const queueWeights = this.#calculateQueueWeights(filteredQueues); + + const choices = []; + for (let i = 0; i < maxCount; i++) { + const chosenIndex = weightedRandomIndex(queueWeights); + + const choice = queueWeights.at(chosenIndex)?.queue; + if (choice) { + queueWeights.splice(chosenIndex, 1); + choices.push(choice); + } + } + + return { + choices, + nextRange: this.nextRangeForParentQueue(parentQueue, consumerId), + }; + } + + async nextCandidateSelection( + parentQueue: string, + consumerId: string + ): Promise<{ range: QueueRange }> { + return { + range: this.nextRangeForParentQueue(parentQueue, consumerId), + }; + } + + #calculateQueueWeights(queues: QueueWithScores[]) { + const avgQueueSize = queues.reduce((acc, { size }) => acc + size, 0) / queues.length; + const avgMessageAge = queues.reduce((acc, { age }) => acc + age, 0) / queues.length; + + return queues.map(({ capacities, age, queue, size }) => { + let totalWeight = 1; + + if (size > avgQueueSize) { + totalWeight += Math.min(size / avgQueueSize, 4); + } + + if (age > avgMessageAge) { + totalWeight += Math.min(age / avgMessageAge, 4); + } + + return { + queue, + totalWeight, + }; + }); + } +} + +function filterQueuesAtCapacity(queues: QueueWithScores[]) { + return queues.filter( + (queue) => + queue.capacities.queue.current < queue.capacities.queue.limit && + queue.capacities.env.current < queue.capacities.env.limit + ); +} + +function weightedRandomIndex(queues: Array<{ queue: string; totalWeight: number }>): number { + const totalWeight = queues.reduce((acc, queue) => acc + queue.totalWeight, 0); + let randomNum = Math.random() * totalWeight; + + for (let i = 0; i < queues.length; i++) { + const queue = queues[i]; + if (randomNum < queue.totalWeight) { + return i; + } + + randomNum -= queue.totalWeight; + } + + // If we get here, we should just return a random queue + return Math.floor(Math.random() * queues.length); +} diff --git a/internal-packages/run-engine/src/run-queue/types.ts b/internal-packages/run-engine/src/run-queue/types.ts new file mode 100644 index 0000000000..2d936264c1 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/types.ts @@ -0,0 +1,120 @@ +import { z } from "zod"; +import { RuntimeEnvironmentType } from "../../../database/src/index.js"; +import { MinimalAuthenticatedEnvironment } from "../shared/index.js"; + +export const InputPayload = z.object({ + runId: z.string(), + taskIdentifier: z.string(), + orgId: z.string(), + projectId: z.string(), + environmentId: z.string(), + environmentType: z.nativeEnum(RuntimeEnvironmentType), + queue: z.string(), + concurrencyKey: z.string().optional(), + timestamp: z.number(), + attempt: z.number(), +}); +export type InputPayload = z.infer; + +export const OutputPayload = InputPayload.extend({ + version: z.literal("1"), + masterQueues: z.string().array(), +}); +export type OutputPayload = z.infer; + +export type QueueCapacity = { + current: number; + limit: number; +}; + +export type QueueCapacities = { + queue: QueueCapacity; + env: QueueCapacity; +}; + +export type QueueWithScores = { + queue: string; + capacities: QueueCapacities; + age: number; + size: number; +}; + +export type QueueRange = { offset: number; count: number }; + +export interface RunQueueKeyProducer { + masterQueueScanPattern(masterQueue: string): string; + queueCurrentConcurrencyScanPattern(): string; + //queue + queueKey(env: MinimalAuthenticatedEnvironment, queue: string, concurrencyKey?: string): string; + envQueueKey(env: MinimalAuthenticatedEnvironment): string; + envQueueKeyFromQueue(queue: string): string; + queueConcurrencyLimitKey(env: MinimalAuthenticatedEnvironment, queue: string): string; + concurrencyLimitKeyFromQueue(queue: string): string; + currentConcurrencyKeyFromQueue(queue: string): string; + currentConcurrencyKey( + env: MinimalAuthenticatedEnvironment, + queue: string, + concurrencyKey?: string + ): string; + disabledConcurrencyLimitKeyFromQueue(queue: string): string; + //env oncurrency + envCurrentConcurrencyKey(env: MinimalAuthenticatedEnvironment): string; + envConcurrencyLimitKey(env: MinimalAuthenticatedEnvironment): string; + envConcurrencyLimitKeyFromQueue(queue: string): string; + envCurrentConcurrencyKeyFromQueue(queue: string): string; + //task concurrency + taskIdentifierCurrentConcurrencyKey( + env: MinimalAuthenticatedEnvironment, + taskIdentifier: string + ): string; + taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queue: string): string; + taskIdentifierCurrentConcurrencyKeyFromQueue(queue: string, taskIdentifier: string): string; + //project concurrency + projectCurrentConcurrencyKey(env: MinimalAuthenticatedEnvironment): string; + projectCurrentConcurrencyKeyFromQueue(queue: string): string; + //message payload + messageKeyPrefixFromQueue(queue: string): string; + messageKey(orgId: string, messageId: string): string; + //utils + stripKeyPrefix(key: string): string; + extractComponentsFromQueue(queue: string): { + orgId: string; + projectId: string; + envId: string; + queue: string; + concurrencyKey: string | undefined; + }; +} + +export type PriorityStrategyChoice = string[] | { abort: true }; + +export interface RunQueuePriorityStrategy { + /** + * chooseQueue is called to select the next queue to process a message from + * + * @param queues + * @param parentQueue + * @param consumerId + * + * @returns The queue to process the message from, or an object with `abort: true` if no queue is available + */ + chooseQueues( + queues: Array, + parentQueue: string, + consumerId: string, + previousRange: QueueRange, + maxCount: number + ): { choices: PriorityStrategyChoice; nextRange: QueueRange }; + + /** + * This function is called to get the next candidate selection for the queue + * The `range` is used to select the set of queues that will be considered for the next selection (passed to chooseQueue) + * The `selectionId` is used to identify the selection and should be passed to chooseQueue + * + * @param parentQueue The parent queue that holds the candidate queues + * @param consumerId The consumerId that is making the request + * + * @returns The scores and the selectionId for the next candidate selection + */ + nextCandidateSelection(parentQueue: string, consumerId: string): Promise<{ range: QueueRange }>; +} diff --git a/internal-packages/run-engine/src/shared/index.ts b/internal-packages/run-engine/src/shared/index.ts new file mode 100644 index 0000000000..3790918eab --- /dev/null +++ b/internal-packages/run-engine/src/shared/index.ts @@ -0,0 +1,39 @@ +import { Attributes } from "@opentelemetry/api"; +import { Prisma } from "@trigger.dev/database"; + +export type AuthenticatedEnvironment = Prisma.RuntimeEnvironmentGetPayload<{ + include: { project: true; organization: true; orgMember: true }; +}>; + +export type MinimalAuthenticatedEnvironment = { + id: AuthenticatedEnvironment["id"]; + type: AuthenticatedEnvironment["type"]; + maximumConcurrencyLimit: AuthenticatedEnvironment["maximumConcurrencyLimit"]; + project: { + id: AuthenticatedEnvironment["project"]["id"]; + }; + organization: { + id: AuthenticatedEnvironment["organization"]["id"]; + }; +}; + +const SemanticEnvResources = { + ENV_ID: "$trigger.env.id", + ENV_TYPE: "$trigger.env.type", + ENV_SLUG: "$trigger.env.slug", + ORG_ID: "$trigger.org.id", + ORG_SLUG: "$trigger.org.slug", + ORG_TITLE: "$trigger.org.title", + PROJECT_ID: "$trigger.project.id", + PROJECT_NAME: "$trigger.project.name", + USER_ID: "$trigger.user.id", +}; + +export function attributesFromAuthenticatedEnv(env: MinimalAuthenticatedEnvironment): Attributes { + return { + [SemanticEnvResources.ENV_ID]: env.id, + [SemanticEnvResources.ENV_TYPE]: env.type, + [SemanticEnvResources.ORG_ID]: env.organization.id, + [SemanticEnvResources.PROJECT_ID]: env.project.id, + }; +} diff --git a/internal-packages/run-engine/tsconfig.json b/internal-packages/run-engine/tsconfig.json new file mode 100644 index 0000000000..0ac9414b19 --- /dev/null +++ b/internal-packages/run-engine/tsconfig.json @@ -0,0 +1,29 @@ +{ + "compilerOptions": { + "target": "ES2019", + "lib": ["ES2019", "DOM", "DOM.Iterable"], + "module": "CommonJS", + "moduleResolution": "Node", + "moduleDetection": "force", + "verbatimModuleSyntax": false, + "types": ["vitest/globals"], + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "preserveWatchOutput": true, + "skipLibCheck": true, + "noEmit": true, + "strict": true, + "paths": { + "@internal/testcontainers": ["../../internal-packages/testcontainers/src/index"], + "@internal/testcontainers/*": ["../../internal-packages/testcontainers/src/*"], + "@internal/redis-worker": ["../../internal-packages/redis-worker/src/index"], + "@internal/redis-worker/*": ["../../internal-packages/redis-worker/src/*"], + "@trigger.dev/core": ["../../packages/core/src/index"], + "@trigger.dev/core/*": ["../../packages/core/src/*"], + "@trigger.dev/database": ["../database/src/index"], + "@trigger.dev/database/*": ["../database/src/*"] + } + }, + "exclude": ["node_modules"] +} diff --git a/internal-packages/run-engine/vitest.config.ts b/internal-packages/run-engine/vitest.config.ts new file mode 100644 index 0000000000..a34eb571d6 --- /dev/null +++ b/internal-packages/run-engine/vitest.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["**/*.test.ts"], + globals: true, + isolate: true, + poolOptions: { + threads: { + singleThread: true, + }, + fileParallelism: false, + }, + }, +}); diff --git a/internal-packages/testcontainers/package.json b/internal-packages/testcontainers/package.json index d64add0cab..5fa73d40cd 100644 --- a/internal-packages/testcontainers/package.json +++ b/internal-packages/testcontainers/package.json @@ -7,12 +7,12 @@ "dependencies": { "@opentelemetry/api": "^1.9.0", "@trigger.dev/database": "workspace:*", - "ioredis": "^5.3.2", - "typescript": "^4.8.4" + "ioredis": "^5.3.2" }, "devDependencies": { "@testcontainers/postgresql": "^10.13.1", "@testcontainers/redis": "^10.13.1", + "@trigger.dev/core": "workspace:*", "testcontainers": "^10.13.1", "tinyexec": "^0.3.0", "vitest": "^1.4.0" diff --git a/internal-packages/testcontainers/src/index.ts b/internal-packages/testcontainers/src/index.ts index c363aea4e8..4461f25549 100644 --- a/internal-packages/testcontainers/src/index.ts +++ b/internal-packages/testcontainers/src/index.ts @@ -6,6 +6,10 @@ import { Network, type StartedNetwork } from "testcontainers"; import { test } from "vitest"; import { createElectricContainer, createPostgresContainer, createRedisContainer } from "./utils"; +export { StartedRedisContainer }; +export * from "./setup"; +export { assertNonNullable } from "./utils"; + type NetworkContext = { network: StartedNetwork }; type PostgresContext = NetworkContext & { @@ -27,7 +31,12 @@ type Use = (value: T) => Promise; const network = async ({}, use: Use) => { const network = await new Network().start(); - await use(network); + try { + await use(network); + } finally { + // Make sure to stop the network after use + await network.stop(); + } }; const postgresContainer = async ( @@ -35,8 +44,11 @@ const postgresContainer = async ( use: Use ) => { const { container } = await createPostgresContainer(network); - await use(container); - await container.stop(); + try { + await use(container); + } finally { + await container.stop(); + } }; const prisma = async ( @@ -50,16 +62,22 @@ const prisma = async ( }, }, }); - await use(prisma); - await prisma.$disconnect(); + try { + await use(prisma); + } finally { + await prisma.$disconnect(); + } }; export const postgresTest = test.extend({ network, postgresContainer, prisma }); const redisContainer = async ({}, use: Use) => { const { container } = await createRedisContainer(); - await use(container); - await container.stop(); + try { + await use(container); + } finally { + await container.stop(); + } }; const redis = async ( @@ -71,8 +89,11 @@ const redis = async ( port: redisContainer.getPort(), password: redisContainer.getPassword(), }); - await use(redis); - await redis.quit(); + try { + await use(redis); + } finally { + await redis.quit(); + } }; export const redisTest = test.extend({ redisContainer, redis }); @@ -85,8 +106,11 @@ const electricOrigin = async ( use: Use ) => { const { origin, container } = await createElectricContainer(postgresContainer, network); - await use(origin); - await container.stop(); + try { + await use(origin); + } finally { + await container.stop(); + } }; export const containerTest = test.extend({ diff --git a/internal-packages/testcontainers/src/setup.ts b/internal-packages/testcontainers/src/setup.ts new file mode 100644 index 0000000000..a45d3df1b2 --- /dev/null +++ b/internal-packages/testcontainers/src/setup.ts @@ -0,0 +1,152 @@ +import { + CURRENT_DEPLOYMENT_LABEL, + generateFriendlyId, + sanitizeQueueName, +} from "@trigger.dev/core/v3/apps"; +import { MachineConfig } from "@trigger.dev/core/v3/schemas"; +import { + BackgroundWorkerTask, + Prisma, + PrismaClient, + RunEngineVersion, + RuntimeEnvironmentType, +} from "@trigger.dev/database"; + +export type AuthenticatedEnvironment = Prisma.RuntimeEnvironmentGetPayload<{ + include: { project: true; organization: true; orgMember: true }; +}>; + +export async function setupAuthenticatedEnvironment( + prisma: PrismaClient, + type: RuntimeEnvironmentType, + engine?: RunEngineVersion +) { + // Your database setup logic here + const org = await prisma.organization.create({ + data: { + title: "Test Organization", + slug: "test-organization", + }, + }); + + const project = await prisma.project.create({ + data: { + name: "Test Project", + slug: "test-project", + externalRef: "proj_1234", + organizationId: org.id, + engine, + }, + }); + + const environment = await prisma.runtimeEnvironment.create({ + data: { + type, + slug: "slug", + projectId: project.id, + organizationId: org.id, + apiKey: "api_key", + pkApiKey: "pk_api_key", + shortcode: "short_code", + maximumConcurrencyLimit: 10, + }, + }); + + return await prisma.runtimeEnvironment.findUniqueOrThrow({ + where: { + id: environment.id, + }, + include: { + project: true, + organization: true, + orgMember: true, + }, + }); +} + +export async function setupBackgroundWorker( + prisma: PrismaClient, + environment: AuthenticatedEnvironment, + taskIdentifier: string | string[], + machineConfig?: MachineConfig +) { + const worker = await prisma.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + contentHash: "hash", + projectId: environment.project.id, + runtimeEnvironmentId: environment.id, + version: "20241015.1", + metadata: {}, + }, + }); + + const taskIdentifiers = Array.isArray(taskIdentifier) ? taskIdentifier : [taskIdentifier]; + + const tasks: BackgroundWorkerTask[] = []; + + for (const identifier of taskIdentifiers) { + const task = await prisma.backgroundWorkerTask.create({ + data: { + friendlyId: generateFriendlyId("task"), + slug: identifier, + filePath: `/trigger/${identifier}.ts`, + exportName: identifier, + workerId: worker.id, + runtimeEnvironmentId: environment.id, + projectId: environment.project.id, + machineConfig, + }, + }); + + tasks.push(task); + + const queueName = sanitizeQueueName(`task/${identifier}`); + const taskQueue = await prisma.taskQueue.create({ + data: { + friendlyId: generateFriendlyId("queue"), + name: queueName, + concurrencyLimit: 10, + runtimeEnvironmentId: worker.runtimeEnvironmentId, + projectId: worker.projectId, + type: "VIRTUAL", + }, + }); + } + + if (environment.type !== "DEVELOPMENT") { + const deployment = await prisma.workerDeployment.create({ + data: { + friendlyId: generateFriendlyId("deployment"), + contentHash: worker.contentHash, + version: worker.version, + shortCode: "short_code", + imageReference: `trigger/${environment.project.externalRef}:${worker.version}.${environment.slug}`, + status: "DEPLOYED", + projectId: environment.project.id, + environmentId: environment.id, + workerId: worker.id, + }, + }); + + const promotion = await prisma.workerDeploymentPromotion.create({ + data: { + label: CURRENT_DEPLOYMENT_LABEL, + deploymentId: deployment.id, + environmentId: environment.id, + }, + }); + + return { + worker, + tasks, + deployment, + promotion, + }; + } + + return { + worker, + tasks, + }; +} diff --git a/internal-packages/testcontainers/src/utils.ts b/internal-packages/testcontainers/src/utils.ts index 140628630d..191a3406d1 100644 --- a/internal-packages/testcontainers/src/utils.ts +++ b/internal-packages/testcontainers/src/utils.ts @@ -3,6 +3,7 @@ import { RedisContainer } from "@testcontainers/redis"; import path from "path"; import { GenericContainer, StartedNetwork } from "testcontainers"; import { x } from "tinyexec"; +import { expect } from "vitest"; export async function createPostgresContainer(network: StartedNetwork) { const container = await new PostgreSqlContainer("docker.io/postgres:14") @@ -70,3 +71,8 @@ export async function createElectricContainer( origin: `http://${container.getHost()}:${container.getMappedPort(3000)}`, }; } + +export function assertNonNullable(value: T): asserts value is NonNullable { + expect(value).toBeDefined(); + expect(value).not.toBeNull(); +} diff --git a/internal-packages/zod-worker/package.json b/internal-packages/zod-worker/package.json index 672d668ddc..712a110a9c 100644 --- a/internal-packages/zod-worker/package.json +++ b/internal-packages/zod-worker/package.json @@ -10,7 +10,6 @@ "@trigger.dev/database": "workspace:*", "graphile-worker": "0.16.6", "lodash.omit": "^4.5.0", - "typescript": "^5.5.4", "zod": "3.23.8" }, "devDependencies": { diff --git a/package.json b/package.json index 50ee8dc9e1..c38925e9c3 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "prettier": "^3.0.0", "tsx": "^3.7.1", "turbo": "^1.10.3", - "typescript": "^5.5.4", + "typescript": "5.5.4", "vite": "^4.1.1", "vite-tsconfig-paths": "^4.0.5", "vitest": "^0.28.4" @@ -72,7 +72,8 @@ "patchedDependencies": { "@changesets/assemble-release-plan@5.2.4": "patches/@changesets__assemble-release-plan@5.2.4.patch", "engine.io-parser@5.2.2": "patches/engine.io-parser@5.2.2.patch", - "graphile-worker@0.16.6": "patches/graphile-worker@0.16.6.patch" + "graphile-worker@0.16.6": "patches/graphile-worker@0.16.6.patch", + "redlock@5.0.0-beta.2": "patches/redlock@5.0.0-beta.2.patch" } } } \ No newline at end of file diff --git a/packages/build/package.json b/packages/build/package.json index 6a3f851a8a..323904be7c 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -57,7 +57,7 @@ } }, "scripts": { - "clean": "rimraf dist", + "clean": "rimraf dist .tshy .tshy-build .turbo", "build": "tshy && pnpm run update-version", "dev": "tshy --watch", "typecheck": "tsc --noEmit -p tsconfig.src.json", @@ -71,10 +71,8 @@ "tsconfck": "3.1.3" }, "devDependencies": { - "@types/node": "20.14.14", "rimraf": "6.0.1", "tshy": "^3.0.2", - "typescript": "^5.5.4", "tsx": "4.17.0", "esbuild": "^0.23.0", "@arethetypeswrong/cli": "^0.15.4" diff --git a/packages/cli-v3/package.json b/packages/cli-v3/package.json index 753e9be0f1..4a53ae2a74 100644 --- a/packages/cli-v3/package.json +++ b/packages/cli-v3/package.json @@ -47,7 +47,6 @@ "devDependencies": { "@epic-web/test-server": "^0.1.0", "@types/gradient-string": "^1.1.2", - "@types/node": "20.14.14", "@types/object-hash": "3.0.6", "@types/react": "^18.2.48", "@types/resolve": "^1.20.6", @@ -62,10 +61,10 @@ "ts-essentials": "10.0.1", "tshy": "^3.0.2", "tsx": "4.17.0", - "typescript": "^5.5.4", "vitest": "^2.0.5" }, "scripts": { + "clean": "rimraf dist .tshy .tshy-build .turbo", "typecheck": "tsc -p tsconfig.src.json --noEmit", "build": "tshy && pnpm run update-version", "dev": "tshy --watch", @@ -89,6 +88,7 @@ "@opentelemetry/semantic-conventions": "1.25.1", "@trigger.dev/build": "workspace:3.3.7", "@trigger.dev/core": "workspace:3.3.7", + "@trigger.dev/worker": "workspace:3.3.7", "c12": "^1.11.1", "chalk": "^5.2.0", "cli-table3": "^0.6.3", @@ -114,6 +114,7 @@ "resolve": "^1.22.8", "semver": "^7.5.0", "signal-exit": "^4.1.0", + "socket.io-client": "4.7.5", "source-map-support": "0.5.21", "std-env": "^3.7.0", "terminal-link": "^3.0.0", diff --git a/packages/cli-v3/src/apiClient.ts b/packages/cli-v3/src/apiClient.ts index 34a1009db1..74d92ed303 100644 --- a/packages/cli-v3/src/apiClient.ts +++ b/packages/cli-v3/src/apiClient.ts @@ -20,6 +20,12 @@ import { FailDeploymentRequestBody, FailDeploymentResponseBody, FinalizeDeploymentRequestBody, + WorkersListResponseBody, + WorkersCreateResponseBody, + WorkersCreateRequestBody, + TriggerTaskRequestBody, + TriggerTaskResponse, + GetLatestDeploymentResponseBody, } from "@trigger.dev/core/v3"; import { zodfetch, ApiError } from "@trigger.dev/core/v3/zodfetch"; @@ -301,6 +307,81 @@ export class CliApiClient { } ); } + + async triggerTaskRun(taskId: string, body?: TriggerTaskRequestBody) { + if (!this.accessToken) { + throw new Error("triggerTaskRun: No access token"); + } + + return wrapZodFetch(TriggerTaskResponse, `${this.apiURL}/api/v1/tasks/${taskId}/trigger`, { + method: "POST", + headers: { + Authorization: `Bearer ${this.accessToken}`, + Accept: "application/json", + }, + body: JSON.stringify(body ?? {}), + }); + } + + get workers() { + return { + list: this.listWorkers.bind(this), + create: this.createWorker.bind(this), + }; + } + + get deployments() { + return { + unmanaged: { + latest: this.getLatestUnmanagedDeployment.bind(this), + }, + }; + } + + private async getLatestUnmanagedDeployment() { + if (!this.accessToken) { + throw new Error("getLatestUnmanagedDeployment: No access token"); + } + + return wrapZodFetch( + GetLatestDeploymentResponseBody, + `${this.apiURL}/api/v1/deployments/latest`, + { + headers: { + Authorization: `Bearer ${this.accessToken}`, + Accept: "application/json", + }, + } + ); + } + + private async listWorkers() { + if (!this.accessToken) { + throw new Error("listWorkers: No access token"); + } + + return wrapZodFetch(WorkersListResponseBody, `${this.apiURL}/api/v1/workers`, { + headers: { + Authorization: `Bearer ${this.accessToken}`, + Accept: "application/json", + }, + }); + } + + private async createWorker(options: WorkersCreateRequestBody) { + if (!this.accessToken) { + throw new Error("createWorker: No access token"); + } + + return wrapZodFetch(WorkersCreateResponseBody, `${this.apiURL}/api/v1/workers`, { + method: "POST", + headers: { + Authorization: `Bearer ${this.accessToken}`, + Accept: "application/json", + }, + body: JSON.stringify(options), + }); + } } type ApiResult = diff --git a/packages/cli-v3/src/build/buildWorker.ts b/packages/cli-v3/src/build/buildWorker.ts index 9b4678a34c..8c6e2d1cc3 100644 --- a/packages/cli-v3/src/build/buildWorker.ts +++ b/packages/cli-v3/src/build/buildWorker.ts @@ -1,9 +1,6 @@ -import { CORE_VERSION } from "@trigger.dev/core/v3"; -import { DEFAULT_RUNTIME, ResolvedConfig } from "@trigger.dev/core/v3/build"; +import { ResolvedConfig } from "@trigger.dev/core/v3/build"; import { BuildManifest, BuildTarget } from "@trigger.dev/core/v3/schemas"; -import { resolveFileSources } from "../utilities/sourceFiles.js"; -import { VERSION } from "../version.js"; -import { BundleResult, bundleWorker } from "./bundle.js"; +import { BundleResult, bundleWorker, createBuildManifestFromBundle } from "./bundle.js"; import { createBuildContext, notifyExtensionOnBuildComplete, @@ -11,13 +8,6 @@ import { resolvePluginsForContext, } from "./extensions.js"; import { createExternalsBuildExtension } from "./externals.js"; -import { - deployIndexController, - deployIndexWorker, - deployRunController, - deployRunWorker, - telemetryEntryPoint, -} from "./packageModules.js"; import { join, relative, sep } from "node:path"; import { generateContainerfile } from "../deploy/buildImage.js"; import { writeFile } from "node:fs/promises"; @@ -57,7 +47,7 @@ export async function buildWorker(options: BuildWorkerOptions) { resolvedConfig, options.forcedExternals ); - const buildContext = createBuildContext("deploy", resolvedConfig); + const buildContext = createBuildContext(options.target, resolvedConfig); buildContext.prependExtension(externalsExtension); await notifyExtensionOnBuildStart(buildContext); const pluginsFromExtensions = resolvePluginsForContext(buildContext); @@ -80,39 +70,18 @@ export async function buildWorker(options: BuildWorkerOptions) { options.listener?.onBundleComplete?.(bundleResult); - let buildManifest: BuildManifest = { - contentHash: bundleResult.contentHash, - runtime: resolvedConfig.runtime ?? DEFAULT_RUNTIME, + let buildManifest = await createBuildManifestFromBundle({ + bundle: bundleResult, + destination: options.destination, + resolvedConfig, environment: options.environment, - packageVersion: sdkVersionExtractor.sdkVersion ?? CORE_VERSION, - cliPackageVersion: VERSION, - target: "deploy", - files: bundleResult.files, - sources: await resolveFileSources(bundleResult.files, resolvedConfig), - config: { - project: resolvedConfig.project, - dirs: resolvedConfig.dirs, - }, - outputPath: options.destination, - runControllerEntryPoint: bundleResult.runControllerEntryPoint ?? deployRunController, - runWorkerEntryPoint: bundleResult.runWorkerEntryPoint ?? deployRunWorker, - indexControllerEntryPoint: bundleResult.indexControllerEntryPoint ?? deployIndexController, - indexWorkerEntryPoint: bundleResult.indexWorkerEntryPoint ?? deployIndexWorker, - loaderEntryPoint: bundleResult.loaderEntryPoint, - configPath: bundleResult.configPath, - customConditions: resolvedConfig.build.conditions ?? [], - deploy: { - env: options.envVars ? options.envVars : {}, - }, - build: {}, - otelImportHook: { - include: resolvedConfig.instrumentedPackageNames ?? [], - }, - }; + target: options.target, + envVars: options.envVars, + }); buildManifest = await notifyExtensionOnBuildComplete(buildContext, buildManifest); - if (options.target === "deploy") { + if (options.target !== "dev") { buildManifest = options.rewritePaths ? rewriteBuildManifestPaths(buildManifest, options.destination) : buildManifest; diff --git a/packages/cli-v3/src/build/bundle.ts b/packages/cli-v3/src/build/bundle.ts index 251e975e22..14fe3198b4 100644 --- a/packages/cli-v3/src/build/bundle.ts +++ b/packages/cli-v3/src/build/bundle.ts @@ -1,5 +1,5 @@ -import { ResolvedConfig } from "@trigger.dev/core/v3/build"; -import { BuildTarget, TaskFile } from "@trigger.dev/core/v3/schemas"; +import { DEFAULT_RUNTIME, ResolvedConfig } from "@trigger.dev/core/v3/build"; +import { BuildManifest, BuildTarget, TaskFile } from "@trigger.dev/core/v3/schemas"; import * as esbuild from "esbuild"; import { createHash } from "node:crypto"; import { join, relative, resolve } from "node:path"; @@ -8,6 +8,10 @@ import { logger } from "../utilities/logger.js"; import { deployEntryPoints, devEntryPoints, + getIndexControllerForTarget, + getIndexWorkerForTarget, + getRunControllerForTarget, + getRunWorkerForTarget, isIndexControllerForTarget, isIndexWorkerForTarget, isLoaderEntryPoint, @@ -15,8 +19,15 @@ import { isRunWorkerForTarget, shims, telemetryEntryPoint, + managedEntryPoints, + unmanagedEntryPoints, } from "./packageModules.js"; import { buildPlugins } from "./plugins.js"; +import { CORE_VERSION } from "@trigger.dev/core/v3"; +import { resolveFileSources } from "../utilities/sourceFiles.js"; +import { copyManifestToDir } from "./manifests.js"; +import { VERSION } from "../version.js"; +import { assertExhaustive } from "../utilities/assertExhaustive.js"; export interface BundleOptions { target: BuildTarget; @@ -223,10 +234,26 @@ async function getEntryPoints(target: BuildTarget, config: ResolvedConfig) { projectEntryPoints.push(config.configFile); } - if (target === "dev") { - projectEntryPoints.push(...devEntryPoints); - } else { - projectEntryPoints.push(...deployEntryPoints); + switch (target) { + case "dev": { + projectEntryPoints.push(...devEntryPoints); + break; + } + case "deploy": { + projectEntryPoints.push(...deployEntryPoints); + break; + } + case "managed": { + projectEntryPoints.push(...managedEntryPoints); + break; + } + case "unmanaged": { + projectEntryPoints.push(...unmanagedEntryPoints); + break; + } + default: { + assertExhaustive(target); + } } if (config.instrumentedPackageNames?.length ?? 0 > 0) { @@ -268,3 +295,61 @@ export function logBuildFailure(errors: esbuild.Message[], warnings: esbuild.Mes } logBuildWarnings(warnings); } + +export async function createBuildManifestFromBundle({ + bundle, + destination, + resolvedConfig, + workerDir, + environment, + target, + envVars, + sdkVersion, +}: { + bundle: BundleResult; + destination: string; + resolvedConfig: ResolvedConfig; + workerDir?: string; + environment: string; + target: BuildTarget; + envVars?: Record; + sdkVersion?: string; +}): Promise { + const buildManifest: BuildManifest = { + contentHash: bundle.contentHash, + runtime: resolvedConfig.runtime ?? DEFAULT_RUNTIME, + environment: environment, + packageVersion: sdkVersion ?? CORE_VERSION, + cliPackageVersion: VERSION, + target: target, + files: bundle.files, + sources: await resolveFileSources(bundle.files, resolvedConfig), + externals: [], + config: { + project: resolvedConfig.project, + dirs: resolvedConfig.dirs, + }, + outputPath: destination, + indexControllerEntryPoint: + bundle.indexControllerEntryPoint ?? getIndexControllerForTarget(target), + indexWorkerEntryPoint: bundle.indexWorkerEntryPoint ?? getIndexWorkerForTarget(target), + runControllerEntryPoint: bundle.runControllerEntryPoint ?? getRunControllerForTarget(target), + runWorkerEntryPoint: bundle.runWorkerEntryPoint ?? getRunWorkerForTarget(target), + loaderEntryPoint: bundle.loaderEntryPoint, + configPath: bundle.configPath, + customConditions: resolvedConfig.build.conditions ?? [], + deploy: { + env: envVars ?? {}, + }, + build: {}, + otelImportHook: { + include: resolvedConfig.instrumentedPackageNames ?? [], + }, + }; + + if (!workerDir) { + return buildManifest; + } + + return copyManifestToDir(buildManifest, destination, workerDir); +} diff --git a/packages/cli-v3/src/build/packageModules.ts b/packages/cli-v3/src/build/packageModules.ts index 537a25c00d..1224404c51 100644 --- a/packages/cli-v3/src/build/packageModules.ts +++ b/packages/cli-v3/src/build/packageModules.ts @@ -1,10 +1,25 @@ import { BuildTarget } from "@trigger.dev/core/v3"; import { join } from "node:path"; import { sourceDir } from "../sourceDir.js"; +import { assertExhaustive } from "../utilities/assertExhaustive.js"; export const devRunWorker = join(sourceDir, "entryPoints", "dev-run-worker.js"); export const devIndexWorker = join(sourceDir, "entryPoints", "dev-index-worker.js"); +export const managedRunController = join(sourceDir, "entryPoints", "managed-run-controller.js"); +export const managedRunWorker = join(sourceDir, "entryPoints", "managed-run-worker.js"); +export const managedIndexController = join(sourceDir, "entryPoints", "managed-index-controller.js"); +export const managedIndexWorker = join(sourceDir, "entryPoints", "managed-index-worker.js"); + +export const unmanagedRunController = join(sourceDir, "entryPoints", "unmanaged-run-controller.js"); +export const unmanagedRunWorker = join(sourceDir, "entryPoints", "unmanaged-run-worker.js"); +export const unmanagedIndexController = join( + sourceDir, + "entryPoints", + "unmanaged-index-controller.js" +); +export const unmanagedIndexWorker = join(sourceDir, "entryPoints", "unmanaged-index-worker.js"); + export const deployRunController = join(sourceDir, "entryPoints", "deploy-run-controller.js"); export const deployRunWorker = join(sourceDir, "entryPoints", "deploy-run-worker.js"); export const deployIndexController = join(sourceDir, "entryPoints", "deploy-index-controller.js"); @@ -13,6 +28,18 @@ export const deployIndexWorker = join(sourceDir, "entryPoints", "deploy-index-wo export const telemetryEntryPoint = join(sourceDir, "entryPoints", "loader.js"); export const devEntryPoints = [devRunWorker, devIndexWorker]; +export const managedEntryPoints = [ + managedRunController, + managedRunWorker, + managedIndexController, + managedIndexWorker, +]; +export const unmanagedEntryPoints = [ + unmanagedRunController, + unmanagedRunWorker, + unmanagedIndexController, + unmanagedIndexWorker, +]; export const deployEntryPoints = [ deployRunController, deployRunWorker, @@ -40,6 +67,70 @@ function isDevIndexWorker(entryPoint: string) { ); } +// IMPORTANT: this may look like it should not work on Windows, but it does (and changing to using path.join will break stuff) +function isManagedRunController(entryPoint: string) { + return ( + entryPoint.includes("dist/esm/entryPoints/managed-run-controller.js") || + entryPoint.includes("src/entryPoints/managed-run-controller.ts") + ); +} + +// IMPORTANT: this may look like it should not work on Windows, but it does (and changing to using path.join will break stuff) +function isManagedRunWorker(entryPoint: string) { + return ( + entryPoint.includes("dist/esm/entryPoints/managed-run-worker.js") || + entryPoint.includes("src/entryPoints/managed-run-worker.ts") + ); +} + +// IMPORTANT: this may look like it should not work on Windows, but it does (and changing to using path.join will break stuff) +function isManagedIndexController(entryPoint: string) { + return ( + entryPoint.includes("dist/esm/entryPoints/managed-index-controller.js") || + entryPoint.includes("src/entryPoints/managed-index-controller.ts") + ); +} + +// IMPORTANT: this may look like it should not work on Windows, but it does (and changing to using path.join will break stuff) +function isManagedIndexWorker(entryPoint: string) { + return ( + entryPoint.includes("dist/esm/entryPoints/managed-index-worker.js") || + entryPoint.includes("src/entryPoints/managed-index-worker.ts") + ); +} + +// IMPORTANT: this may look like it should not work on Windows, but it does (and changing to using path.join will break stuff) +function isUnmanagedRunController(entryPoint: string) { + return ( + entryPoint.includes("dist/esm/entryPoints/unmanaged-run-controller.js") || + entryPoint.includes("src/entryPoints/unmanaged-run-controller.ts") + ); +} + +// IMPORTANT: this may look like it should not work on Windows, but it does (and changing to using path.join will break stuff) +function isUnmanagedRunWorker(entryPoint: string) { + return ( + entryPoint.includes("dist/esm/entryPoints/unmanaged-run-worker.js") || + entryPoint.includes("src/entryPoints/unmanaged-run-worker.ts") + ); +} + +// IMPORTANT: this may look like it should not work on Windows, but it does (and changing to using path.join will break stuff) +function isUnmanagedIndexController(entryPoint: string) { + return ( + entryPoint.includes("dist/esm/entryPoints/unmanaged-index-controller.js") || + entryPoint.includes("src/entryPoints/unmanaged-index-controller.ts") + ); +} + +// IMPORTANT: this may look like it should not work on Windows, but it does (and changing to using path.join will break stuff) +function isUnmanagedIndexWorker(entryPoint: string) { + return ( + entryPoint.includes("dist/esm/entryPoints/unmanaged-index-worker.js") || + entryPoint.includes("src/entryPoints/unmanaged-index-worker.ts") + ); +} + // IMPORTANT: this may look like it should not work on Windows, but it does (and changing to using path.join will break stuff) function isDeployIndexController(entryPoint: string) { return ( @@ -80,35 +171,123 @@ export function isLoaderEntryPoint(entryPoint: string) { } export function isRunWorkerForTarget(entryPoint: string, target: BuildTarget) { - if (target === "dev") { - return isDevRunWorker(entryPoint); - } else { - return isDeployRunWorker(entryPoint); + switch (target) { + case "dev": + return isDevRunWorker(entryPoint); + case "deploy": + return isDeployRunWorker(entryPoint); + case "managed": + return isManagedRunWorker(entryPoint); + case "unmanaged": + return isUnmanagedRunWorker(entryPoint); + default: + assertExhaustive(target); + } +} + +export function getRunWorkerForTarget(target: BuildTarget) { + switch (target) { + case "dev": + return devRunWorker; + case "deploy": + return deployRunWorker; + case "managed": + return managedRunWorker; + case "unmanaged": + return unmanagedRunWorker; + default: + assertExhaustive(target); } } export function isRunControllerForTarget(entryPoint: string, target: BuildTarget) { - if (target === "deploy") { - return isDeployRunController(entryPoint); + switch (target) { + case "dev": + return false; + case "deploy": + return isDeployRunController(entryPoint); + case "managed": + return isManagedRunController(entryPoint); + case "unmanaged": + return isUnmanagedRunController(entryPoint); + default: + assertExhaustive(target); } +} - return false; +export function getRunControllerForTarget(target: BuildTarget) { + switch (target) { + case "dev": + return undefined; + case "deploy": + return deployRunController; + case "managed": + return managedRunController; + case "unmanaged": + return unmanagedRunController; + default: + assertExhaustive(target); + } } export function isIndexWorkerForTarget(entryPoint: string, target: BuildTarget) { - if (target === "dev") { - return isDevIndexWorker(entryPoint); - } else { - return isDeployIndexWorker(entryPoint); + switch (target) { + case "dev": + return isDevIndexWorker(entryPoint); + case "deploy": + return isDeployIndexWorker(entryPoint); + case "managed": + return isManagedIndexWorker(entryPoint); + case "unmanaged": + return isUnmanagedIndexWorker(entryPoint); + default: + assertExhaustive(target); + } +} + +export function getIndexWorkerForTarget(target: BuildTarget) { + switch (target) { + case "dev": + return devIndexWorker; + case "deploy": + return deployIndexWorker; + case "managed": + return managedIndexWorker; + case "unmanaged": + return unmanagedIndexWorker; + default: + assertExhaustive(target); } } export function isIndexControllerForTarget(entryPoint: string, target: BuildTarget) { - if (target === "deploy") { - return isDeployIndexController(entryPoint); + switch (target) { + case "dev": + return false; + case "deploy": + return isDeployIndexController(entryPoint); + case "managed": + return isManagedIndexController(entryPoint); + case "unmanaged": + return isUnmanagedIndexController(entryPoint); + default: + assertExhaustive(target); } +} - return false; +export function getIndexControllerForTarget(target: BuildTarget) { + switch (target) { + case "dev": + return undefined; + case "deploy": + return deployIndexController; + case "managed": + return managedIndexController; + case "unmanaged": + return unmanagedIndexController; + default: + assertExhaustive(target); + } } export function isConfigEntryPoint(entryPoint: string) { diff --git a/packages/cli-v3/src/cli/common.ts b/packages/cli-v3/src/cli/common.ts index cddde4736e..31fa09258c 100644 --- a/packages/cli-v3/src/cli/common.ts +++ b/packages/cli-v3/src/cli/common.ts @@ -7,20 +7,22 @@ import { fromZodError } from "zod-validation-error"; import { logger } from "../utilities/logger.js"; import { outro } from "@clack/prompts"; import { chalkError } from "../utilities/cliOutput.js"; +import { CLOUD_API_URL } from "../consts.js"; +import { readAuthConfigCurrentProfileName } from "../utilities/configFiles.js"; export const CommonCommandOptions = z.object({ apiUrl: z.string().optional(), logLevel: z.enum(["debug", "info", "log", "warn", "error", "none"]).default("log"), skipTelemetry: z.boolean().default(false), - profile: z.string().default("default"), + profile: z.string().default(readAuthConfigCurrentProfileName()), }); export type CommonCommandOptions = z.infer; export function commonOptions(command: Command) { return command - .option("--profile ", "The login profile to use", "default") - .option("-a, --api-url ", "Override the API URL", "https://api.trigger.dev") + .option("--profile ", "The login profile to use", readAuthConfigCurrentProfileName()) + .option("-a, --api-url ", "Override the API URL", CLOUD_API_URL) .option( "-l, --log-level ", "The CLI log level to use (debug, info, log, warn, error, none). This does not effect the log level of your trigger.dev tasks.", diff --git a/packages/cli-v3/src/cli/index.ts b/packages/cli-v3/src/cli/index.ts index 3c31d030f2..b061a1cf14 100644 --- a/packages/cli-v3/src/cli/index.ts +++ b/packages/cli-v3/src/cli/index.ts @@ -10,6 +10,9 @@ import { configureUpdateCommand } from "../commands/update.js"; import { VERSION } from "../version.js"; import { configureDeployCommand } from "../commands/deploy.js"; import { installExitHandler } from "./common.js"; +import { configureWorkersCommand } from "../commands/workers/index.js"; +import { configureSwitchProfilesCommand } from "../commands/switch.js"; +import { configureTriggerTaskCommand } from "../commands/trigger.js"; export const program = new Command(); @@ -25,6 +28,9 @@ configureDeployCommand(program); configureWhoamiCommand(program); configureLogoutCommand(program); configureListProfilesCommand(program); +configureSwitchProfilesCommand(program); configureUpdateCommand(program); +// configureWorkersCommand(program); +// configureTriggerTaskCommand(program); installExitHandler(); diff --git a/packages/cli-v3/src/commands/deploy.ts b/packages/cli-v3/src/commands/deploy.ts index eb6f6aef11..42f6952cdf 100644 --- a/packages/cli-v3/src/commands/deploy.ts +++ b/packages/cli-v3/src/commands/deploy.ts @@ -1,11 +1,8 @@ import { intro, outro } from "@clack/prompts"; import { prepareDeploymentError } from "@trigger.dev/core/v3"; -import { ResolvedConfig } from "@trigger.dev/core/v3/build"; -import { BuildManifest, InitializeDeploymentResponseBody } from "@trigger.dev/core/v3/schemas"; +import { InitializeDeploymentResponseBody } from "@trigger.dev/core/v3/schemas"; import { Command, Option as CommandOption } from "commander"; -import { writeFile } from "node:fs/promises"; -import { join, relative, resolve } from "node:path"; -import { readPackageJSON, writePackageJSON } from "pkg-types"; +import { resolve } from "node:path"; import { z } from "zod"; import { CliApiClient } from "../apiClient.js"; import { buildWorker } from "../build/buildWorker.js"; @@ -17,7 +14,7 @@ import { wrapCommandAction, } from "../cli/common.js"; import { loadConfig } from "../config.js"; -import { buildImage, generateContainerfile } from "../deploy/buildImage.js"; +import { buildImage } from "../deploy/buildImage.js"; import { checkLogsForErrors, checkLogsForWarnings, @@ -25,10 +22,8 @@ import { printWarnings, saveLogs, } from "../deploy/logs.js"; -import { buildManifestToJSON } from "../utilities/buildManifest.js"; import { chalkError, cliLink, isLinksSupported, prettyError } from "../utilities/cliOutput.js"; import { loadDotEnvVars } from "../utilities/dotEnv.js"; -import { writeJSONFile } from "../utilities/fileSystem.js"; import { printStandloneInitialBanner } from "../utilities/initialBanner.js"; import { logger } from "../utilities/logger.js"; import { getProjectClient } from "../utilities/session.js"; @@ -51,7 +46,6 @@ const DeployCommandOptions = CommonCommandOptions.extend({ push: z.boolean().default(false), config: z.string().optional(), projectRef: z.string().optional(), - apiUrl: z.string().optional(), saveLogs: z.boolean().default(false), skipUpdateCheck: z.boolean().default(false), noCache: z.boolean().default(false), @@ -217,8 +211,10 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) { const forcedExternals = await resolveAlwaysExternal(projectClient.client); + const { features } = resolvedConfig; + const buildManifest = await buildWorker({ - target: "deploy", + target: features.run_engine_v2 ? "managed" : "deploy", environment: options.env, destination: destination.path, resolvedConfig, @@ -250,6 +246,7 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) { selfHosted: options.selfHosted, registryHost: options.registry, namespace: options.namespace, + type: features.run_engine_v2 ? "MANAGED" : "V1", }); if (!deploymentResponse.success) { @@ -459,104 +456,6 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) { ); } -function rewriteBuildManifestPaths( - buildManifest: BuildManifest, - destinationDir: string -): BuildManifest { - return { - ...buildManifest, - files: buildManifest.files.map((file) => ({ - ...file, - entry: cleanEntryPath(file.entry), - out: rewriteOutputPath(destinationDir, file.out), - })), - outputPath: rewriteOutputPath(destinationDir, buildManifest.outputPath), - configPath: rewriteOutputPath(destinationDir, buildManifest.configPath), - runControllerEntryPoint: buildManifest.runControllerEntryPoint - ? rewriteOutputPath(destinationDir, buildManifest.runControllerEntryPoint) - : undefined, - runWorkerEntryPoint: rewriteOutputPath(destinationDir, buildManifest.runWorkerEntryPoint), - indexControllerEntryPoint: buildManifest.indexControllerEntryPoint - ? rewriteOutputPath(destinationDir, buildManifest.indexControllerEntryPoint) - : undefined, - indexWorkerEntryPoint: rewriteOutputPath(destinationDir, buildManifest.indexWorkerEntryPoint), - loaderEntryPoint: buildManifest.loaderEntryPoint - ? rewriteOutputPath(destinationDir, buildManifest.loaderEntryPoint) - : undefined, - }; -} - -async function writeProjectFiles( - buildManifest: BuildManifest, - resolvedConfig: ResolvedConfig, - outputPath: string -) { - // Step 1. Read the package.json file - const packageJson = await readProjectPackageJson(resolvedConfig.packageJsonPath); - - if (!packageJson) { - throw new Error("Could not read the package.json file"); - } - - const dependencies = - buildManifest.externals?.reduce( - (acc, external) => { - acc[external.name] = external.version; - - return acc; - }, - {} as Record - ) ?? {}; - - // Step 3: Write the resolved dependencies to the package.json file - await writePackageJSON(join(outputPath, "package.json"), { - ...packageJson, - name: packageJson.name ?? "trigger-project", - dependencies: { - ...dependencies, - }, - trustedDependencies: Object.keys(dependencies), - devDependencies: {}, - peerDependencies: {}, - scripts: {}, - }); - - await writeJSONFile(join(outputPath, "build.json"), buildManifestToJSON(buildManifest)); - await writeContainerfile(outputPath, buildManifest); -} - -async function readProjectPackageJson(packageJsonPath: string) { - const packageJson = await readPackageJSON(packageJsonPath); - - return packageJson; -} - -// Remove any query parameters from the entry path -// For example, src/trigger/ai.ts?sentryProxyModule=true -> src/trigger/ai.ts -function cleanEntryPath(entry: string): string { - return entry.split("?")[0]!; -} - -function rewriteOutputPath(destinationDir: string, filePath: string) { - return `/app/${relative(destinationDir, filePath)}`; -} - -async function writeContainerfile(outputPath: string, buildManifest: BuildManifest) { - if (!buildManifest.runControllerEntryPoint || !buildManifest.indexControllerEntryPoint) { - throw new Error("Something went wrong with the build. Aborting deployment. [code 7789]"); - } - - const containerfile = await generateContainerfile({ - runtime: buildManifest.runtime, - entrypoint: buildManifest.runControllerEntryPoint, - build: buildManifest.build, - image: buildManifest.image, - indexScript: buildManifest.indexControllerEntryPoint, - }); - - await writeFile(join(outputPath, "Containerfile"), containerfile); -} - export async function syncEnvVarsWithServer( apiClient: CliApiClient, projectRef: string, diff --git a/packages/cli-v3/src/commands/list-profiles.ts b/packages/cli-v3/src/commands/list-profiles.ts index 0a0a0c34e9..ac0c7e12fd 100644 --- a/packages/cli-v3/src/commands/list-profiles.ts +++ b/packages/cli-v3/src/commands/list-profiles.ts @@ -7,7 +7,10 @@ import { readAuthConfigFile } from "../utilities/configFiles.js"; import { printInitialBanner } from "../utilities/initialBanner.js"; import { logger } from "../utilities/logger.js"; -const ListProfilesOptions = CommonCommandOptions; +const ListProfilesOptions = CommonCommandOptions.pick({ + logLevel: true, + skipTelemetry: true, +}); type ListProfilesOptions = z.infer; @@ -43,12 +46,12 @@ export async function listProfiles(options: ListProfilesOptions) { return; } - const profiles = Object.keys(authConfig); + const profileNames = Object.keys(authConfig.profiles); log.message("Profiles:"); - for (const profile of profiles) { - const profileConfig = authConfig[profile]; + for (const profile of profileNames) { + const profileConfig = authConfig.profiles[profile]; log.info(`${profile}${profileConfig?.apiUrl ? ` - ${chalkGrey(profileConfig.apiUrl)}` : ""}`); } diff --git a/packages/cli-v3/src/commands/login.ts b/packages/cli-v3/src/commands/login.ts index 67e136c771..e820ad1b13 100644 --- a/packages/cli-v3/src/commands/login.ts +++ b/packages/cli-v3/src/commands/login.ts @@ -23,6 +23,7 @@ import { spinner } from "../utilities/windows.js"; import { isLinuxServer } from "../utilities/linux.js"; import { VERSION } from "../version.js"; import { env } from "std-env"; +import { CLOUD_API_URL } from "../consts.js"; export const LoginCommandOptions = CommonCommandOptions.extend({ apiUrl: z.string(), @@ -66,7 +67,7 @@ export async function login(options?: LoginOptions): Promise { return await tracer.startActiveSpan("login", async (span) => { try { const opts = { - defaultApiUrl: "https://api.trigger.dev", + defaultApiUrl: CLOUD_API_URL, embedded: false, silent: false, ...options, @@ -86,7 +87,7 @@ export async function login(options?: LoginOptions): Promise { if (accessTokenFromEnv) { const auth = { accessToken: accessTokenFromEnv, - apiUrl: env.TRIGGER_API_URL ?? opts.defaultApiUrl ?? "https://api.trigger.dev", + apiUrl: env.TRIGGER_API_URL ?? opts.defaultApiUrl ?? CLOUD_API_URL, }; const apiClient = new CliApiClient(auth.apiUrl, auth.accessToken); const userData = await apiClient.whoAmI(); diff --git a/packages/cli-v3/src/commands/switch.ts b/packages/cli-v3/src/commands/switch.ts new file mode 100644 index 0000000000..f62d94099c --- /dev/null +++ b/packages/cli-v3/src/commands/switch.ts @@ -0,0 +1,89 @@ +import { intro, isCancel, outro, select } from "@clack/prompts"; +import { Command } from "commander"; +import { z } from "zod"; +import { + CommonCommandOptions, + handleTelemetry, + OutroCommandError, + wrapCommandAction, +} from "../cli/common.js"; +import { chalkGrey } from "../utilities/cliOutput.js"; +import { readAuthConfigFile, writeAuthConfigCurrentProfileName } from "../utilities/configFiles.js"; +import { printInitialBanner } from "../utilities/initialBanner.js"; +import { logger } from "../utilities/logger.js"; +import { CLOUD_API_URL } from "../consts.js"; + +const SwitchProfilesOptions = CommonCommandOptions.pick({ + logLevel: true, + skipTelemetry: true, +}); + +type SwitchProfilesOptions = z.infer; + +export function configureSwitchProfilesCommand(program: Command) { + return program + .command("switch") + .description("Set your default CLI profile") + .option( + "-l, --log-level ", + "The CLI log level to use (debug, info, log, warn, error, none). This does not effect the log level of your trigger.dev tasks.", + "log" + ) + .option("--skip-telemetry", "Opt-out of sending telemetry") + .action(async (options) => { + await handleTelemetry(async () => { + await switchProfilesCommand(options); + }); + }); +} + +export async function switchProfilesCommand(options: unknown) { + return await wrapCommandAction("switch", SwitchProfilesOptions, options, async (opts) => { + await printInitialBanner(false); + return await switchProfiles(opts); + }); +} + +export async function switchProfiles(options: SwitchProfilesOptions) { + intro("Switch profiles"); + + const authConfig = readAuthConfigFile(); + + if (!authConfig) { + logger.info("No profiles found"); + return; + } + + const profileNames = Object.keys(authConfig.profiles).sort((a, b) => { + // Default profile should always be first + if (a === authConfig.currentProfile) return -1; + if (b === authConfig.currentProfile) return 1; + + return a.localeCompare(b); + }); + + const profileSelection = await select({ + message: "Please select a new profile", + initialValue: authConfig.currentProfile, + options: profileNames.map((profile) => ({ + value: profile, + hint: authConfig.profiles[profile]?.apiUrl + ? authConfig.profiles[profile].apiUrl === CLOUD_API_URL + ? undefined + : chalkGrey(authConfig.profiles[profile].apiUrl) + : undefined, + })), + }); + + if (isCancel(profileSelection)) { + throw new OutroCommandError(); + } + + writeAuthConfigCurrentProfileName(profileSelection); + + if (profileSelection === authConfig.currentProfile) { + outro(`No change made`); + } else { + outro(`Switched to ${profileSelection}`); + } +} diff --git a/packages/cli-v3/src/commands/trigger.ts b/packages/cli-v3/src/commands/trigger.ts new file mode 100644 index 0000000000..1bf6425d6a --- /dev/null +++ b/packages/cli-v3/src/commands/trigger.ts @@ -0,0 +1,121 @@ +import { intro, outro } from "@clack/prompts"; +import { Command } from "commander"; +import { z } from "zod"; +import { CommonCommandOptions, handleTelemetry, wrapCommandAction } from "../cli/common.js"; +import { printInitialBanner } from "../utilities/initialBanner.js"; +import { logger } from "../utilities/logger.js"; +import { resolve } from "path"; +import { loadConfig } from "../config.js"; +import { getProjectClient } from "../utilities/session.js"; +import { login } from "./login.js"; +import { chalkGrey, chalkLink, cliLink } from "../utilities/cliOutput.js"; + +const TriggerTaskOptions = CommonCommandOptions.extend({ + env: z.enum(["prod", "staging"]), + config: z.string().optional(), + projectRef: z.string().optional(), +}); + +type TriggerTaskOptions = z.infer; + +export function configureTriggerTaskCommand(program: Command) { + return program + .command("trigger") + .description("Trigger a task") + .argument("[task-name]", "The name of the task") + .option( + "-l, --log-level ", + "The CLI log level to use (debug, info, log, warn, error, none). This does not effect the log level of your trigger.dev tasks.", + "log" + ) + .option("--skip-telemetry", "Opt-out of sending telemetry") + .option( + "-e, --env ", + "Deploy to a specific environment (currently only prod and staging are supported)", + "prod" + ) + .option("-c, --config ", "The name of the config file, found at [path]") + .option( + "-p, --project-ref ", + "The project ref. Required if there is no config file. This will override the project specified in the config file." + ) + .action(async (path, options) => { + await handleTelemetry(async () => { + await triggerTaskCommand(path, options); + }); + }); +} + +export async function triggerTaskCommand(taskName: string, options: unknown) { + return await wrapCommandAction("trigger", TriggerTaskOptions, options, async (opts) => { + await printInitialBanner(false); + return await triggerTask(taskName, opts); + }); +} + +export async function triggerTask(taskName: string, options: TriggerTaskOptions) { + if (!taskName) { + throw new Error("You must provide a task name"); + } + + intro(`Triggering task ${taskName}`); + + const authorization = await login({ + embedded: true, + defaultApiUrl: options.apiUrl, + profile: options.profile, + silent: true, + }); + + if (!authorization.ok) { + if (authorization.error === "fetch failed") { + throw new Error( + `Failed to connect to ${authorization.auth?.apiUrl}. Are you sure it's the correct URL?` + ); + } else { + throw new Error( + `You must login first. Use the \`login\` CLI command.\n\n${authorization.error}` + ); + } + } + + const projectPath = resolve(process.cwd(), "."); + + const resolvedConfig = await loadConfig({ + cwd: projectPath, + overrides: { project: options.projectRef }, + configFile: options.config, + }); + + logger.debug("Resolved config", resolvedConfig); + + const projectClient = await getProjectClient({ + accessToken: authorization.auth.accessToken, + apiUrl: authorization.auth.apiUrl, + projectRef: resolvedConfig.project, + env: options.env, + profile: options.profile, + }); + + if (!projectClient) { + throw new Error("Failed to get project client"); + } + + const triggered = await projectClient.client.triggerTaskRun(taskName, { + payload: { + message: "Triggered by CLI", + }, + }); + + if (!triggered.success) { + throw new Error("Failed to trigger task"); + } + + const baseUrl = `${authorization.dashboardUrl}/projects/v3/${resolvedConfig.project}`; + const runUrl = `${baseUrl}/runs/${triggered.data.id}`; + + const pipe = chalkGrey("|"); + const link = chalkLink(cliLink("View run", runUrl)); + + outro(`Success! ${pipe} ${link}`); +} diff --git a/packages/cli-v3/src/commands/workers/build.ts b/packages/cli-v3/src/commands/workers/build.ts new file mode 100644 index 0000000000..e28896de9b --- /dev/null +++ b/packages/cli-v3/src/commands/workers/build.ts @@ -0,0 +1,550 @@ +import { intro, outro, log } from "@clack/prompts"; +import { parseDockerImageReference, prepareDeploymentError } from "@trigger.dev/core/v3"; +import { InitializeDeploymentResponseBody } from "@trigger.dev/core/v3/schemas"; +import { Command, Option as CommandOption } from "commander"; +import { resolve } from "node:path"; +import { z } from "zod"; +import { CliApiClient } from "../../apiClient.js"; +import { buildWorker } from "../../build/buildWorker.js"; +import { + CommonCommandOptions, + commonOptions, + handleTelemetry, + SkipLoggingError, + wrapCommandAction, +} from "../../cli/common.js"; +import { loadConfig } from "../../config.js"; +import { buildImage } from "../../deploy/buildImage.js"; +import { + checkLogsForErrors, + checkLogsForWarnings, + printErrors, + printWarnings, + saveLogs, +} from "../../deploy/logs.js"; +import { chalkError, cliLink, isLinksSupported, prettyError } from "../../utilities/cliOutput.js"; +import { loadDotEnvVars } from "../../utilities/dotEnv.js"; +import { printStandloneInitialBanner } from "../../utilities/initialBanner.js"; +import { logger } from "../../utilities/logger.js"; +import { getProjectClient } from "../../utilities/session.js"; +import { getTmpDir } from "../../utilities/tempDirectories.js"; +import { spinner } from "../../utilities/windows.js"; +import { login } from "../login.js"; +import { updateTriggerPackages } from "../update.js"; +import { resolveAlwaysExternal } from "../../build/externals.js"; + +const WorkersBuildCommandOptions = CommonCommandOptions.extend({ + // docker build options + load: z.boolean().default(false), + platform: z.enum(["linux/amd64", "linux/arm64"]).default("linux/amd64"), + network: z.enum(["default", "none", "host"]).optional(), + tag: z.string().optional(), + push: z.boolean().default(false), + noCache: z.boolean().default(false), + // trigger options + local: z.boolean().default(false), // TODO: default to true when webapp has no remote build support + dryRun: z.boolean().default(false), + skipSyncEnvVars: z.boolean().default(false), + env: z.enum(["prod", "staging"]), + config: z.string().optional(), + projectRef: z.string().optional(), + apiUrl: z.string().optional(), + saveLogs: z.boolean().default(false), + skipUpdateCheck: z.boolean().default(false), + envFile: z.string().optional(), +}); + +type WorkersBuildCommandOptions = z.infer; + +type Deployment = InitializeDeploymentResponseBody; + +export function configureWorkersBuildCommand(program: Command) { + return commonOptions( + program + .command("build") + .description("Build a self-hosted worker image") + .argument("[path]", "The path to the project", ".") + .option( + "-e, --env ", + "Deploy to a specific environment (currently only prod and staging are supported)", + "prod" + ) + .option("--skip-update-check", "Skip checking for @trigger.dev package updates") + .option("-c, --config ", "The name of the config file, found at [path]") + .option( + "-p, --project-ref ", + "The project ref. Required if there is no config file. This will override the project specified in the config file." + ) + .option( + "--skip-sync-env-vars", + "Skip syncing environment variables when using the syncEnvVars extension." + ) + .option( + "--env-file ", + "Path to the .env file to load into the CLI process. Defaults to .env in the project directory." + ) + ) + .addOption( + new CommandOption( + "--dry-run", + "This will only create the build context without actually building the image. This can be useful for debugging." + ).hideHelp() + ) + .addOption( + new CommandOption( + "--no-cache", + "Do not use any build cache. This will significantly slow down the build process but can be useful to fix caching issues." + ).hideHelp() + ) + .option("--local", "Force building the image locally.") + .option("--push", "Push the image to the configured registry.") + .option( + "-t, --tag ", + "Specify the full name of the resulting image with an optional tag. The tag will always be overridden for remote builds." + ) + .option("--load", "Load the built image into your local docker") + .option( + "--network ", + "The networking mode for RUN instructions when using --local", + "host" + ) + .option( + "--platform ", + "The platform to build the deployment image for", + "linux/amd64" + ) + .option("--save-logs", "If provided, will save logs even for successful builds") + .action(async (path, options) => { + await handleTelemetry(async () => { + await printStandloneInitialBanner(true); + await workersBuildCommand(path, options); + }); + }); +} + +async function workersBuildCommand(dir: string, options: unknown) { + return await wrapCommandAction( + "workerBuildCommand", + WorkersBuildCommandOptions, + options, + async (opts) => { + return await _workerBuildCommand(dir, opts); + } + ); +} + +async function _workerBuildCommand(dir: string, options: WorkersBuildCommandOptions) { + intro("Building worker image"); + + if (!options.skipUpdateCheck) { + await updateTriggerPackages(dir, { ...options }, true, true); + } + + const projectPath = resolve(process.cwd(), dir); + + const authorization = await login({ + embedded: true, + defaultApiUrl: options.apiUrl, + profile: options.profile, + }); + + if (!authorization.ok) { + if (authorization.error === "fetch failed") { + throw new Error( + `Failed to connect to ${authorization.auth?.apiUrl}. Are you sure it's the correct URL?` + ); + } else { + throw new Error( + `You must login first. Use the \`login\` CLI command.\n\n${authorization.error}` + ); + } + } + + const resolvedConfig = await loadConfig({ + cwd: projectPath, + overrides: { project: options.projectRef }, + configFile: options.config, + }); + + logger.debug("Resolved config", resolvedConfig); + + const projectClient = await getProjectClient({ + accessToken: authorization.auth.accessToken, + apiUrl: authorization.auth.apiUrl, + projectRef: resolvedConfig.project, + env: options.env, + profile: options.profile, + }); + + if (!projectClient) { + throw new Error("Failed to get project client"); + } + + const serverEnvVars = await projectClient.client.getEnvironmentVariables(resolvedConfig.project); + loadDotEnvVars(resolvedConfig.workingDir, options.envFile); + + const destination = getTmpDir(resolvedConfig.workingDir, "build", options.dryRun); + + const $buildSpinner = spinner(); + + const forcedExternals = await resolveAlwaysExternal(projectClient.client); + + const buildManifest = await buildWorker({ + target: "unmanaged", + environment: options.env, + destination: destination.path, + resolvedConfig, + rewritePaths: true, + envVars: serverEnvVars.success ? serverEnvVars.data.variables : {}, + forcedExternals, + listener: { + onBundleStart() { + $buildSpinner.start("Building project"); + }, + onBundleComplete(result) { + $buildSpinner.stop("Successfully built project"); + + logger.debug("Bundle result", result); + }, + }, + }); + + logger.debug("Successfully built project to", destination.path); + + if (options.dryRun) { + logger.info(`Dry run complete. View the built project at ${destination.path}`); + return; + } + + const tagParts = parseDockerImageReference(options.tag ?? ""); + + // Account for empty strings to preserve existing behavior + const registry = tagParts.registry ? tagParts.registry : undefined; + const namespace = tagParts.repo ? tagParts.repo : undefined; + + const deploymentResponse = await projectClient.client.initializeDeployment({ + contentHash: buildManifest.contentHash, + userId: authorization.userId, + selfHosted: options.local, + registryHost: registry, + namespace: namespace, + type: "UNMANAGED", + }); + + if (!deploymentResponse.success) { + throw new Error(`Failed to start deployment: ${deploymentResponse.error}`); + } + + const deployment = deploymentResponse.data; + + let local = options.local; + + // If the deployment doesn't have any externalBuildData, then we can't use the remote image builder + if (!deployment.externalBuildData && !options.local) { + log.warn( + "This webapp instance does not support remote builds, falling back to local build. Please use the `--local` flag to skip this warning." + ); + local = true; + } + + if ( + buildManifest.deploy.sync && + buildManifest.deploy.sync.env && + Object.keys(buildManifest.deploy.sync.env).length > 0 + ) { + const numberOfEnvVars = Object.keys(buildManifest.deploy.sync.env).length; + const vars = numberOfEnvVars === 1 ? "var" : "vars"; + + if (!options.skipSyncEnvVars) { + const $spinner = spinner(); + $spinner.start(`Syncing ${numberOfEnvVars} env ${vars} with the server`); + const success = await syncEnvVarsWithServer( + projectClient.client, + resolvedConfig.project, + options.env, + buildManifest.deploy.sync.env + ); + + if (!success) { + await failDeploy( + projectClient.client, + deployment, + { + name: "SyncEnvVarsError", + message: `Failed to sync ${numberOfEnvVars} env ${vars} with the server`, + }, + "", + $spinner + ); + } else { + $spinner.stop(`Successfully synced ${numberOfEnvVars} env ${vars} with the server`); + } + } else { + logger.log( + "Skipping syncing env vars. The environment variables in your project have changed, but the --skip-sync-env-vars flag was provided." + ); + } + } + + const version = deployment.version; + + const deploymentLink = cliLink( + "View deployment", + `${authorization.dashboardUrl}/projects/v3/${resolvedConfig.project}/deployments/${deployment.shortCode}` + ); + + const testLink = cliLink( + "Test tasks", + `${authorization.dashboardUrl}/projects/v3/${resolvedConfig.project}/test?environment=${ + options.env === "prod" ? "prod" : "stg" + }` + ); + + const $spinner = spinner(); + + if (isLinksSupported) { + $spinner.start(`Building worker version ${version} ${deploymentLink}`); + } else { + $spinner.start(`Building worker version ${version}`); + } + + const buildResult = await buildImage({ + selfHosted: local, + buildPlatform: options.platform, + noCache: options.noCache, + push: options.push, + registryHost: registry, + registry: registry, + deploymentId: deployment.id, + deploymentVersion: deployment.version, + imageTag: deployment.imageTag, + loadImage: options.load, + contentHash: deployment.contentHash, + externalBuildId: deployment.externalBuildData?.buildId, + externalBuildToken: deployment.externalBuildData?.buildToken, + externalBuildProjectId: deployment.externalBuildData?.projectId, + projectId: projectClient.id, + projectRef: resolvedConfig.project, + apiUrl: projectClient.client.apiURL, + apiKey: projectClient.client.accessToken!, + authAccessToken: authorization.auth.accessToken, + compilationPath: destination.path, + buildEnvVars: buildManifest.build.env, + network: options.network, + }); + + logger.debug("Build result", buildResult); + + const warnings = checkLogsForWarnings(buildResult.logs); + + if (!warnings.ok) { + await failDeploy( + projectClient.client, + deployment, + { name: "BuildError", message: warnings.summary }, + buildResult.logs, + $spinner, + warnings.warnings, + warnings.errors + ); + + throw new SkipLoggingError("Failed to build image"); + } + + if (!buildResult.ok) { + await failDeploy( + projectClient.client, + deployment, + { name: "BuildError", message: buildResult.error }, + buildResult.logs, + $spinner, + warnings.warnings + ); + + throw new SkipLoggingError("Failed to build image"); + } + + // Index the deployment + // const runtime = new UnmanagedWorkerRuntime({ + // name: projectClient.name, + // config: resolvedConfig, + // args: { + // ...options, + // debugOtel: false, + // }, + // client: projectClient.client, + // dashboardUrl: authorization.dashboardUrl, + // }); + // await runtime.init(); + + // console.log("buildManifest", buildManifest); + + // await runtime.initializeWorker(buildManifest); + + const getDeploymentResponse = await projectClient.client.getDeployment(deployment.id); + + if (!getDeploymentResponse.success) { + await failDeploy( + projectClient.client, + deployment, + { name: "DeploymentError", message: getDeploymentResponse.error }, + buildResult.logs, + $spinner + ); + + throw new SkipLoggingError("Failed to get deployment with worker"); + } + + const deploymentWithWorker = getDeploymentResponse.data; + + if (!deploymentWithWorker.worker) { + await failDeploy( + projectClient.client, + deployment, + { name: "DeploymentError", message: "Failed to get deployment with worker" }, + buildResult.logs, + $spinner + ); + + throw new SkipLoggingError("Failed to get deployment with worker"); + } + + $spinner.stop(`Successfully built worker version ${version}`); + + const taskCount = deploymentWithWorker.worker?.tasks.length ?? 0; + + log.message(`Detected ${taskCount} task${taskCount === 1 ? "" : "s"}`); + + if (taskCount > 0) { + logger.table( + deploymentWithWorker.worker.tasks.map((task) => ({ + id: task.slug, + export: task.exportName, + path: task.filePath, + })) + ); + } + + outro( + `Version ${version} built and ready to deploy: ${buildResult.image} ${ + isLinksSupported ? `| ${deploymentLink} | ${testLink}` : "" + }` + ); +} + +export async function syncEnvVarsWithServer( + apiClient: CliApiClient, + projectRef: string, + environmentSlug: string, + envVars: Record +) { + const uploadResult = await apiClient.importEnvVars(projectRef, environmentSlug, { + variables: envVars, + override: true, + }); + + return uploadResult.success; +} + +async function failDeploy( + client: CliApiClient, + deployment: Deployment, + error: { name: string; message: string }, + logs: string, + $spinner: ReturnType, + warnings?: string[], + errors?: string[] +) { + $spinner.stop(`Failed to deploy project`); + + const doOutputLogs = async (prefix: string = "Error") => { + if (logs.trim() !== "") { + const logPath = await saveLogs(deployment.shortCode, logs); + + printWarnings(warnings); + printErrors(errors); + + checkLogsForErrors(logs); + + outro( + `${chalkError(`${prefix}:`)} ${ + error.message + }. Full build logs have been saved to ${logPath}` + ); + } else { + outro(`${chalkError(`${prefix}:`)} ${error.message}.`); + } + }; + + const exitCommand = (message: string) => { + throw new SkipLoggingError(message); + }; + + const deploymentResponse = await client.getDeployment(deployment.id); + + if (!deploymentResponse.success) { + logger.debug(`Failed to get deployment with worker: ${deploymentResponse.error}`); + } else { + const serverDeployment = deploymentResponse.data; + + switch (serverDeployment.status) { + case "PENDING": + case "DEPLOYING": + case "BUILDING": { + await doOutputLogs(); + + await client.failDeployment(deployment.id, { + error, + }); + + exitCommand("Failed to deploy project"); + + break; + } + case "CANCELED": { + await doOutputLogs("Canceled"); + + exitCommand("Failed to deploy project"); + + break; + } + case "FAILED": { + const errorData = serverDeployment.errorData + ? prepareDeploymentError(serverDeployment.errorData) + : undefined; + + if (errorData) { + prettyError(errorData.name, errorData.stack, errorData.stderr); + + if (logs.trim() !== "") { + const logPath = await saveLogs(deployment.shortCode, logs); + + outro(`Aborting deployment. Full build logs have been saved to ${logPath}`); + } else { + outro(`Aborting deployment`); + } + } else { + await doOutputLogs("Failed"); + } + + exitCommand("Failed to deploy project"); + + break; + } + case "DEPLOYED": { + await doOutputLogs("Deployed with errors"); + + exitCommand("Deployed with errors"); + + break; + } + case "TIMED_OUT": { + await doOutputLogs("TimedOut"); + + exitCommand("Timed out"); + + break; + } + } + } +} diff --git a/packages/cli-v3/src/commands/workers/create.ts b/packages/cli-v3/src/commands/workers/create.ts new file mode 100644 index 0000000000..9f93c7ad73 --- /dev/null +++ b/packages/cli-v3/src/commands/workers/create.ts @@ -0,0 +1,135 @@ +import { Command } from "commander"; +import { printStandloneInitialBanner } from "../../utilities/initialBanner.js"; +import { + CommonCommandOptions, + commonOptions, + handleTelemetry, + OutroCommandError, + wrapCommandAction, +} from "../../cli/common.js"; +import { login } from "../login.js"; +import { loadConfig } from "../../config.js"; +import { resolve } from "path"; +import { getProjectClient } from "../../utilities/session.js"; +import { logger } from "../../utilities/logger.js"; +import { z } from "zod"; +import { intro, isCancel, outro, text } from "@clack/prompts"; + +const WorkersCreateCommandOptions = CommonCommandOptions.extend({ + env: z.enum(["prod", "staging"]), + config: z.string().optional(), + projectRef: z.string().optional(), +}); +type WorkersCreateCommandOptions = z.infer; + +export function configureWorkersCreateCommand(program: Command) { + return commonOptions( + program + .command("create") + .description("List all available workers") + .argument("[path]", "The path to the project", ".") + .option( + "-e, --env ", + "Deploy to a specific environment (currently only prod and staging are supported)", + "prod" + ) + .option("-c, --config ", "The name of the config file, found at [path]") + .option( + "-p, --project-ref ", + "The project ref. Required if there is no config file. This will override the project specified in the config file." + ) + .action(async (path, options) => { + await handleTelemetry(async () => { + await printStandloneInitialBanner(true); + await workersCreateCommand(path, options); + }); + }) + ); +} + +async function workersCreateCommand(dir: string, options: unknown) { + return await wrapCommandAction( + "workerCreateCommand", + WorkersCreateCommandOptions, + options, + async (opts) => { + return await _workersCreateCommand(dir, opts); + } + ); +} + +async function _workersCreateCommand(dir: string, options: WorkersCreateCommandOptions) { + intro("Creating new worker group"); + + const authorization = await login({ + embedded: true, + defaultApiUrl: options.apiUrl, + profile: options.profile, + silent: true, + }); + + if (!authorization.ok) { + if (authorization.error === "fetch failed") { + throw new Error( + `Failed to connect to ${authorization.auth?.apiUrl}. Are you sure it's the correct URL?` + ); + } else { + throw new Error( + `You must login first. Use the \`login\` CLI command.\n\n${authorization.error}` + ); + } + } + + const projectPath = resolve(process.cwd(), dir); + + const resolvedConfig = await loadConfig({ + cwd: projectPath, + overrides: { project: options.projectRef }, + configFile: options.config, + }); + + logger.debug("Resolved config", resolvedConfig); + + const projectClient = await getProjectClient({ + accessToken: authorization.auth.accessToken, + apiUrl: authorization.auth.apiUrl, + projectRef: resolvedConfig.project, + env: options.env, + profile: options.profile, + }); + + if (!projectClient) { + throw new Error("Failed to get project client"); + } + + const name = await text({ + message: "What would you like to call the new worker?", + placeholder: "", + }); + + if (isCancel(name)) { + throw new OutroCommandError(); + } + + const description = await text({ + message: "What is the purpose of this worker?", + placeholder: "", + }); + + if (isCancel(description)) { + throw new OutroCommandError(); + } + + const newWorker = await projectClient.client.workers.create({ + name, + description, + }); + + if (!newWorker.success) { + throw new Error(`Failed to create worker: ${newWorker.error}`); + } + + outro( + `Successfully created worker ${newWorker.data.workerGroup.name} with token ${newWorker.data.token.plaintext}` + ); +} diff --git a/packages/cli-v3/src/commands/workers/index.ts b/packages/cli-v3/src/commands/workers/index.ts new file mode 100644 index 0000000000..84ef7ba2ef --- /dev/null +++ b/packages/cli-v3/src/commands/workers/index.ts @@ -0,0 +1,16 @@ +import { Command } from "commander"; +import { configureWorkersBuildCommand } from "./build.js"; +import { configureWorkersListCommand } from "./list.js"; +import { configureWorkersCreateCommand } from "./create.js"; +import { configureWorkersRunCommand } from "./run.js"; + +export function configureWorkersCommand(program: Command) { + const workers = program.command("workers").description("Subcommands for managing workers"); + + configureWorkersBuildCommand(workers); + configureWorkersListCommand(workers); + configureWorkersCreateCommand(workers); + configureWorkersRunCommand(workers); + + return workers; +} diff --git a/packages/cli-v3/src/commands/workers/list.ts b/packages/cli-v3/src/commands/workers/list.ts new file mode 100644 index 0000000000..ae4467a107 --- /dev/null +++ b/packages/cli-v3/src/commands/workers/list.ts @@ -0,0 +1,119 @@ +import { Command } from "commander"; +import { printStandloneInitialBanner } from "../../utilities/initialBanner.js"; +import { + CommonCommandOptions, + commonOptions, + handleTelemetry, + wrapCommandAction, +} from "../../cli/common.js"; +import { login } from "../login.js"; +import { loadConfig } from "../../config.js"; +import { resolve } from "path"; +import { getProjectClient } from "../../utilities/session.js"; +import { logger } from "../../utilities/logger.js"; +import { z } from "zod"; +import { intro } from "@clack/prompts"; + +const WorkersListCommandOptions = CommonCommandOptions.extend({ + env: z.enum(["prod", "staging"]), + config: z.string().optional(), + projectRef: z.string().optional(), +}); +type WorkersListCommandOptions = z.infer; + +export function configureWorkersListCommand(program: Command) { + return commonOptions( + program + .command("list") + .description("List all available workers") + .argument("[path]", "The path to the project", ".") + .option( + "-e, --env ", + "Deploy to a specific environment (currently only prod and staging are supported)", + "prod" + ) + .option("-c, --config ", "The name of the config file, found at [path]") + .option( + "-p, --project-ref ", + "The project ref. Required if there is no config file. This will override the project specified in the config file." + ) + .action(async (path, options) => { + await handleTelemetry(async () => { + await printStandloneInitialBanner(true); + await workersListCommand(path, options); + }); + }) + ); +} + +async function workersListCommand(dir: string, options: unknown) { + return await wrapCommandAction( + "workerListCommand", + WorkersListCommandOptions, + options, + async (opts) => { + return await _workersListCommand(dir, opts); + } + ); +} + +async function _workersListCommand(dir: string, options: WorkersListCommandOptions) { + intro("Listing workers"); + + const authorization = await login({ + embedded: true, + defaultApiUrl: options.apiUrl, + profile: options.profile, + silent: true, + }); + + if (!authorization.ok) { + if (authorization.error === "fetch failed") { + throw new Error( + `Failed to connect to ${authorization.auth?.apiUrl}. Are you sure it's the correct URL?` + ); + } else { + throw new Error( + `You must login first. Use the \`login\` CLI command.\n\n${authorization.error}` + ); + } + } + + const projectPath = resolve(process.cwd(), dir); + + const resolvedConfig = await loadConfig({ + cwd: projectPath, + overrides: { project: options.projectRef }, + configFile: options.config, + }); + + logger.debug("Resolved config", resolvedConfig); + + const projectClient = await getProjectClient({ + accessToken: authorization.auth.accessToken, + apiUrl: authorization.auth.apiUrl, + projectRef: resolvedConfig.project, + env: options.env, + profile: options.profile, + }); + + if (!projectClient) { + throw new Error("Failed to get project client"); + } + + const workers = await projectClient.client.workers.list(); + + if (!workers.success) { + throw new Error(`Failed to list workers: ${workers.error}`); + } + + logger.table( + workers.data.map((worker) => ({ + default: worker.isDefault ? "x" : "-", + type: worker.type, + name: worker.name, + description: worker.description ?? "-", + "updated at": worker.updatedAt.toLocaleString(), + })) + ); +} diff --git a/packages/cli-v3/src/commands/workers/run.ts b/packages/cli-v3/src/commands/workers/run.ts new file mode 100644 index 0000000000..f4e9ab7c55 --- /dev/null +++ b/packages/cli-v3/src/commands/workers/run.ts @@ -0,0 +1,151 @@ +import { Command } from "commander"; +import { printStandloneInitialBanner } from "../../utilities/initialBanner.js"; +import { + CommonCommandOptions, + commonOptions, + handleTelemetry, + wrapCommandAction, +} from "../../cli/common.js"; +import { login } from "../login.js"; +import { loadConfig } from "../../config.js"; +import { resolve } from "path"; +import { getProjectClient } from "../../utilities/session.js"; +import { logger } from "../../utilities/logger.js"; +import { z } from "zod"; +import { env } from "std-env"; +import { x } from "tinyexec"; + +const WorkersRunCommandOptions = CommonCommandOptions.extend({ + env: z.enum(["prod", "staging"]), + config: z.string().optional(), + projectRef: z.string().optional(), + token: z.string().default(env.TRIGGER_WORKER_TOKEN ?? ""), + network: z.enum(["default", "none", "host"]).default("default"), +}); +type WorkersRunCommandOptions = z.infer; + +export function configureWorkersRunCommand(program: Command) { + return commonOptions( + program + .command("run") + .description("Runs a worker locally") + .argument("[path]", "The path to the project", ".") + .option( + "-e, --env ", + "Deploy to a specific environment (currently only prod and staging are supported)", + "prod" + ) + .option("-c, --config ", "The name of the config file, found at [path]") + .option( + "-p, --project-ref ", + "The project ref. Required if there is no config file. This will override the project specified in the config file." + ) + .option("-t, --token ", "The worker token to use for authentication") + .option("--network ", "The networking mode for the container", "host") + .action(async (path, options) => { + await handleTelemetry(async () => { + await printStandloneInitialBanner(true); + await workersRunCommand(path, options); + }); + }) + ); +} + +async function workersRunCommand(dir: string, options: unknown) { + return await wrapCommandAction( + "workerRunCommand", + WorkersRunCommandOptions, + options, + async (opts) => { + return await _workersRunCommand(dir, opts); + } + ); +} + +async function _workersRunCommand(dir: string, options: WorkersRunCommandOptions) { + if (!options.token) { + throw new Error( + "You must provide a worker token to run a worker locally. Either use the `--token` flag or set the `TRIGGER_WORKER_TOKEN` environment variable." + ); + } + + logger.log("Running worker locally"); + + const authorization = await login({ + embedded: true, + defaultApiUrl: options.apiUrl, + profile: options.profile, + silent: true, + }); + + if (!authorization.ok) { + if (authorization.error === "fetch failed") { + throw new Error( + `Failed to connect to ${authorization.auth?.apiUrl}. Are you sure it's the correct URL?` + ); + } else { + throw new Error( + `You must login first. Use the \`login\` CLI command.\n\n${authorization.error}` + ); + } + } + + const projectPath = resolve(process.cwd(), dir); + + const resolvedConfig = await loadConfig({ + cwd: projectPath, + overrides: { project: options.projectRef }, + configFile: options.config, + }); + + logger.debug("Resolved config", resolvedConfig); + + const projectClient = await getProjectClient({ + accessToken: authorization.auth.accessToken, + apiUrl: authorization.auth.apiUrl, + projectRef: resolvedConfig.project, + env: options.env, + profile: options.profile, + }); + + if (!projectClient) { + throw new Error("Failed to get project client"); + } + + const deployment = await projectClient.client.deployments.unmanaged.latest(); + + if (!deployment.success) { + throw new Error("Failed to get latest deployment"); + } + + const { version, imageReference } = deployment.data; + + if (!imageReference) { + throw new Error("No image reference found for the latest deployment"); + } + + logger.log(`Version ${version}`); + logger.log(`Image: ${imageReference}`); + + const command = "docker"; + const args = [ + "run", + "--rm", + "--network", + options.network, + "-e", + `TRIGGER_WORKER_TOKEN=${options.token}`, + "-e", + `TRIGGER_API_URL=${authorization.auth.apiUrl}`, + imageReference, + ]; + + logger.debug(`Command: ${command} ${args.join(" ")}`); + logger.log(); // spacing + + const proc = x("docker", args); + + for await (const line of proc) { + logger.log(line); + } +} diff --git a/packages/cli-v3/src/config.ts b/packages/cli-v3/src/config.ts index 3283c15eb6..3a7235cdc3 100644 --- a/packages/cli-v3/src/config.ts +++ b/packages/cli-v3/src/config.ts @@ -1,4 +1,10 @@ -import { ResolveEnvironmentVariablesFunction, TriggerConfig } from "@trigger.dev/core/v3"; +import { + BuildRuntime, + CompatibilityFlag, + CompatibilityFlagFeatures, + ResolveEnvironmentVariablesFunction, + TriggerConfig, +} from "@trigger.dev/core/v3"; import { DEFAULT_RUNTIME, ResolvedConfig } from "@trigger.dev/core/v3/build"; import * as c12 from "c12"; import { defu } from "defu"; @@ -131,6 +137,12 @@ export function configPlugin(resolvedConfig: ResolvedConfig): esbuild.Plugin | u }; } +function featuresFromCompatibilityFlags(flags: CompatibilityFlag[]): CompatibilityFlagFeatures { + return { + run_engine_v2: flags.includes("run_engine_v2"), + }; +} + async function resolveConfig( cwd: string, result: c12.ResolvedConfig, @@ -157,6 +169,10 @@ async function resolveConfig( dirs = dirs.map((dir) => (isAbsolute(dir) ? relative(workingDir, dir) : dir)); + const features = featuresFromCompatibilityFlags(config.compatibilityFlags ?? []); + + const defaultRuntime: BuildRuntime = features.run_engine_v2 ? "node-22" : DEFAULT_RUNTIME; + const mergedConfig = defu( { workingDir: packageJsonPath ? dirname(packageJsonPath) : cwd, @@ -170,7 +186,7 @@ async function resolveConfig( config, { dirs, - runtime: DEFAULT_RUNTIME, + runtime: defaultRuntime, tsconfig: tsconfigPath, build: { jsx: { @@ -182,6 +198,8 @@ async function resolveConfig( external: [], conditions: [], }, + compatibilityFlags: [], + features, } ) as ResolvedConfig; // TODO: For some reason, without this, there is a weird type error complaining about tsconfigPath being string | nullish, which can't be assigned to string | undefined @@ -189,6 +207,7 @@ async function resolveConfig( ...mergedConfig, dirs: Array.from(new Set(mergedConfig.dirs)), instrumentedPackageNames: getInstrumentedPackageNames(mergedConfig), + runtime: mergedConfig.runtime, }; } diff --git a/packages/cli-v3/src/deploy/buildImage.ts b/packages/cli-v3/src/deploy/buildImage.ts index 62c2da2844..60d959404d 100644 --- a/packages/cli-v3/src/deploy/buildImage.ts +++ b/packages/cli-v3/src/deploy/buildImage.ts @@ -20,7 +20,7 @@ export interface BuildImageOptions { loadImage?: boolean; // Flattened properties from nested structures - registryHost: string; + registryHost?: string; authAccessToken: string; imageTag: string; deploymentId: string; @@ -41,51 +41,50 @@ export interface BuildImageOptions { deploymentSpinner?: any; // Replace 'any' with the actual type if known } -export async function buildImage(options: BuildImageOptions) { - const { - selfHosted, - buildPlatform, - noCache, - push, - registry, - loadImage, - registryHost, - authAccessToken, - imageTag, - deploymentId, - deploymentVersion, - contentHash, - externalBuildId, - externalBuildToken, - externalBuildProjectId, - compilationPath, - projectId, - projectRef, - extraCACerts, - apiUrl, - apiKey, - buildEnvVars, - } = options; - +export async function buildImage({ + selfHosted, + buildPlatform, + noCache, + push, + registry, + loadImage, + registryHost, + authAccessToken, + imageTag, + deploymentId, + deploymentVersion, + contentHash, + externalBuildId, + externalBuildToken, + externalBuildProjectId, + compilationPath, + projectId, + projectRef, + extraCACerts, + apiUrl, + apiKey, + buildEnvVars, + network, +}: BuildImageOptions) { if (selfHosted) { return selfHostedBuildImage({ - registryHost: registryHost, - imageTag: imageTag, + registryHost, + imageTag, cwd: compilationPath, - projectId: projectId, - deploymentId: deploymentId, - deploymentVersion: deploymentVersion, - contentHash: contentHash, - projectRef: projectRef, + projectId, + deploymentId, + deploymentVersion, + contentHash, + projectRef, buildPlatform: buildPlatform, pushImage: push, selfHostedRegistry: !!registry, - noCache: noCache, - extraCACerts: extraCACerts, + noCache, + extraCACerts, apiUrl, apiKey, buildEnvVars, - network: options.network, + network, }); } @@ -95,6 +94,12 @@ export async function buildImage(options: BuildImageOptions) { ); } + if (!registryHost) { + throw new Error( + "Failed to initialize deployment. The deployment does not have a registry host. To deploy this project, you must use the --self-hosted or --local flag to build and push the image yourself." + ); + } + return depotBuildImage({ registryHost, auth: authAccessToken, @@ -263,7 +268,7 @@ async function depotBuildImage(options: DepotBuildImageOptions): Promise = { + bun: "imbios/bun-node:1.1.24-22-slim@sha256:eec3c2937e30c579258a92c60847e5515488513337b23ec13996228f6a146ff5", + node: "node:21.7.3-bookworm-slim@sha256:dfc05dee209a1d7adf2ef189bd97396daad4e97c6eaa85778d6f75205ba1b0fb", + "node-22": + "node:22.12.0-bookworm-slim@sha256:a4b757cd491c7f0b57f57951f35f4e85b7e1ad54dbffca4cf9af0725e1650cd8", +}; + const DEFAULT_PACKAGES = ["busybox", "ca-certificates", "dumb-init", "git", "openssl"]; export async function generateContainerfile(options: GenerateContainerfileOptions) { switch (options.runtime) { - case "node": { + case "node": + case "node-22": { return await generateNodeContainerfile(options); } case "bun": { @@ -447,7 +460,7 @@ export async function generateContainerfile(options: GenerateContainerfileOption } } -async function generateBunContainerfile(options: GenerateContainerfileOptions) { +const parseGenerateOptions = (options: GenerateContainerfileOptions) => { const buildArgs = Object.entries(options.build.env || {}) .flatMap(([key]) => `ARG ${key}`) .join("\n"); @@ -463,19 +476,38 @@ async function generateBunContainerfile(options: GenerateContainerfileOptions) { " " ); + return { + baseImage: BASE_IMAGE[options.runtime], + baseInstructions, + buildArgs, + buildEnvVars, + packages, + postInstallCommands, + }; +}; + +async function generateBunContainerfile(options: GenerateContainerfileOptions) { + const { baseImage, buildArgs, buildEnvVars, postInstallCommands, baseInstructions, packages } = + parseGenerateOptions(options); + return `# syntax=docker/dockerfile:1 -FROM imbios/bun-node:1.1.24-22-slim@sha256:9cfb7cd87529261c482fe17d8894c0986263f3a5ccf84ad65c00ec0e1ed539c6 AS base +FROM ${baseImage} AS base ${baseInstructions} ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get --fix-broken install -y && apt-get install -y --no-install-recommends ${packages} && apt-get clean && rm -rf /var/lib/apt/lists/* +RUN apt-get update && \ + apt-get --fix-broken install -y && \ + apt-get install -y --no-install-recommends ${packages} && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* FROM base AS build -RUN apt-get update && apt-get install -y --no-install-recommends \ - python3 make g++ && \ - apt-get clean && rm -rf /var/lib/apt/lists/* +RUN apt-get update && \ + apt-get install -y --no-install-recommends python3 make g++ && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* USER bun WORKDIR /app @@ -553,35 +585,27 @@ CMD [] } async function generateNodeContainerfile(options: GenerateContainerfileOptions) { - const buildArgs = Object.entries(options.build.env || {}) - .flatMap(([key]) => `ARG ${key}`) - .join("\n"); - - const buildEnvVars = Object.entries(options.build.env || {}) - .flatMap(([key]) => `ENV ${key}=$${key}`) - .join("\n"); - - const postInstallCommands = (options.build.commands || []).map((cmd) => `RUN ${cmd}`).join("\n"); - - const baseInstructions = (options.image?.instructions || []).join("\n"); - const packages = Array.from(new Set(DEFAULT_PACKAGES.concat(options.image?.pkgs || []))).join( - " " - ); + const { baseImage, buildArgs, buildEnvVars, postInstallCommands, baseInstructions, packages } = + parseGenerateOptions(options); return `# syntax=docker/dockerfile:1 -FROM node:21-bookworm-slim@sha256:99afef5df7400a8d118e0504576d32ca700de5034c4f9271d2ff7c91cc12d170 AS base +FROM ${baseImage} AS base ${baseInstructions} ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get --fix-broken install -y && apt-get install -y --no-install-recommends ${packages} && apt-get clean && rm -rf /var/lib/apt/lists/* +RUN apt-get update && \ + apt-get --fix-broken install -y && \ + apt-get install -y --no-install-recommends ${packages} && \ + apt-get clean && rm -rf /var/lib/apt/lists/* FROM base AS build # Install build dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - python3 make g++ && \ - apt-get clean && rm -rf /var/lib/apt/lists/* +RUN apt-get update && \ + apt-get install -y --no-install-recommends python3 make g++ && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* USER node WORKDIR /app diff --git a/packages/cli-v3/src/dev/devSession.ts b/packages/cli-v3/src/dev/devSession.ts index 2de079ffbc..6d9e1529fe 100644 --- a/packages/cli-v3/src/dev/devSession.ts +++ b/packages/cli-v3/src/dev/devSession.ts @@ -1,11 +1,10 @@ -import { CORE_VERSION } from "@trigger.dev/core/v3"; -import { DEFAULT_RUNTIME, ResolvedConfig } from "@trigger.dev/core/v3/build"; -import { BuildManifest } from "@trigger.dev/core/v3/schemas"; +import { ResolvedConfig } from "@trigger.dev/core/v3/build"; import * as esbuild from "esbuild"; import { CliApiClient } from "../apiClient.js"; import { BundleResult, bundleWorker, + createBuildManifestFromBundle, getBundleResultFromBuild, logBuildFailure, logBuildWarnings, @@ -17,14 +16,10 @@ import { resolvePluginsForContext, } from "../build/extensions.js"; import { createExternalsBuildExtension, resolveAlwaysExternal } from "../build/externals.js"; -import { copyManifestToDir } from "../build/manifests.js"; -import { devIndexWorker, devRunWorker, telemetryEntryPoint } from "../build/packageModules.js"; import { type DevCommandOptions } from "../commands/dev.js"; import { eventBus } from "../utilities/eventBus.js"; import { logger } from "../utilities/logger.js"; -import { resolveFileSources } from "../utilities/sourceFiles.js"; import { EphemeralDirectory, getTmpDir } from "../utilities/tempDirectories.js"; -import { VERSION } from "../version.js"; import { startDevOutput } from "./devOutput.js"; import { startWorkerRuntime } from "./workerRuntime.js"; @@ -82,12 +77,14 @@ export async function startDevSession({ const pluginsFromExtensions = resolvePluginsForContext(buildContext); async function updateBundle(bundle: BundleResult, workerDir?: EphemeralDirectory) { - let buildManifest = await createBuildManifestFromBundle( + let buildManifest = await createBuildManifestFromBundle({ bundle, - destination.path, - rawConfig, - workerDir?.path - ); + destination: destination.path, + resolvedConfig: rawConfig, + workerDir: workerDir?.path, + environment: "dev", + target: "dev", + }); logger.debug("Created build manifest from bundle", { buildManifest }); @@ -182,45 +179,3 @@ export async function startDevSession({ }, }; } - -async function createBuildManifestFromBundle( - bundle: BundleResult, - destination: string, - resolvedConfig: ResolvedConfig, - workerDir: string | undefined -): Promise { - const buildManifest: BuildManifest = { - contentHash: bundle.contentHash, - runtime: resolvedConfig.runtime ?? DEFAULT_RUNTIME, - cliPackageVersion: VERSION, - packageVersion: CORE_VERSION, - environment: "dev", - target: "dev", - files: bundle.files, - sources: await resolveFileSources(bundle.files, resolvedConfig), - externals: [], - config: { - project: resolvedConfig.project, - dirs: resolvedConfig.dirs, - }, - outputPath: destination, - runWorkerEntryPoint: bundle.runWorkerEntryPoint ?? devRunWorker, - indexWorkerEntryPoint: bundle.indexWorkerEntryPoint ?? devIndexWorker, - loaderEntryPoint: bundle.loaderEntryPoint, - configPath: bundle.configPath, - customConditions: resolvedConfig.build.conditions ?? [], - deploy: { - env: {}, - }, - build: {}, - otelImportHook: { - include: resolvedConfig.instrumentedPackageNames ?? [], - }, - }; - - if (!workerDir) { - return buildManifest; - } - - return copyManifestToDir(buildManifest, destination, workerDir); -} diff --git a/packages/cli-v3/src/entryPoints/deploy-index-worker.ts b/packages/cli-v3/src/entryPoints/deploy-index-worker.ts index 73de86535d..491f8973ae 100644 --- a/packages/cli-v3/src/entryPoints/deploy-index-worker.ts +++ b/packages/cli-v3/src/entryPoints/deploy-index-worker.ts @@ -169,7 +169,7 @@ await sendMessageInCatalog( "TASKS_FAILED_TO_PARSE", { zodIssues: err.error.issues, tasks }, async (msg) => { - await process.send?.(msg); + process.send?.(msg); } ); } else { diff --git a/packages/cli-v3/src/entryPoints/deploy-run-controller.ts b/packages/cli-v3/src/entryPoints/deploy-run-controller.ts index e17a152241..8b082a03f2 100644 --- a/packages/cli-v3/src/entryPoints/deploy-run-controller.ts +++ b/packages/cli-v3/src/entryPoints/deploy-run-controller.ts @@ -879,7 +879,10 @@ class ProdWorker { this._taskRunProcess = new TaskRunProcess({ workerManifest: this.workerManifest, env, - serverWorker: execution.worker, + serverWorker: { + ...execution.worker, + engine: "V1", + }, payload: createAttempt.result.executionPayload, messageId: message.lazyPayload.messageId, }); diff --git a/packages/cli-v3/src/entryPoints/deploy-run-worker.ts b/packages/cli-v3/src/entryPoints/deploy-run-worker.ts index 59ac12bb62..daa10c6975 100644 --- a/packages/cli-v3/src/entryPoints/deploy-run-worker.ts +++ b/packages/cli-v3/src/entryPoints/deploy-run-worker.ts @@ -18,8 +18,8 @@ import { waitUntil, apiClientManager, } from "@trigger.dev/core/v3"; -import { TriggerTracer } from "@trigger.dev/core/v3/tracer"; import { ProdRuntimeManager } from "@trigger.dev/core/v3/prod"; +import { TriggerTracer } from "@trigger.dev/core/v3/tracer"; import { ConsoleInterceptor, DevUsageManager, @@ -40,11 +40,11 @@ import { } from "@trigger.dev/core/v3/workers"; import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc"; import { readFile } from "node:fs/promises"; +import { setInterval, setTimeout } from "node:timers/promises"; import sourceMapSupport from "source-map-support"; -import { VERSION } from "../version.js"; -import { setTimeout, setInterval } from "node:timers/promises"; import { env } from "std-env"; import { normalizeImportPath } from "../utilities/normalizeImportPath.js"; +import { VERSION } from "../version.js"; sourceMapSupport.install({ handleUncaughtExceptions: false, @@ -476,7 +476,7 @@ const prodRuntimeManager = new ProdRuntimeManager(zodIpc, { runtime.setGlobalRuntimeManager(prodRuntimeManager); -process.title = "trigger-dev-worker"; +process.title = "trigger-deploy-worker"; const heartbeatInterval = parseInt(heartbeatIntervalMs ?? "30000", 10); diff --git a/packages/cli-v3/src/entryPoints/dev-index-worker.ts b/packages/cli-v3/src/entryPoints/dev-index-worker.ts index 9e6e8e05e9..2ef18444eb 100644 --- a/packages/cli-v3/src/entryPoints/dev-index-worker.ts +++ b/packages/cli-v3/src/entryPoints/dev-index-worker.ts @@ -153,7 +153,7 @@ await sendMessageInCatalog( "TASKS_FAILED_TO_PARSE", { zodIssues: err.error.issues, tasks }, async (msg) => { - await process.send?.(msg); + process.send?.(msg); } ); } else { diff --git a/packages/cli-v3/src/entryPoints/dev-run-worker.ts b/packages/cli-v3/src/entryPoints/dev-run-worker.ts index 5d3052de1e..18e44ff4c4 100644 --- a/packages/cli-v3/src/entryPoints/dev-run-worker.ts +++ b/packages/cli-v3/src/entryPoints/dev-run-worker.ts @@ -18,8 +18,8 @@ import { waitUntil, apiClientManager, } from "@trigger.dev/core/v3"; -import { TriggerTracer } from "@trigger.dev/core/v3/tracer"; import { DevRuntimeManager } from "@trigger.dev/core/v3/dev"; +import { TriggerTracer } from "@trigger.dev/core/v3/tracer"; import { ConsoleInterceptor, DevUsageManager, @@ -40,9 +40,9 @@ import { import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc"; import { readFile } from "node:fs/promises"; import sourceMapSupport from "source-map-support"; -import { VERSION } from "../version.js"; import { env } from "std-env"; import { normalizeImportPath } from "../utilities/normalizeImportPath.js"; +import { VERSION } from "../version.js"; sourceMapSupport.install({ handleUncaughtExceptions: false, diff --git a/packages/cli-v3/src/entryPoints/managed-index-controller.ts b/packages/cli-v3/src/entryPoints/managed-index-controller.ts new file mode 100644 index 0000000000..5822acecd5 --- /dev/null +++ b/packages/cli-v3/src/entryPoints/managed-index-controller.ts @@ -0,0 +1,117 @@ +import { + BuildManifest, + CreateBackgroundWorkerRequestBody, + serializeIndexingError, +} from "@trigger.dev/core/v3"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { env } from "std-env"; +import { CliApiClient } from "../apiClient.js"; +import { indexWorkerManifest } from "../indexing/indexWorkerManifest.js"; +import { resolveSourceFiles } from "../utilities/sourceFiles.js"; +import { execOptionsForRuntime } from "@trigger.dev/core/v3/build"; + +async function loadBuildManifest() { + const manifestContents = await readFile("./build.json", "utf-8"); + const raw = JSON.parse(manifestContents); + + return BuildManifest.parse(raw); +} + +async function bootstrap() { + const buildManifest = await loadBuildManifest(); + + if (typeof env.TRIGGER_API_URL !== "string") { + console.error("TRIGGER_API_URL is not set"); + process.exit(1); + } + + const cliApiClient = new CliApiClient(env.TRIGGER_API_URL, env.TRIGGER_SECRET_KEY); + + if (!env.TRIGGER_PROJECT_REF) { + console.error("TRIGGER_PROJECT_REF is not set"); + process.exit(1); + } + + if (!env.TRIGGER_DEPLOYMENT_ID) { + console.error("TRIGGER_DEPLOYMENT_ID is not set"); + process.exit(1); + } + + return { + buildManifest, + cliApiClient, + projectRef: env.TRIGGER_PROJECT_REF, + deploymentId: env.TRIGGER_DEPLOYMENT_ID, + }; +} + +type BootstrapResult = Awaited>; + +async function indexDeployment({ + cliApiClient, + projectRef, + deploymentId, + buildManifest, +}: BootstrapResult) { + const stdout: string[] = []; + const stderr: string[] = []; + + try { + const $env = await cliApiClient.getEnvironmentVariables(projectRef); + + if (!$env.success) { + throw new Error(`Failed to fetch environment variables: ${$env.error}`); + } + + const workerManifest = await indexWorkerManifest({ + runtime: buildManifest.runtime, + indexWorkerPath: buildManifest.indexWorkerEntryPoint, + buildManifestPath: "./build.json", + nodeOptions: execOptionsForRuntime(buildManifest.runtime, buildManifest), + env: $env.data.variables, + otelHookExclude: buildManifest.otelImportHook?.exclude, + otelHookInclude: buildManifest.otelImportHook?.include, + handleStdout(data) { + stdout.push(data); + }, + handleStderr(data) { + if (!data.includes("DeprecationWarning")) { + stderr.push(data); + } + }, + }); + + console.log("Writing index.json", process.cwd()); + + await writeFile(join(process.cwd(), "index.json"), JSON.stringify(workerManifest, null, 2)); + + const sourceFiles = resolveSourceFiles(buildManifest.sources, workerManifest.tasks); + + const backgroundWorkerBody: CreateBackgroundWorkerRequestBody = { + localOnly: true, + metadata: { + contentHash: buildManifest.contentHash, + packageVersion: buildManifest.packageVersion, + cliPackageVersion: buildManifest.cliPackageVersion, + tasks: workerManifest.tasks, + sourceFiles, + }, + supportsLazyAttempts: true, + }; + + await cliApiClient.createDeploymentBackgroundWorker(deploymentId, backgroundWorkerBody); + } catch (error) { + const serialiedIndexError = serializeIndexingError(error, stderr.join("\n")); + + console.error("Failed to index deployment", serialiedIndexError); + + await cliApiClient.failDeployment(deploymentId, { error: serialiedIndexError }); + + process.exit(1); + } +} + +const results = await bootstrap(); + +await indexDeployment(results); diff --git a/packages/cli-v3/src/entryPoints/managed-index-worker.ts b/packages/cli-v3/src/entryPoints/managed-index-worker.ts new file mode 100644 index 0000000000..2ef18444eb --- /dev/null +++ b/packages/cli-v3/src/entryPoints/managed-index-worker.ts @@ -0,0 +1,170 @@ +import { + BuildManifest, + type HandleErrorFunction, + indexerToWorkerMessages, + taskCatalog, + type TaskManifest, + TriggerConfig, +} from "@trigger.dev/core/v3"; +import { + StandardTaskCatalog, + TracingDiagnosticLogLevel, + TracingSDK, +} from "@trigger.dev/core/v3/workers"; +import { sendMessageInCatalog, ZodSchemaParsedError } from "@trigger.dev/core/v3/zodMessageHandler"; +import { readFile } from "node:fs/promises"; +import sourceMapSupport from "source-map-support"; +import { registerTasks } from "../indexing/registerTasks.js"; +import { env } from "std-env"; +import { normalizeImportPath } from "../utilities/normalizeImportPath.js"; + +sourceMapSupport.install({ + handleUncaughtExceptions: false, + environment: "node", + hookRequire: false, +}); + +process.on("uncaughtException", function (error, origin) { + if (error instanceof Error) { + process.send && + process.send({ + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { name: error.name, message: error.message, stack: error.stack }, + origin, + }, + version: "v1", + }); + } else { + process.send && + process.send({ + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { + name: "Error", + message: typeof error === "string" ? error : JSON.stringify(error), + }, + origin, + }, + version: "v1", + }); + } +}); + +taskCatalog.setGlobalTaskCatalog(new StandardTaskCatalog()); + +async function importConfig( + configPath: string +): Promise<{ config: TriggerConfig; handleError?: HandleErrorFunction }> { + const configModule = await import(normalizeImportPath(configPath)); + + const config = configModule?.default ?? configModule?.config; + + return { + config, + handleError: configModule?.handleError, + }; +} + +async function loadBuildManifest() { + const manifestContents = await readFile(env.TRIGGER_BUILD_MANIFEST_PATH!, "utf-8"); + const raw = JSON.parse(manifestContents); + + return BuildManifest.parse(raw); +} + +async function bootstrap() { + const buildManifest = await loadBuildManifest(); + + const { config } = await importConfig(buildManifest.configPath); + + // This needs to run or the PrismaInstrumentation will throw an error + const tracingSDK = new TracingSDK({ + url: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318", + instrumentations: config.instrumentations ?? [], + diagLogLevel: (env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none", + forceFlushTimeoutMillis: 30_000, + }); + + const importErrors = await registerTasks(buildManifest); + + return { + tracingSDK, + config, + buildManifest, + importErrors, + }; +} + +const { buildManifest, importErrors, config } = await bootstrap(); + +let tasks = taskCatalog.listTaskManifests(); + +// If the config has retry defaults, we need to apply them to all tasks that don't have any retry settings +if (config.retries?.default) { + tasks = tasks.map((task) => { + if (!task.retry) { + return { + ...task, + retry: config.retries?.default, + } satisfies TaskManifest; + } + + return task; + }); +} + +// If the config has a maxDuration, we need to apply it to all tasks that don't have a maxDuration +if (typeof config.maxDuration === "number") { + tasks = tasks.map((task) => { + if (typeof task.maxDuration !== "number") { + return { + ...task, + maxDuration: config.maxDuration, + } satisfies TaskManifest; + } + + return task; + }); +} + +await sendMessageInCatalog( + indexerToWorkerMessages, + "INDEX_COMPLETE", + { + manifest: { + tasks, + configPath: buildManifest.configPath, + runtime: buildManifest.runtime, + workerEntryPoint: buildManifest.runWorkerEntryPoint, + controllerEntryPoint: buildManifest.runControllerEntryPoint, + loaderEntryPoint: buildManifest.loaderEntryPoint, + customConditions: buildManifest.customConditions, + }, + importErrors, + }, + async (msg) => { + process.send?.(msg); + } +).catch((err) => { + if (err instanceof ZodSchemaParsedError) { + return sendMessageInCatalog( + indexerToWorkerMessages, + "TASKS_FAILED_TO_PARSE", + { zodIssues: err.error.issues, tasks }, + async (msg) => { + process.send?.(msg); + } + ); + } else { + console.error("Failed to send TASKS_READY message", err); + } + + return; +}); + +await new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, 10); +}); diff --git a/packages/cli-v3/src/entryPoints/managed-run-controller.ts b/packages/cli-v3/src/entryPoints/managed-run-controller.ts new file mode 100644 index 0000000000..9394b97c15 --- /dev/null +++ b/packages/cli-v3/src/entryPoints/managed-run-controller.ts @@ -0,0 +1,707 @@ +import { logger } from "../utilities/logger.js"; +import { OnWaitMessage, TaskRunProcess } from "../executions/taskRunProcess.js"; +import { env as stdEnv } from "std-env"; +import { z } from "zod"; +import { randomUUID } from "crypto"; +import { readJSONFile } from "../utilities/fileSystem.js"; +import { + DequeuedMessage, + HeartbeatService, + RunExecutionData, + WorkerManifest, +} from "@trigger.dev/core/v3"; +import { + WORKLOAD_HEADER_NAME, + WorkloadClientToServerEvents, + WorkloadHttpClient, + WorkloadServerToClientEvents, + type WorkloadRunAttemptStartResponseBody, +} from "@trigger.dev/worker"; +import { assertExhaustive } from "../utilities/assertExhaustive.js"; +import { setTimeout as wait } from "timers/promises"; +import { io, Socket } from "socket.io-client"; + +// All IDs are friendly IDs +const Env = z.object({ + // Set at build time + TRIGGER_CONTENT_HASH: z.string(), + TRIGGER_DEPLOYMENT_ID: z.string(), + TRIGGER_DEPLOYMENT_VERSION: z.string(), + TRIGGER_PROJECT_ID: z.string(), + TRIGGER_PROJECT_REF: z.string(), + NODE_ENV: z.string().default("production"), + NODE_EXTRA_CA_CERTS: z.string().optional(), + + // Set at runtime + TRIGGER_WORKER_API_URL: z.string().url(), + TRIGGER_WORKLOAD_CONTROLLER_ID: z.string().default(`controller_${randomUUID()}`), + TRIGGER_ENV_ID: z.string(), + TRIGGER_RUN_ID: z.string().optional(), // This is only useful for cold starts + TRIGGER_SNAPSHOT_ID: z.string().optional(), // This is only useful for cold starts + OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(), + TRIGGER_WARM_START_URL: z.string().optional(), + TRIGGER_MACHINE_CPU: z.string().default("0"), + TRIGGER_MACHINE_MEMORY: z.string().default("0"), +}); + +const env = Env.parse(stdEnv); + +logger.loggerLevel = "debug"; + +type ManagedRunControllerOptions = { + workerManifest: WorkerManifest; + heartbeatIntervalSeconds?: number; +}; + +type Run = { + friendlyId: string; +}; + +type Snapshot = { + friendlyId: string; +}; + +class ManagedRunController { + private taskRunProcess?: TaskRunProcess; + + private workerManifest: WorkerManifest; + + private readonly httpClient: WorkloadHttpClient; + + private socket?: Socket; + + private readonly heartbeatService: HeartbeatService; + private readonly heartbeatIntervalSeconds: number; + + private readonly snapshotPollService: HeartbeatService; + private readonly snapshotPollIntervalSeconds: number; + + private state: + | { + phase: "RUN"; + run: Run; + snapshot: Snapshot; + } + | { + phase: "IDLE" | "WARM_START"; + }; + + private enterIdlePhase() { + this.state = { phase: "IDLE" }; + } + + private enterRunPhase(run: Run, snapshot: Snapshot) { + this.state = { phase: "RUN", run, snapshot }; + } + + private updateSnapshot(snapshot: Snapshot) { + if (this.state.phase !== "RUN") { + throw new Error(`Invalid phase for updating snapshot: ${this.state.phase}`); + } + + this.state.snapshot = snapshot; + } + + private enterWarmStartPhase() { + this.state = { phase: "WARM_START" }; + } + + private get runFriendlyId() { + if (this.state.phase !== "RUN") { + return undefined; + } + + return this.state.run.friendlyId; + } + + private get snapshotFriendlyId() { + if (this.state.phase !== "RUN") { + return undefined; + } + + return this.state.snapshot.friendlyId; + } + + constructor(opts: ManagedRunControllerOptions) { + logger.debug("[ManagedRunController] Creating controller", { env }); + + this.workerManifest = opts.workerManifest; + // TODO: This should be dynamic and set by (or at least overridden by) the managed worker / platform + this.heartbeatIntervalSeconds = opts.heartbeatIntervalSeconds || 30; + this.snapshotPollIntervalSeconds = 5; + + if (env.TRIGGER_RUN_ID) { + if (!env.TRIGGER_SNAPSHOT_ID) { + throw new Error("Missing snapshot ID"); + } + + this.state = { + phase: "RUN", + run: { friendlyId: env.TRIGGER_RUN_ID }, + snapshot: { friendlyId: env.TRIGGER_SNAPSHOT_ID }, + }; + } else { + this.enterIdlePhase(); + } + + this.httpClient = new WorkloadHttpClient({ + workerApiUrl: env.TRIGGER_WORKER_API_URL, + deploymentId: env.TRIGGER_DEPLOYMENT_ID, + }); + + this.snapshotPollService = new HeartbeatService({ + heartbeat: async () => { + if (!this.runFriendlyId) { + logger.debug("[ManagedRunController] Skipping snapshot poll, no run ID"); + return; + } + + console.debug("[ManagedRunController] Polling for latest snapshot"); + + const response = await this.httpClient.getRunExecutionData(this.runFriendlyId); + + if (!response.success) { + console.error("[ManagedRunController] Snapshot poll failed", { error: response.error }); + return; + } + + const { snapshot } = response.data.execution; + + if (snapshot.friendlyId === this.snapshotFriendlyId) { + console.debug("[ManagedRunController] Snapshot not changed", { + snapshotId: this.snapshotFriendlyId, + }); + return; + } + + console.log("Snapshot changed", { + oldSnapshotId: this.snapshotFriendlyId, + newSnapshotId: snapshot.friendlyId, + }); + + this.updateSnapshot(snapshot); + + await this.handleSnapshotChange(response.data.execution); + }, + intervalMs: this.snapshotPollIntervalSeconds * 1000, + leadingEdge: false, + onError: async (error) => { + console.error("[ManagedRunController] Failed to poll for snapshot", { error }); + }, + }); + + this.heartbeatService = new HeartbeatService({ + heartbeat: async () => { + if (!this.runFriendlyId || !this.snapshotFriendlyId) { + logger.debug("[ManagedRunController] Skipping heartbeat, no run ID or snapshot ID"); + return; + } + + console.debug("[ManagedRunController] Sending heartbeat"); + + const response = await this.httpClient.heartbeatRun( + this.runFriendlyId, + this.snapshotFriendlyId, + { + cpu: 0, + memory: 0, + } + ); + + if (!response.success) { + console.error("[ManagedRunController] Heartbeat failed", { error: response.error }); + } + }, + intervalMs: this.heartbeatIntervalSeconds * 1000, + leadingEdge: false, + onError: async (error) => { + console.error("[ManagedRunController] Failed to send heartbeat", { error }); + }, + }); + + process.on("SIGTERM", async () => { + logger.debug("[ManagedRunController] Received SIGTERM, stopping worker"); + await this.stop(); + }); + } + + private async handleSnapshotChange({ run, snapshot, completedWaitpoints }: RunExecutionData) { + console.log("Got latest snapshot", { snapshot, currentSnapshotId: this.snapshotFriendlyId }); + + this.updateSnapshot(snapshot); + + switch (snapshot.executionStatus) { + case "PENDING_CANCEL": { + try { + await this.cancelAttempt(run.friendlyId); + } catch (error) { + console.error("Failed to cancel attempt, shutting down", { + error, + }); + process.exit(1); + } + break; + } + case "FINISHED": { + console.log("Run is finished, shutting down shortly"); + return; + } + default: { + console.log("Status change not handled yet", { status: snapshot.executionStatus }); + // assertExhaustive(snapshot.executionStatus); + break; + } + } + + if (completedWaitpoints.length > 0) { + console.log("Got completed waitpoints", { completedWaitpoints }); + completedWaitpoints.forEach((waitpoint) => { + this.taskRunProcess?.waitpointCompleted(waitpoint); + }); + } + } + + private async startAndExecuteRunAttempt(isWarmStart = false) { + if (!this.runFriendlyId || !this.snapshotFriendlyId) { + logger.debug("[ManagedRunController] Missing run ID or snapshot ID", { + runId: this.runFriendlyId, + snapshotId: this.snapshotFriendlyId, + }); + process.exit(1); + } + + if (!this.socket) { + console.warn("[ManagedRunController] Starting run without socket connection"); + } + + this.socket?.emit("run:start", { + version: "1", + run: { friendlyId: this.runFriendlyId }, + snapshot: { friendlyId: this.snapshotFriendlyId }, + }); + + const start = await this.httpClient.startRunAttempt( + this.runFriendlyId, + this.snapshotFriendlyId, + { + isWarmStart, + } + ); + + if (!start.success) { + console.error("[ManagedRunController] Failed to start run", { error: start.error }); + process.exit(1); + } + + const { run, snapshot, execution, envVars } = start.data; + + logger.debug("[ManagedRunController] Started run", { + runId: run.friendlyId, + snapshot: snapshot.friendlyId, + }); + + this.updateSnapshot(snapshot); + + const taskRunEnv = { + ...gatherProcessEnv(), + ...envVars, + }; + + try { + return await this.executeRun({ run, snapshot, envVars: taskRunEnv, execution }); + } catch (error) { + console.error("Error while executing attempt", { + error, + }); + + console.log("Submitting attempt completion", { + runId: run.friendlyId, + snapshotId: snapshot.friendlyId, + updatedSnapshotId: this.snapshotFriendlyId, + }); + + const completionResult = await this.httpClient.completeRunAttempt( + this.runFriendlyId, + this.snapshotFriendlyId, + { + completion: { + id: execution.run.id, + ok: false, + retry: undefined, + error: TaskRunProcess.parseExecuteError(error), + }, + } + ); + + if (!completionResult.success) { + console.error("Failed to submit completion after error", { + error: completionResult.error, + }); + process.exit(1); + } + + logger.log("Attempt completion submitted after error", completionResult.data.result); + } + } + + private async waitForNextRun() { + this.enterWarmStartPhase(); + + try { + const warmStartUrl = new URL( + "/warm-start", + env.TRIGGER_WARM_START_URL ?? env.TRIGGER_WORKER_API_URL + ); + + const res = await longPoll( + warmStartUrl.href, + { + method: "GET", + headers: { + "x-trigger-workload-controller-id": env.TRIGGER_WORKLOAD_CONTROLLER_ID, + "x-trigger-deployment-id": env.TRIGGER_DEPLOYMENT_ID, + "x-trigger-deployment-version": env.TRIGGER_DEPLOYMENT_VERSION, + "x-trigger-machine-cpu": env.TRIGGER_MACHINE_CPU, + "x-trigger-machine-memory": env.TRIGGER_MACHINE_MEMORY, + }, + }, + { + timeoutMs: 10_000, + totalDurationMs: 60_000, + } + ); + + if (!res.ok) { + console.error("Failed to poll for next run", { error: res.error }); + process.exit(0); + } + + const nextRun = DequeuedMessage.parse(res.data); + + console.log("Got next run", { nextRun }); + + this.enterRunPhase(nextRun.run, nextRun.snapshot); + + this.startAndExecuteRunAttempt(true); + } catch (error) { + console.error("Unexpected error while polling for next run", { error }); + process.exit(1); + } + } + + createSocket() { + const wsUrl = new URL(env.TRIGGER_WORKER_API_URL); + wsUrl.pathname = "/workload"; + + this.socket = io(wsUrl.href, { + transports: ["websocket"], + extraHeaders: { + [WORKLOAD_HEADER_NAME.WORKLOAD_DEPLOYMENT_ID]: env.TRIGGER_DEPLOYMENT_ID, + }, + }); + this.socket.on("run:notify", async ({ version, run }) => { + console.log("[ManagedRunController] Received run notification", { version, run }); + + if (!this.runFriendlyId) { + logger.debug("[ManagedRunController] Ignoring notification, no local run ID", { + runId: run.friendlyId, + currentRunId: this.runFriendlyId, + currentSnapshotId: this.snapshotFriendlyId, + }); + return; + } + + if (run.friendlyId !== this.runFriendlyId) { + console.log("[ManagedRunController] Ignoring notification for different run", { + runId: run.friendlyId, + currentRunId: this.runFriendlyId, + currentSnapshotId: this.snapshotFriendlyId, + }); + return; + } + + const latestSnapshot = await this.httpClient.getRunExecutionData(this.runFriendlyId); + + if (!latestSnapshot.success) { + console.error("Failed to get latest snapshot data", latestSnapshot.error); + return; + } + + await this.handleSnapshotChange(latestSnapshot.data.execution); + }); + this.socket.on("connect", () => { + console.log("[ManagedRunController] Connected to platform"); + }); + this.socket.on("connect_error", (error) => { + console.error("[ManagedRunController] Connection error", { error }); + }); + this.socket.on("disconnect", (reason, description) => { + console.log("[ManagedRunController] Disconnected from platform", { reason, description }); + }); + } + + private async executeRun({ + run, + snapshot, + envVars, + execution, + }: WorkloadRunAttemptStartResponseBody) { + this.taskRunProcess = new TaskRunProcess({ + workerManifest: this.workerManifest, + env: envVars, + serverWorker: { + id: "unmanaged", + contentHash: env.TRIGGER_CONTENT_HASH, + version: env.TRIGGER_DEPLOYMENT_VERSION, + engine: "V2", + }, + payload: { + execution, + traceContext: execution.run.traceContext ?? {}, + }, + messageId: run.friendlyId, + }); + + this.taskRunProcess.onWait.attach(this.handleWait.bind(this)); + + await this.taskRunProcess.initialize(); + + logger.log("executing task run process", { + attemptId: execution.attempt.id, + runId: execution.run.id, + }); + + const completion = await this.taskRunProcess.execute(); + + logger.log("Completed run", completion); + + try { + await this.taskRunProcess.cleanup(true); + } catch (error) { + console.error("Failed to cleanup task run process, submitting completion anyway", { + error, + }); + } + + if (!this.runFriendlyId || !this.snapshotFriendlyId) { + console.error("Missing run ID or snapshot ID after execution", { + runId: this.runFriendlyId, + snapshotId: this.snapshotFriendlyId, + }); + process.exit(1); + } + + const completionResult = await this.httpClient.completeRunAttempt( + this.runFriendlyId, + this.snapshotFriendlyId, + { + completion, + } + ); + + if (!completionResult.success) { + console.error("Failed to submit completion", { + error: completionResult.error, + }); + process.exit(1); + } + + logger.log("Attempt completion submitted", completionResult.data.result); + + const { attemptStatus, snapshot: completionSnapshot } = completionResult.data.result; + + this.updateSnapshot(completionSnapshot); + + if (attemptStatus === "RUN_FINISHED") { + logger.debug("Run finished"); + this.waitForNextRun(); + return; + } + + if (attemptStatus === "RUN_PENDING_CANCEL") { + logger.debug("Run pending cancel"); + return; + } + + if (attemptStatus === "RETRY_QUEUED") { + logger.debug("Retry queued"); + this.waitForNextRun(); + return; + } + + if (attemptStatus === "RETRY_IMMEDIATELY") { + if (completion.ok) { + throw new Error("Should retry but completion OK."); + } + + if (!completion.retry) { + throw new Error("Should retry but missing retry params."); + } + + await wait(completion.retry.delay); + + this.startAndExecuteRunAttempt(); + return; + } + + assertExhaustive(attemptStatus); + } + + private async handleWait({ wait }: OnWaitMessage) { + if (!this.runFriendlyId || !this.snapshotFriendlyId) { + logger.debug("[ManagedRunController] Ignoring wait, no run ID or snapshot ID"); + return; + } + + switch (wait.type) { + case "DATETIME": { + logger.log("Waiting for duration", { wait }); + + const waitpoint = await this.httpClient.waitForDuration( + this.runFriendlyId, + this.snapshotFriendlyId, + { + date: wait.date, + } + ); + + if (!waitpoint.success) { + console.error("Failed to wait for datetime", { error: waitpoint.error }); + return; + } + + logger.log("Waitpoint created", { waitpointData: waitpoint.data }); + + this.taskRunProcess?.waitpointCreated(wait.id, waitpoint.data.waitpoint.id); + + break; + } + default: { + console.error("Wait type not implemented", { wait }); + } + } + } + + async cancelAttempt(runId: string) { + logger.log("cancelling attempt", { runId }); + + await this.taskRunProcess?.cancel(); + } + + async start() { + logger.debug("[ManagedRunController] Starting up"); + + // TODO: remove this after testing + setTimeout(() => { + console.error("[ManagedRunController] Exiting after 5 minutes"); + process.exit(1); + }, 60 * 5000); + + this.heartbeatService.start(); + this.createSocket(); + + this.startAndExecuteRunAttempt(); + this.snapshotPollService.start(); + } + + async stop() { + logger.debug("[ManagedRunController] Shutting down"); + + if (this.taskRunProcess) { + await this.taskRunProcess.cleanup(true); + } + + this.heartbeatService.stop(); + this.socket?.close(); + } +} + +const workerManifest = await loadWorkerManifest(); + +const prodWorker = new ManagedRunController({ workerManifest }); +await prodWorker.start(); + +function gatherProcessEnv(): Record { + const $env = { + NODE_ENV: env.NODE_ENV, + NODE_EXTRA_CA_CERTS: env.NODE_EXTRA_CA_CERTS, + OTEL_EXPORTER_OTLP_ENDPOINT: env.OTEL_EXPORTER_OTLP_ENDPOINT, + }; + + // Filter out undefined values + return Object.fromEntries( + Object.entries($env).filter(([key, value]) => value !== undefined) + ) as Record; +} + +async function loadWorkerManifest() { + const manifest = await readJSONFile("./index.json"); + return WorkerManifest.parse(manifest); +} + +const longPoll = async ( + url: string, + requestInit: Omit, + { + timeoutMs, + totalDurationMs, + }: { + timeoutMs: number; + totalDurationMs: number; + } +): Promise< + | { + ok: true; + data: T; + } + | { + ok: false; + error: string; + } +> => { + const endTime = Date.now() + totalDurationMs; + + while (Date.now() < endTime) { + try { + const controller = new AbortController(); + const signal = controller.signal; + + // TODO: Think about using a random timeout instead + const timeoutId = setTimeout(() => controller.abort(), timeoutMs); + + const response = await fetch(url, { ...requestInit, signal }); + + clearTimeout(timeoutId); + + if (response.ok) { + const data = await response.json(); + + return { + ok: true, + data, + }; + } else { + return { + ok: false, + error: `Server error: ${response.status}`, + }; + } + } catch (error) { + if (error instanceof Error && error.name === "AbortError") { + console.log("Request timed out, retrying..."); + continue; + } else { + console.error("Error during fetch, retrying...", error); + + // TODO: exponential backoff + await wait(1000); + continue; + } + } + } + + return { + ok: false, + error: "TotalDurationExceeded", + }; +}; diff --git a/packages/cli-v3/src/entryPoints/managed-run-worker.ts b/packages/cli-v3/src/entryPoints/managed-run-worker.ts new file mode 100644 index 0000000000..e1c38c836d --- /dev/null +++ b/packages/cli-v3/src/entryPoints/managed-run-worker.ts @@ -0,0 +1,488 @@ +import type { Tracer } from "@opentelemetry/api"; +import type { Logger } from "@opentelemetry/api-logs"; +import { + clock, + type HandleErrorFunction, + logger, + LogLevel, + runtime, + taskCatalog, + TaskRunErrorCodes, + TaskRunExecution, + WorkerToExecutorMessageCatalog, + TriggerConfig, + WorkerManifest, + ExecutorToWorkerMessageCatalog, + timeout, + runMetadata, + waitUntil, + apiClientManager, +} from "@trigger.dev/core/v3"; +import { TriggerTracer } from "@trigger.dev/core/v3/tracer"; +import { + ConsoleInterceptor, + DevUsageManager, + DurableClock, + getEnvVar, + getNumberEnvVar, + logLevels, + OtelTaskLogger, + ProdUsageManager, + StandardTaskCatalog, + TaskExecutor, + TracingDiagnosticLogLevel, + TracingSDK, + usage, + UsageTimeoutManager, + StandardMetadataManager, + StandardWaitUntilManager, + ManagedRuntimeManager, +} from "@trigger.dev/core/v3/workers"; +import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc"; +import { readFile } from "node:fs/promises"; +import { setInterval, setTimeout } from "node:timers/promises"; +import sourceMapSupport from "source-map-support"; +import { env } from "std-env"; +import { normalizeImportPath } from "../utilities/normalizeImportPath.js"; +import { VERSION } from "../version.js"; + +sourceMapSupport.install({ + handleUncaughtExceptions: false, + environment: "node", + hookRequire: false, +}); + +process.on("uncaughtException", function (error, origin) { + console.error("Uncaught exception", { error, origin }); + if (error instanceof Error) { + process.send && + process.send({ + type: "EVENT", + message: { + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { name: error.name, message: error.message, stack: error.stack }, + origin, + }, + version: "v1", + }, + }); + } else { + process.send && + process.send({ + type: "EVENT", + message: { + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { + name: "Error", + message: typeof error === "string" ? error : JSON.stringify(error), + }, + origin, + }, + version: "v1", + }, + }); + } +}); + +const usageIntervalMs = getEnvVar("USAGE_HEARTBEAT_INTERVAL_MS"); +const usageEventUrl = getEnvVar("USAGE_EVENT_URL"); +const triggerJWT = getEnvVar("TRIGGER_JWT"); +const heartbeatIntervalMs = getEnvVar("HEARTBEAT_INTERVAL_MS"); + +const devUsageManager = new DevUsageManager(); +const prodUsageManager = new ProdUsageManager(devUsageManager, { + heartbeatIntervalMs: usageIntervalMs ? parseInt(usageIntervalMs, 10) : undefined, + url: usageEventUrl, + jwt: triggerJWT, +}); + +usage.setGlobalUsageManager(prodUsageManager); +timeout.setGlobalManager(new UsageTimeoutManager(devUsageManager)); + +taskCatalog.setGlobalTaskCatalog(new StandardTaskCatalog()); +const durableClock = new DurableClock(); +clock.setGlobalClock(durableClock); +const runMetadataManager = new StandardMetadataManager( + apiClientManager.clientOrThrow(), + getEnvVar("TRIGGER_STREAM_URL", getEnvVar("TRIGGER_API_URL")) ?? "https://api.trigger.dev" +); +runMetadata.setGlobalManager(runMetadataManager); +const waitUntilManager = new StandardWaitUntilManager(); +waitUntil.setGlobalManager(waitUntilManager); +// Wait for all streams to finish before completing the run +waitUntil.register({ + requiresResolving: () => runMetadataManager.hasActiveStreams(), + promise: () => runMetadataManager.waitForAllStreams(), +}); + +const triggerLogLevel = getEnvVar("TRIGGER_LOG_LEVEL"); + +async function importConfig( + configPath: string +): Promise<{ config: TriggerConfig; handleError?: HandleErrorFunction }> { + const configModule = await import(configPath); + + const config = configModule?.default ?? configModule?.config; + + return { + config, + handleError: configModule?.handleError, + }; +} + +async function loadWorkerManifest() { + const manifestContents = await readFile("./index.json", "utf-8"); + const raw = JSON.parse(manifestContents); + + return WorkerManifest.parse(raw); +} + +async function bootstrap() { + const workerManifest = await loadWorkerManifest(); + + const { config, handleError } = await importConfig( + normalizeImportPath(workerManifest.configPath) + ); + + const tracingSDK = new TracingSDK({ + url: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318", + instrumentations: config.instrumentations ?? [], + diagLogLevel: (env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none", + forceFlushTimeoutMillis: 30_000, + }); + + const otelTracer: Tracer = tracingSDK.getTracer("trigger-dev-worker", VERSION); + const otelLogger: Logger = tracingSDK.getLogger("trigger-dev-worker", VERSION); + + const tracer = new TriggerTracer({ tracer: otelTracer, logger: otelLogger }); + const consoleInterceptor = new ConsoleInterceptor( + otelLogger, + typeof config.enableConsoleLogging === "boolean" ? config.enableConsoleLogging : true + ); + + const configLogLevel = triggerLogLevel ?? config.logLevel ?? "info"; + + const otelTaskLogger = new OtelTaskLogger({ + logger: otelLogger, + tracer: tracer, + level: logLevels.includes(configLogLevel as any) ? (configLogLevel as LogLevel) : "info", + }); + + logger.setGlobalTaskLogger(otelTaskLogger); + + for (const task of workerManifest.tasks) { + taskCatalog.registerTaskFileMetadata(task.id, { + exportName: task.exportName, + filePath: task.filePath, + entryPoint: task.entryPoint, + }); + } + + return { + tracer, + tracingSDK, + consoleInterceptor, + config, + handleErrorFn: handleError, + workerManifest, + }; +} + +let _execution: TaskRunExecution | undefined; +let _isRunning = false; +let _tracingSDK: TracingSDK | undefined; + +const zodIpc = new ZodIpcConnection({ + listenSchema: WorkerToExecutorMessageCatalog, + emitSchema: ExecutorToWorkerMessageCatalog, + process, + handlers: { + EXECUTE_TASK_RUN: async ({ execution, traceContext, metadata }, sender) => { + console.log(`[${new Date().toISOString()}] Received EXECUTE_TASK_RUN`, execution); + + if (_isRunning) { + console.error("Worker is already running a task"); + + await sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + ok: false, + id: execution.run.id, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.TASK_ALREADY_RUNNING, + }, + usage: { + durationMs: 0, + }, + }, + }); + + return; + } + + try { + const { tracer, tracingSDK, consoleInterceptor, config, handleErrorFn, workerManifest } = + await bootstrap(); + + _tracingSDK = tracingSDK; + + const taskManifest = workerManifest.tasks.find((t) => t.id === execution.task.id); + + if (!taskManifest) { + console.error(`Could not find task ${execution.task.id}`); + + await sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + ok: false, + id: execution.run.id, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.COULD_NOT_FIND_TASK, + message: `Could not find task ${execution.task.id}. Make sure the task is exported and the ID is correct.`, + }, + usage: { + durationMs: 0, + }, + }, + }); + + return; + } + + try { + const beforeImport = performance.now(); + await import(normalizeImportPath(taskManifest.entryPoint)); + const durationMs = performance.now() - beforeImport; + + console.log( + `Imported task ${execution.task.id} [${taskManifest.entryPoint}] in ${durationMs}ms` + ); + } catch (err) { + console.error(`Failed to import task ${execution.task.id}`, err); + + await sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + ok: false, + id: execution.run.id, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.COULD_NOT_IMPORT_TASK, + message: err instanceof Error ? err.message : String(err), + stackTrace: err instanceof Error ? err.stack : undefined, + }, + usage: { + durationMs: 0, + }, + }, + }); + + return; + } + + process.title = `trigger-dev-worker: ${execution.task.id} ${execution.run.id}`; + + // Import the task module + const task = taskCatalog.getTask(execution.task.id); + + if (!task) { + console.error(`Could not find task ${execution.task.id}`); + + await sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + ok: false, + id: execution.run.id, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.COULD_NOT_FIND_EXECUTOR, + }, + usage: { + durationMs: 0, + }, + }, + }); + + return; + } + + const executor = new TaskExecutor(task, { + tracer, + tracingSDK, + consoleInterceptor, + config, + handleErrorFn, + }); + + try { + _execution = execution; + _isRunning = true; + + runMetadataManager.startPeriodicFlush( + getNumberEnvVar("TRIGGER_RUN_METADATA_FLUSH_INTERVAL", 1000) + ); + + const measurement = usage.start(); + + // This lives outside of the executor because this will eventually be moved to the controller level + const signal = execution.run.maxDuration + ? timeout.abortAfterTimeout(execution.run.maxDuration) + : undefined; + + signal?.addEventListener("abort", async (e) => { + if (_isRunning) { + _isRunning = false; + _execution = undefined; + + const usageSample = usage.stop(measurement); + + await sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + ok: false, + id: execution.run.id, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.MAX_DURATION_EXCEEDED, + message: + signal.reason instanceof Error + ? signal.reason.message + : String(signal.reason), + }, + usage: { + durationMs: usageSample.cpuTime, + }, + }, + }); + } + }); + + const { result } = await executor.execute( + execution, + metadata, + traceContext, + measurement, + signal + ); + + const usageSample = usage.stop(measurement); + + if (_isRunning) { + return sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + ...result, + usage: { + durationMs: usageSample.cpuTime, + }, + }, + }); + } + } finally { + _execution = undefined; + _isRunning = false; + } + } catch (err) { + console.error("Failed to execute task", err); + + await sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + ok: false, + id: execution.run.id, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.CONFIGURED_INCORRECTLY, + }, + usage: { + durationMs: 0, + }, + }, + }); + } + }, + TASK_RUN_COMPLETED_NOTIFICATION: async () => { + await managedWorkerRuntime.completeWaitpoints([]); + }, + WAIT_COMPLETED_NOTIFICATION: async () => { + await managedWorkerRuntime.completeWaitpoints([]); + }, + FLUSH: async ({ timeoutInMs }, sender) => { + await flushAll(timeoutInMs); + }, + WAITPOINT_CREATED: async ({ wait, waitpoint }) => { + managedWorkerRuntime.associateWaitWithWaitpoint(wait.id, waitpoint.id); + }, + WAITPOINT_COMPLETED: async ({ waitpoint }) => { + managedWorkerRuntime.completeWaitpoints([waitpoint]); + }, + }, +}); + +async function flushAll(timeoutInMs: number = 10_000) { + const now = performance.now(); + + await Promise.all([ + flushUsage(timeoutInMs), + flushTracingSDK(timeoutInMs), + flushMetadata(timeoutInMs), + ]); + + const duration = performance.now() - now; + + console.log(`Flushed all in ${duration}ms`); +} + +async function flushUsage(timeoutInMs: number = 10_000) { + const now = performance.now(); + + await Promise.race([prodUsageManager.flush(), setTimeout(timeoutInMs)]); + + const duration = performance.now() - now; + + console.log(`Flushed usage in ${duration}ms`); +} + +async function flushTracingSDK(timeoutInMs: number = 10_000) { + const now = performance.now(); + + await Promise.race([_tracingSDK?.flush(), setTimeout(timeoutInMs)]); + + const duration = performance.now() - now; + + console.log(`Flushed tracingSDK in ${duration}ms`); +} + +async function flushMetadata(timeoutInMs: number = 10_000) { + const now = performance.now(); + + await Promise.race([runMetadataManager.flush(), setTimeout(timeoutInMs)]); + + const duration = performance.now() - now; + + console.log(`Flushed runMetadata in ${duration}ms`); +} + +const managedWorkerRuntime = new ManagedRuntimeManager(zodIpc); + +runtime.setGlobalRuntimeManager(managedWorkerRuntime); + +process.title = "trigger-managed-worker"; + +const heartbeatInterval = parseInt(heartbeatIntervalMs ?? "30000", 10); + +for await (const _ of setInterval(heartbeatInterval)) { + if (_isRunning && _execution) { + try { + await zodIpc.send("TASK_HEARTBEAT", { id: _execution.attempt.id }); + } catch (err) { + console.error("Failed to send HEARTBEAT message", err); + } + } +} + +console.log(`[${new Date().toISOString()}] Executor started`); diff --git a/packages/cli-v3/src/entryPoints/unmanaged-index-controller.ts b/packages/cli-v3/src/entryPoints/unmanaged-index-controller.ts new file mode 100644 index 0000000000..5822acecd5 --- /dev/null +++ b/packages/cli-v3/src/entryPoints/unmanaged-index-controller.ts @@ -0,0 +1,117 @@ +import { + BuildManifest, + CreateBackgroundWorkerRequestBody, + serializeIndexingError, +} from "@trigger.dev/core/v3"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { env } from "std-env"; +import { CliApiClient } from "../apiClient.js"; +import { indexWorkerManifest } from "../indexing/indexWorkerManifest.js"; +import { resolveSourceFiles } from "../utilities/sourceFiles.js"; +import { execOptionsForRuntime } from "@trigger.dev/core/v3/build"; + +async function loadBuildManifest() { + const manifestContents = await readFile("./build.json", "utf-8"); + const raw = JSON.parse(manifestContents); + + return BuildManifest.parse(raw); +} + +async function bootstrap() { + const buildManifest = await loadBuildManifest(); + + if (typeof env.TRIGGER_API_URL !== "string") { + console.error("TRIGGER_API_URL is not set"); + process.exit(1); + } + + const cliApiClient = new CliApiClient(env.TRIGGER_API_URL, env.TRIGGER_SECRET_KEY); + + if (!env.TRIGGER_PROJECT_REF) { + console.error("TRIGGER_PROJECT_REF is not set"); + process.exit(1); + } + + if (!env.TRIGGER_DEPLOYMENT_ID) { + console.error("TRIGGER_DEPLOYMENT_ID is not set"); + process.exit(1); + } + + return { + buildManifest, + cliApiClient, + projectRef: env.TRIGGER_PROJECT_REF, + deploymentId: env.TRIGGER_DEPLOYMENT_ID, + }; +} + +type BootstrapResult = Awaited>; + +async function indexDeployment({ + cliApiClient, + projectRef, + deploymentId, + buildManifest, +}: BootstrapResult) { + const stdout: string[] = []; + const stderr: string[] = []; + + try { + const $env = await cliApiClient.getEnvironmentVariables(projectRef); + + if (!$env.success) { + throw new Error(`Failed to fetch environment variables: ${$env.error}`); + } + + const workerManifest = await indexWorkerManifest({ + runtime: buildManifest.runtime, + indexWorkerPath: buildManifest.indexWorkerEntryPoint, + buildManifestPath: "./build.json", + nodeOptions: execOptionsForRuntime(buildManifest.runtime, buildManifest), + env: $env.data.variables, + otelHookExclude: buildManifest.otelImportHook?.exclude, + otelHookInclude: buildManifest.otelImportHook?.include, + handleStdout(data) { + stdout.push(data); + }, + handleStderr(data) { + if (!data.includes("DeprecationWarning")) { + stderr.push(data); + } + }, + }); + + console.log("Writing index.json", process.cwd()); + + await writeFile(join(process.cwd(), "index.json"), JSON.stringify(workerManifest, null, 2)); + + const sourceFiles = resolveSourceFiles(buildManifest.sources, workerManifest.tasks); + + const backgroundWorkerBody: CreateBackgroundWorkerRequestBody = { + localOnly: true, + metadata: { + contentHash: buildManifest.contentHash, + packageVersion: buildManifest.packageVersion, + cliPackageVersion: buildManifest.cliPackageVersion, + tasks: workerManifest.tasks, + sourceFiles, + }, + supportsLazyAttempts: true, + }; + + await cliApiClient.createDeploymentBackgroundWorker(deploymentId, backgroundWorkerBody); + } catch (error) { + const serialiedIndexError = serializeIndexingError(error, stderr.join("\n")); + + console.error("Failed to index deployment", serialiedIndexError); + + await cliApiClient.failDeployment(deploymentId, { error: serialiedIndexError }); + + process.exit(1); + } +} + +const results = await bootstrap(); + +await indexDeployment(results); diff --git a/packages/cli-v3/src/entryPoints/unmanaged-index-worker.ts b/packages/cli-v3/src/entryPoints/unmanaged-index-worker.ts new file mode 100644 index 0000000000..2ef18444eb --- /dev/null +++ b/packages/cli-v3/src/entryPoints/unmanaged-index-worker.ts @@ -0,0 +1,170 @@ +import { + BuildManifest, + type HandleErrorFunction, + indexerToWorkerMessages, + taskCatalog, + type TaskManifest, + TriggerConfig, +} from "@trigger.dev/core/v3"; +import { + StandardTaskCatalog, + TracingDiagnosticLogLevel, + TracingSDK, +} from "@trigger.dev/core/v3/workers"; +import { sendMessageInCatalog, ZodSchemaParsedError } from "@trigger.dev/core/v3/zodMessageHandler"; +import { readFile } from "node:fs/promises"; +import sourceMapSupport from "source-map-support"; +import { registerTasks } from "../indexing/registerTasks.js"; +import { env } from "std-env"; +import { normalizeImportPath } from "../utilities/normalizeImportPath.js"; + +sourceMapSupport.install({ + handleUncaughtExceptions: false, + environment: "node", + hookRequire: false, +}); + +process.on("uncaughtException", function (error, origin) { + if (error instanceof Error) { + process.send && + process.send({ + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { name: error.name, message: error.message, stack: error.stack }, + origin, + }, + version: "v1", + }); + } else { + process.send && + process.send({ + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { + name: "Error", + message: typeof error === "string" ? error : JSON.stringify(error), + }, + origin, + }, + version: "v1", + }); + } +}); + +taskCatalog.setGlobalTaskCatalog(new StandardTaskCatalog()); + +async function importConfig( + configPath: string +): Promise<{ config: TriggerConfig; handleError?: HandleErrorFunction }> { + const configModule = await import(normalizeImportPath(configPath)); + + const config = configModule?.default ?? configModule?.config; + + return { + config, + handleError: configModule?.handleError, + }; +} + +async function loadBuildManifest() { + const manifestContents = await readFile(env.TRIGGER_BUILD_MANIFEST_PATH!, "utf-8"); + const raw = JSON.parse(manifestContents); + + return BuildManifest.parse(raw); +} + +async function bootstrap() { + const buildManifest = await loadBuildManifest(); + + const { config } = await importConfig(buildManifest.configPath); + + // This needs to run or the PrismaInstrumentation will throw an error + const tracingSDK = new TracingSDK({ + url: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318", + instrumentations: config.instrumentations ?? [], + diagLogLevel: (env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none", + forceFlushTimeoutMillis: 30_000, + }); + + const importErrors = await registerTasks(buildManifest); + + return { + tracingSDK, + config, + buildManifest, + importErrors, + }; +} + +const { buildManifest, importErrors, config } = await bootstrap(); + +let tasks = taskCatalog.listTaskManifests(); + +// If the config has retry defaults, we need to apply them to all tasks that don't have any retry settings +if (config.retries?.default) { + tasks = tasks.map((task) => { + if (!task.retry) { + return { + ...task, + retry: config.retries?.default, + } satisfies TaskManifest; + } + + return task; + }); +} + +// If the config has a maxDuration, we need to apply it to all tasks that don't have a maxDuration +if (typeof config.maxDuration === "number") { + tasks = tasks.map((task) => { + if (typeof task.maxDuration !== "number") { + return { + ...task, + maxDuration: config.maxDuration, + } satisfies TaskManifest; + } + + return task; + }); +} + +await sendMessageInCatalog( + indexerToWorkerMessages, + "INDEX_COMPLETE", + { + manifest: { + tasks, + configPath: buildManifest.configPath, + runtime: buildManifest.runtime, + workerEntryPoint: buildManifest.runWorkerEntryPoint, + controllerEntryPoint: buildManifest.runControllerEntryPoint, + loaderEntryPoint: buildManifest.loaderEntryPoint, + customConditions: buildManifest.customConditions, + }, + importErrors, + }, + async (msg) => { + process.send?.(msg); + } +).catch((err) => { + if (err instanceof ZodSchemaParsedError) { + return sendMessageInCatalog( + indexerToWorkerMessages, + "TASKS_FAILED_TO_PARSE", + { zodIssues: err.error.issues, tasks }, + async (msg) => { + process.send?.(msg); + } + ); + } else { + console.error("Failed to send TASKS_READY message", err); + } + + return; +}); + +await new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, 10); +}); diff --git a/packages/cli-v3/src/entryPoints/unmanaged-run-controller.ts b/packages/cli-v3/src/entryPoints/unmanaged-run-controller.ts new file mode 100644 index 0000000000..11989e89da --- /dev/null +++ b/packages/cli-v3/src/entryPoints/unmanaged-run-controller.ts @@ -0,0 +1,168 @@ +import { logger } from "../utilities/logger.js"; +import { TaskRunProcess } from "../executions/taskRunProcess.js"; +import { env as stdEnv } from "std-env"; +import { z } from "zod"; +import { CLOUD_API_URL } from "../consts.js"; +import { randomUUID } from "crypto"; +import { readJSONFile } from "../utilities/fileSystem.js"; +import { WorkerManifest } from "@trigger.dev/core/v3"; +import { SupervisorSession } from "@trigger.dev/worker"; + +const Env = z.object({ + TRIGGER_API_URL: z.string().url().default(CLOUD_API_URL), + TRIGGER_CONTENT_HASH: z.string(), + TRIGGER_WORKER_TOKEN: z.string(), + TRIGGER_WORKER_INSTANCE_NAME: z.string().default(randomUUID()), + TRIGGER_DEPLOYMENT_ID: z.string(), + TRIGGER_DEPLOYMENT_VERSION: z.string(), + NODE_ENV: z.string().default("production"), + NODE_EXTRA_CA_CERTS: z.string().optional(), + OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(), +}); + +const env = Env.parse(stdEnv); + +logger.loggerLevel = "debug"; +logger.debug("Creating unmanaged worker", { env }); + +class UnmanagedRunController { + private readonly session: SupervisorSession; + private taskRunProcess?: TaskRunProcess; + + constructor(private workerManifest: WorkerManifest) { + this.session = new SupervisorSession({ + workerToken: env.TRIGGER_WORKER_TOKEN, + apiUrl: env.TRIGGER_API_URL, + instanceName: env.TRIGGER_WORKER_INSTANCE_NAME, + deploymentId: env.TRIGGER_DEPLOYMENT_ID, + dequeueIntervalMs: 1000, + }); + + this.session.on("runQueueMessage", async ({ time, message }) => { + logger.debug("[UnmanagedRunController] Received runQueueMessage", { time, message }); + + this.session.emit("requestRunAttemptStart", { + time: new Date(), + run: { + friendlyId: message.run.id, + }, + snapshot: { + friendlyId: message.snapshot.id, + }, + }); + }); + + this.session.on("runAttemptStarted", async ({ time, run, snapshot, execution, envVars }) => { + const taskRunEnv = { + ...gatherProcessEnv(), + ...envVars, + }; + + this.taskRunProcess = new TaskRunProcess({ + workerManifest: this.workerManifest, + env: taskRunEnv, + serverWorker: { + id: "unmanaged", + contentHash: env.TRIGGER_CONTENT_HASH, + version: env.TRIGGER_DEPLOYMENT_VERSION, + engine: "V2", + }, + payload: { + execution, + traceContext: execution.run.traceContext ?? {}, + }, + messageId: run.id, + }); + + try { + await this.taskRunProcess.initialize(); + + logger.log("executing task run process", { + attemptId: execution.attempt.id, + runId: execution.run.id, + }); + + const completion = await this.taskRunProcess.execute(); + + logger.log("completed", completion); + + try { + await this.taskRunProcess.cleanup(true); + } catch (error) { + console.error("Failed to cleanup task run process, submitting completion anyway", { + error, + }); + } + + this.session.emit("runAttemptCompleted", { + time: new Date(), + run: { + friendlyId: run.id, + }, + snapshot: { + friendlyId: snapshot.id, + }, + completion, + }); + } catch (error) { + console.error("Failed to complete lazy attempt", { + error, + }); + + this.session.emit("runAttemptCompleted", { + time: new Date(), + run: { + friendlyId: run.id, + }, + snapshot: { + friendlyId: snapshot.id, + }, + completion: { + id: execution.run.id, + ok: false, + retry: undefined, + error: TaskRunProcess.parseExecuteError(error), + }, + }); + } + }); + + process.on("SIGTERM", async () => { + logger.debug("[UnmanagedRunController] Received SIGTERM, stopping worker"); + await this.stop(); + }); + } + + async start() { + logger.debug("[UnmanagedRunController] Starting up"); + await this.session.start(); + } + + async stop() { + logger.debug("[UnmanagedRunController] Shutting down"); + await this.session.stop(); + } +} + +const workerManifest = await loadWorkerManifest(); + +const prodWorker = new UnmanagedRunController(workerManifest); +await prodWorker.start(); + +function gatherProcessEnv(): Record { + const $env = { + NODE_ENV: env.NODE_ENV, + NODE_EXTRA_CA_CERTS: env.NODE_EXTRA_CA_CERTS, + OTEL_EXPORTER_OTLP_ENDPOINT: env.OTEL_EXPORTER_OTLP_ENDPOINT, + }; + + // Filter out undefined values + return Object.fromEntries( + Object.entries($env).filter(([key, value]) => value !== undefined) + ) as Record; +} + +async function loadWorkerManifest() { + const manifest = await readJSONFile("./index.json"); + return WorkerManifest.parse(manifest); +} diff --git a/packages/cli-v3/src/entryPoints/unmanaged-run-worker.ts b/packages/cli-v3/src/entryPoints/unmanaged-run-worker.ts new file mode 100644 index 0000000000..c2f03fb052 --- /dev/null +++ b/packages/cli-v3/src/entryPoints/unmanaged-run-worker.ts @@ -0,0 +1,482 @@ +import type { Tracer } from "@opentelemetry/api"; +import type { Logger } from "@opentelemetry/api-logs"; +import { + clock, + type HandleErrorFunction, + logger, + LogLevel, + runtime, + taskCatalog, + TaskRunErrorCodes, + TaskRunExecution, + WorkerToExecutorMessageCatalog, + TriggerConfig, + WorkerManifest, + ExecutorToWorkerMessageCatalog, + timeout, + runMetadata, + waitUntil, + apiClientManager, +} from "@trigger.dev/core/v3"; +import { TriggerTracer } from "@trigger.dev/core/v3/tracer"; +import { + ConsoleInterceptor, + DevUsageManager, + DurableClock, + getEnvVar, + getNumberEnvVar, + logLevels, + OtelTaskLogger, + ProdUsageManager, + StandardTaskCatalog, + TaskExecutor, + TracingDiagnosticLogLevel, + TracingSDK, + usage, + UsageTimeoutManager, + StandardMetadataManager, + StandardWaitUntilManager, + UnmanagedRuntimeManager, +} from "@trigger.dev/core/v3/workers"; +import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc"; +import { readFile } from "node:fs/promises"; +import { setInterval, setTimeout } from "node:timers/promises"; +import sourceMapSupport from "source-map-support"; +import { env } from "std-env"; +import { normalizeImportPath } from "../utilities/normalizeImportPath.js"; +import { VERSION } from "../version.js"; + +sourceMapSupport.install({ + handleUncaughtExceptions: false, + environment: "node", + hookRequire: false, +}); + +process.on("uncaughtException", function (error, origin) { + console.error("Uncaught exception", { error, origin }); + if (error instanceof Error) { + process.send && + process.send({ + type: "EVENT", + message: { + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { name: error.name, message: error.message, stack: error.stack }, + origin, + }, + version: "v1", + }, + }); + } else { + process.send && + process.send({ + type: "EVENT", + message: { + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { + name: "Error", + message: typeof error === "string" ? error : JSON.stringify(error), + }, + origin, + }, + version: "v1", + }, + }); + } +}); + +const usageIntervalMs = getEnvVar("USAGE_HEARTBEAT_INTERVAL_MS"); +const usageEventUrl = getEnvVar("USAGE_EVENT_URL"); +const triggerJWT = getEnvVar("TRIGGER_JWT"); +const heartbeatIntervalMs = getEnvVar("HEARTBEAT_INTERVAL_MS"); + +const devUsageManager = new DevUsageManager(); +const prodUsageManager = new ProdUsageManager(devUsageManager, { + heartbeatIntervalMs: usageIntervalMs ? parseInt(usageIntervalMs, 10) : undefined, + url: usageEventUrl, + jwt: triggerJWT, +}); + +usage.setGlobalUsageManager(prodUsageManager); +timeout.setGlobalManager(new UsageTimeoutManager(devUsageManager)); + +taskCatalog.setGlobalTaskCatalog(new StandardTaskCatalog()); +const durableClock = new DurableClock(); +clock.setGlobalClock(durableClock); +const runMetadataManager = new StandardMetadataManager( + apiClientManager.clientOrThrow(), + getEnvVar("TRIGGER_STREAM_URL", getEnvVar("TRIGGER_API_URL")) ?? "https://api.trigger.dev" +); +runMetadata.setGlobalManager(runMetadataManager); +const waitUntilManager = new StandardWaitUntilManager(); +waitUntil.setGlobalManager(waitUntilManager); +// Wait for all streams to finish before completing the run +waitUntil.register({ + requiresResolving: () => runMetadataManager.hasActiveStreams(), + promise: () => runMetadataManager.waitForAllStreams(), +}); + +const triggerLogLevel = getEnvVar("TRIGGER_LOG_LEVEL"); + +async function importConfig( + configPath: string +): Promise<{ config: TriggerConfig; handleError?: HandleErrorFunction }> { + const configModule = await import(configPath); + + const config = configModule?.default ?? configModule?.config; + + return { + config, + handleError: configModule?.handleError, + }; +} + +async function loadWorkerManifest() { + const manifestContents = await readFile("./index.json", "utf-8"); + const raw = JSON.parse(manifestContents); + + return WorkerManifest.parse(raw); +} + +async function bootstrap() { + const workerManifest = await loadWorkerManifest(); + + const { config, handleError } = await importConfig( + normalizeImportPath(workerManifest.configPath) + ); + + const tracingSDK = new TracingSDK({ + url: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318", + instrumentations: config.instrumentations ?? [], + diagLogLevel: (env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none", + forceFlushTimeoutMillis: 30_000, + }); + + const otelTracer: Tracer = tracingSDK.getTracer("trigger-dev-worker", VERSION); + const otelLogger: Logger = tracingSDK.getLogger("trigger-dev-worker", VERSION); + + const tracer = new TriggerTracer({ tracer: otelTracer, logger: otelLogger }); + const consoleInterceptor = new ConsoleInterceptor( + otelLogger, + typeof config.enableConsoleLogging === "boolean" ? config.enableConsoleLogging : true + ); + + const configLogLevel = triggerLogLevel ?? config.logLevel ?? "info"; + + const otelTaskLogger = new OtelTaskLogger({ + logger: otelLogger, + tracer: tracer, + level: logLevels.includes(configLogLevel as any) ? (configLogLevel as LogLevel) : "info", + }); + + logger.setGlobalTaskLogger(otelTaskLogger); + + for (const task of workerManifest.tasks) { + taskCatalog.registerTaskFileMetadata(task.id, { + exportName: task.exportName, + filePath: task.filePath, + entryPoint: task.entryPoint, + }); + } + + return { + tracer, + tracingSDK, + consoleInterceptor, + config, + handleErrorFn: handleError, + workerManifest, + }; +} + +let _execution: TaskRunExecution | undefined; +let _isRunning = false; +let _tracingSDK: TracingSDK | undefined; + +const zodIpc = new ZodIpcConnection({ + listenSchema: WorkerToExecutorMessageCatalog, + emitSchema: ExecutorToWorkerMessageCatalog, + process, + handlers: { + EXECUTE_TASK_RUN: async ({ execution, traceContext, metadata }, sender) => { + console.log(`[${new Date().toISOString()}] Received EXECUTE_TASK_RUN`, execution); + + if (_isRunning) { + console.error("Worker is already running a task"); + + await sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + ok: false, + id: execution.run.id, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.TASK_ALREADY_RUNNING, + }, + usage: { + durationMs: 0, + }, + }, + }); + + return; + } + + try { + const { tracer, tracingSDK, consoleInterceptor, config, handleErrorFn, workerManifest } = + await bootstrap(); + + _tracingSDK = tracingSDK; + + const taskManifest = workerManifest.tasks.find((t) => t.id === execution.task.id); + + if (!taskManifest) { + console.error(`Could not find task ${execution.task.id}`); + + await sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + ok: false, + id: execution.run.id, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.COULD_NOT_FIND_TASK, + message: `Could not find task ${execution.task.id}. Make sure the task is exported and the ID is correct.`, + }, + usage: { + durationMs: 0, + }, + }, + }); + + return; + } + + try { + const beforeImport = performance.now(); + await import(normalizeImportPath(taskManifest.entryPoint)); + const durationMs = performance.now() - beforeImport; + + console.log( + `Imported task ${execution.task.id} [${taskManifest.entryPoint}] in ${durationMs}ms` + ); + } catch (err) { + console.error(`Failed to import task ${execution.task.id}`, err); + + await sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + ok: false, + id: execution.run.id, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.COULD_NOT_IMPORT_TASK, + message: err instanceof Error ? err.message : String(err), + stackTrace: err instanceof Error ? err.stack : undefined, + }, + usage: { + durationMs: 0, + }, + }, + }); + + return; + } + + process.title = `trigger-dev-worker: ${execution.task.id} ${execution.run.id}`; + + // Import the task module + const task = taskCatalog.getTask(execution.task.id); + + if (!task) { + console.error(`Could not find task ${execution.task.id}`); + + await sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + ok: false, + id: execution.run.id, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.COULD_NOT_FIND_EXECUTOR, + }, + usage: { + durationMs: 0, + }, + }, + }); + + return; + } + + const executor = new TaskExecutor(task, { + tracer, + tracingSDK, + consoleInterceptor, + config, + handleErrorFn, + }); + + try { + _execution = execution; + _isRunning = true; + + runMetadataManager.startPeriodicFlush( + getNumberEnvVar("TRIGGER_RUN_METADATA_FLUSH_INTERVAL", 1000) + ); + + const measurement = usage.start(); + + // This lives outside of the executor because this will eventually be moved to the controller level + const signal = execution.run.maxDuration + ? timeout.abortAfterTimeout(execution.run.maxDuration) + : undefined; + + signal?.addEventListener("abort", async (e) => { + if (_isRunning) { + _isRunning = false; + _execution = undefined; + + const usageSample = usage.stop(measurement); + + await sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + ok: false, + id: execution.run.id, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.MAX_DURATION_EXCEEDED, + message: + signal.reason instanceof Error + ? signal.reason.message + : String(signal.reason), + }, + usage: { + durationMs: usageSample.cpuTime, + }, + }, + }); + } + }); + + const { result } = await executor.execute( + execution, + metadata, + traceContext, + measurement, + signal + ); + + const usageSample = usage.stop(measurement); + + if (_isRunning) { + return sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + ...result, + usage: { + durationMs: usageSample.cpuTime, + }, + }, + }); + } + } finally { + _execution = undefined; + _isRunning = false; + } + } catch (err) { + console.error("Failed to execute task", err); + + await sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + ok: false, + id: execution.run.id, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.CONFIGURED_INCORRECTLY, + }, + usage: { + durationMs: 0, + }, + }, + }); + } + }, + TASK_RUN_COMPLETED_NOTIFICATION: async () => { + await unmanagedWorkerRuntime.completeWaitpoints([]); + }, + WAIT_COMPLETED_NOTIFICATION: async () => { + await unmanagedWorkerRuntime.completeWaitpoints([]); + }, + FLUSH: async ({ timeoutInMs }, sender) => { + await flushAll(timeoutInMs); + }, + }, +}); + +async function flushAll(timeoutInMs: number = 10_000) { + const now = performance.now(); + + await Promise.all([ + flushUsage(timeoutInMs), + flushTracingSDK(timeoutInMs), + flushMetadata(timeoutInMs), + ]); + + const duration = performance.now() - now; + + console.log(`Flushed all in ${duration}ms`); +} + +async function flushUsage(timeoutInMs: number = 10_000) { + const now = performance.now(); + + await Promise.race([prodUsageManager.flush(), setTimeout(timeoutInMs)]); + + const duration = performance.now() - now; + + console.log(`Flushed usage in ${duration}ms`); +} + +async function flushTracingSDK(timeoutInMs: number = 10_000) { + const now = performance.now(); + + await Promise.race([_tracingSDK?.flush(), setTimeout(timeoutInMs)]); + + const duration = performance.now() - now; + + console.log(`Flushed tracingSDK in ${duration}ms`); +} + +async function flushMetadata(timeoutInMs: number = 10_000) { + const now = performance.now(); + + await Promise.race([runMetadataManager.flush(), setTimeout(timeoutInMs)]); + + const duration = performance.now() - now; + + console.log(`Flushed runMetadata in ${duration}ms`); +} + +const unmanagedWorkerRuntime = new UnmanagedRuntimeManager(); + +runtime.setGlobalRuntimeManager(unmanagedWorkerRuntime); + +process.title = "trigger-unmanaged-worker"; + +const heartbeatInterval = parseInt(heartbeatIntervalMs ?? "30000", 10); + +for await (const _ of setInterval(heartbeatInterval)) { + if (_isRunning && _execution) { + try { + await zodIpc.send("TASK_HEARTBEAT", { id: _execution.attempt.id }); + } catch (err) { + console.error("Failed to send HEARTBEAT message", err); + } + } +} + +console.log(`[${new Date().toISOString()}] Executor started`); diff --git a/packages/cli-v3/src/executions/taskRunProcess.ts b/packages/cli-v3/src/executions/taskRunProcess.ts index 22e3c9f6d5..118467ef4a 100644 --- a/packages/cli-v3/src/executions/taskRunProcess.ts +++ b/packages/cli-v3/src/executions/taskRunProcess.ts @@ -1,4 +1,5 @@ import { + CompletedWaitpoint, ExecutorToWorkerMessageCatalog, ServerBackgroundWorker, TaskRunErrorCodes, @@ -40,6 +41,7 @@ export type OnWaitForBatchMessage = InferSocketMessageSchema< typeof ExecutorToWorkerMessageCatalog, "WAIT_FOR_BATCH" >; +export type OnWaitMessage = InferSocketMessageSchema; export type TaskRunProcessOptions = { workerManifest: WorkerManifest; @@ -75,6 +77,7 @@ export class TaskRunProcess { public onWaitForDuration: Evt = new Evt(); public onWaitForTask: Evt = new Evt(); public onWaitForBatch: Evt = new Evt(); + public onWait: Evt = new Evt(); constructor(public readonly options: TaskRunProcessOptions) {} @@ -186,6 +189,12 @@ export class TaskRunProcess { WAIT_FOR_DURATION: async (message) => { this.onWaitForDuration.post(message); }, + UNCAUGHT_EXCEPTION: async (message) => { + logger.debug(`[${this.runId}] uncaught exception in task run process`, { ...message }); + }, + WAIT: async (message) => { + this.onWait.post(message); + }, }, }); @@ -274,6 +283,37 @@ export class TaskRunProcess { this._ipc?.send("WAIT_COMPLETED_NOTIFICATION", {}); } + waitpointCreated(waitId: string, waitpointId: string) { + if (!this._child?.connected || this._isBeingKilled || this._child.killed) { + console.error( + "Child process not connected or being killed, can't send waitpoint created notification" + ); + return; + } + + this._ipc?.send("WAITPOINT_CREATED", { + wait: { + id: waitId, + }, + waitpoint: { + id: waitpointId, + }, + }); + } + + waitpointCompleted(waitpoint: CompletedWaitpoint) { + if (!this._child?.connected || this._isBeingKilled || this._child.killed) { + console.error( + "Child process not connected or being killed, can't send waitpoint completed notification" + ); + return; + } + + this._ipc?.send("WAITPOINT_COMPLETED", { + waitpoint, + }); + } + async #handleExit(code: number | null, signal: NodeJS.Signals | null) { logger.debug("handling child exit", { code, signal }); diff --git a/packages/cli-v3/src/utilities/configFiles.ts b/packages/cli-v3/src/utilities/configFiles.ts index dc009ff57e..6e1c9052e7 100644 --- a/packages/cli-v3/src/utilities/configFiles.ts +++ b/packages/cli-v3/src/utilities/configFiles.ts @@ -1,4 +1,4 @@ -import { mkdirSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import path from "node:path"; import { z } from "zod"; import { xdgAppPaths } from "../imports/xdg-app-paths.js"; @@ -11,74 +11,140 @@ function getGlobalConfigFolderPath() { return configDir; } -//auth config file -export const UserAuthConfigSchema = z.object({ +export const DEFFAULT_PROFILE = "default"; + +const CONFIG_FILE = "config.json"; +const OLD_CONFIG_FILE = "default.json"; + +const CliConfigProfileSettings = z.object({ accessToken: z.string().optional(), apiUrl: z.string().optional(), }); +type CliConfigProfileSettings = z.infer; -export type UserAuthConfig = z.infer; +const OldCliConfigFile = z.record(CliConfigProfileSettings); +type OldCliConfigFile = z.infer; -const UserAuthConfigFileSchema = z.record(UserAuthConfigSchema); +const CliConfigFile = z.object({ + version: z.literal(2), + currentProfile: z.string().default(DEFFAULT_PROFILE), + profiles: z.record(CliConfigProfileSettings), +}); +type CliConfigFile = z.infer; -type UserAuthConfigFile = z.infer; +function getOldAuthConfigFilePath() { + return path.join(getGlobalConfigFolderPath(), OLD_CONFIG_FILE); +} function getAuthConfigFilePath() { - return path.join(getGlobalConfigFolderPath(), "default.json"); + return path.join(getGlobalConfigFolderPath(), CONFIG_FILE); +} + +function getAuthConfigFileBackupPath() { + // Multiple calls won't overwrite old backups + return path.join(getGlobalConfigFolderPath(), `${CONFIG_FILE}.bak-${Date.now()}`); } -export function writeAuthConfigProfile(config: UserAuthConfig, profile: string = "default") { - const existingConfig = readAuthConfigFile() || {}; +function getBlankConfig(): CliConfigFile { + return { + version: 2, + currentProfile: DEFFAULT_PROFILE, + profiles: {}, + }; +} + +function getConfig() { + return readAuthConfigFile() ?? getBlankConfig(); +} - existingConfig[profile] = config; +export function writeAuthConfigCurrentProfileName(profile: string) { + const config = getConfig(); - writeAuthConfigFile(existingConfig); + config.currentProfile = profile; + + writeAuthConfigFile(config); } -export function readAuthConfigProfile(profile: string = "default"): UserAuthConfig | undefined { - try { - const authConfigFilePath = getAuthConfigFilePath(); +export function readAuthConfigCurrentProfileName(): string { + const config = getConfig(); + return config.currentProfile; +} - logger.debug(`Reading auth config file`, { authConfigFilePath }); +export function writeAuthConfigProfile( + settings: CliConfigProfileSettings, + profile: string = DEFFAULT_PROFILE +) { + const config = getConfig(); - const json = readJSONFileSync(authConfigFilePath); - const parsed = UserAuthConfigFileSchema.parse(json); - return parsed[profile]; + config.profiles[profile] = settings; + + writeAuthConfigFile(config); +} + +export function readAuthConfigProfile( + profile: string = DEFFAULT_PROFILE +): CliConfigProfileSettings | undefined { + try { + const config = getConfig(); + return config.profiles[profile]; } catch (error) { logger.debug(`Error reading auth config file: ${error}`); return undefined; } } -export function deleteAuthConfigProfile(profile: string = "default") { - const existingConfig = readAuthConfigFile() || {}; +export function deleteAuthConfigProfile(profile: string = DEFFAULT_PROFILE) { + const config = getConfig(); + + delete config.profiles[profile]; - delete existingConfig[profile]; + if (config.currentProfile === profile) { + config.currentProfile = DEFFAULT_PROFILE; + } - writeAuthConfigFile(existingConfig); + writeAuthConfigFile(config); } -export function readAuthConfigFile(): UserAuthConfigFile | undefined { +export function readAuthConfigFile(): CliConfigFile | null { try { - const authConfigFilePath = getAuthConfigFilePath(); + const configFilePath = getAuthConfigFilePath(); + const configFileExists = existsSync(configFilePath); - logger.debug(`Reading auth config file`, { authConfigFilePath }); + logger.debug(`Reading auth config file`, { configFilePath, configFileExists }); - const json = readJSONFileSync(authConfigFilePath); - const parsed = UserAuthConfigFileSchema.parse(json); - return parsed; + const json = readJSONFileSync(configFileExists ? configFilePath : getOldAuthConfigFilePath()); + + if ("currentProfile" in json) { + // This is the new format + const parsed = CliConfigFile.parse(json); + return parsed; + } + + // This is the old format and we need to convert it + const oldConfigFormat = OldCliConfigFile.parse(json); + + const newConfigFormat = { + version: 2, + currentProfile: DEFFAULT_PROFILE, + profiles: oldConfigFormat, + } satisfies CliConfigFile; + + // Save to new config file location, the old file will remain untouched + writeAuthConfigFile(newConfigFormat); + + return newConfigFormat; } catch (error) { logger.debug(`Error reading auth config file: ${error}`); - return undefined; + return null; } } -export function writeAuthConfigFile(config: UserAuthConfigFile) { +export function writeAuthConfigFile(config: CliConfigFile) { const authConfigFilePath = getAuthConfigFilePath(); mkdirSync(path.dirname(authConfigFilePath), { recursive: true, }); - writeFileSync(path.join(authConfigFilePath), JSON.stringify(config), { + writeFileSync(path.join(authConfigFilePath), JSON.stringify(config, undefined, 2), { encoding: "utf-8", }); } diff --git a/packages/cli-v3/src/utilities/initialBanner.ts b/packages/cli-v3/src/utilities/initialBanner.ts index 034f483291..20f4735597 100644 --- a/packages/cli-v3/src/utilities/initialBanner.ts +++ b/packages/cli-v3/src/utilities/initialBanner.ts @@ -4,9 +4,32 @@ import { VERSION } from "../version.js"; import { chalkGrey, chalkRun, chalkTask, chalkWorker, logo } from "./cliOutput.js"; import { logger } from "./logger.js"; import { spinner } from "./windows.js"; +import { + DEFFAULT_PROFILE, + readAuthConfigCurrentProfileName, + readAuthConfigProfile, +} from "./configFiles.js"; +import { CLOUD_API_URL } from "../consts.js"; + +function getProfileInfo() { + const currentProfile = readAuthConfigCurrentProfileName(); + const profile = readAuthConfigProfile(currentProfile); + + if (currentProfile === DEFFAULT_PROFILE || !profile) { + return; + } + + return `Profile: ${currentProfile}${ + profile.apiUrl === CLOUD_API_URL ? "" : ` - ${profile.apiUrl}` + }`; +} export async function printInitialBanner(performUpdateCheck = true) { - const text = `\n${logo()} ${chalkGrey(`(${VERSION})`)}\n`; + const profileInfo = getProfileInfo(); + + const text = `\n${logo()} ${chalkGrey(`(${VERSION})`)}${ + profileInfo ? chalkGrey(` | ${profileInfo}`) : "" + }\n`; logger.info(text); @@ -40,19 +63,23 @@ After installation, run Trigger.dev with \`npx trigger.dev\`.` } export async function printStandloneInitialBanner(performUpdateCheck = true) { + const profileInfo = getProfileInfo(); + const profileText = profileInfo ? chalkGrey(` | ${profileInfo}`) : ""; + + let versionText = `\n${logo()} ${chalkGrey(`(${VERSION})`)}`; + if (performUpdateCheck) { const maybeNewVersion = await updateCheck(); // Log a slightly more noticeable message if this is a major bump if (maybeNewVersion !== undefined) { - logger.log(`\n${logo()} ${chalkGrey(`(${VERSION} -> ${chalk.green(maybeNewVersion)})`)}`); + versionText = `\n${logo()} ${chalkGrey(`(${VERSION} -> ${chalk.green(maybeNewVersion)})`)}`; } else { - logger.log(`\n${logo()} ${chalkGrey(`(${VERSION})`)}`); + versionText = `\n${logo()} ${chalkGrey(`(${VERSION})`)}`; } - } else { - logger.log(`\n${logo()} ${chalkGrey(`(${VERSION})`)}`); } + logger.log(`${versionText}${profileText}`); logger.log(`${chalkGrey("-".repeat(54))}`); } diff --git a/packages/core/package.json b/packages/core/package.json index 7d75387a6c..4d21f2914b 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -173,7 +173,7 @@ }, "sideEffects": false, "scripts": { - "clean": "rimraf dist", + "clean": "rimraf dist .tshy .tshy-build .turbo", "update-version": "tsx ../../scripts/updateVersion.ts", "build": "tshy && pnpm run update-version", "dev": "tshy --watch", @@ -182,6 +182,7 @@ "check-exports": "attw --pack ." }, "dependencies": { + "@bugsnag/cuid": "^3.1.1", "@electric-sql/client": "1.0.0-beta.1", "@google-cloud/precise-date": "^4.0.0", "@jsonhero/path": "^1.0.21", @@ -212,8 +213,8 @@ "@ai-sdk/provider-utils": "^1.0.22", "@arethetypeswrong/cli": "^0.15.4", "@epic-web/test-server": "^0.1.0", + "@trigger.dev/database": "workspace:*", "@types/humanize-duration": "^3.27.1", - "@types/node": "20.14.14", "@types/readable-stream": "^4.0.14", "ai": "^3.4.33", "defu": "^6.1.4", @@ -223,7 +224,6 @@ "ts-essentials": "10.0.1", "tshy": "^3.0.2", "tsx": "4.17.0", - "typescript": "^5.5.4", "vitest": "^1.6.0" }, "engines": { diff --git a/packages/core/src/v3/apiClient/index.ts b/packages/core/src/v3/apiClient/index.ts index 2b778a14d8..0b062914ea 100644 --- a/packages/core/src/v3/apiClient/index.ts +++ b/packages/core/src/v3/apiClient/index.ts @@ -692,6 +692,7 @@ export class ApiClient { "Content-Type": "application/json", Authorization: `Bearer ${this.accessToken}`, "trigger-version": VERSION, + "x-trigger-engine-version": taskContext.worker?.engine ?? "V1", ...Object.entries(additionalHeaders ?? {}).reduce( (acc, [key, value]) => { if (value !== undefined) { diff --git a/packages/core/src/v3/apps/consts.ts b/packages/core/src/v3/apps/consts.ts new file mode 100644 index 0000000000..6789d897a9 --- /dev/null +++ b/packages/core/src/v3/apps/consts.ts @@ -0,0 +1,2 @@ +export const CURRENT_DEPLOYMENT_LABEL = "current"; +export const CURRENT_UNMANAGED_DEPLOYMENT_LABEL = "current-unmanaged"; diff --git a/packages/core/src/v3/apps/duration.ts b/packages/core/src/v3/apps/duration.ts index 85c0dbc88c..d14271c8c9 100644 --- a/packages/core/src/v3/apps/duration.ts +++ b/packages/core/src/v3/apps/duration.ts @@ -49,3 +49,25 @@ export function parseNaturalLanguageDuration(duration: string): Date | undefined return undefined; } + +export function stringifyDuration(seconds: number): string | undefined { + if (seconds <= 0) { + return; + } + + const units = { + w: Math.floor(seconds / 604800), + d: Math.floor((seconds % 604800) / 86400), + h: Math.floor((seconds % 86400) / 3600), + m: Math.floor((seconds % 3600) / 60), + s: Math.floor(seconds % 60), + }; + + // Filter the units having non-zero values and join them + const result: string = Object.entries(units) + .filter(([unit, val]) => val != 0) + .map(([unit, val]) => `${val}${unit}`) + .join(""); + + return result; +} diff --git a/packages/core/src/v3/apps/friendlyId.ts b/packages/core/src/v3/apps/friendlyId.ts index 1036edf297..c95813eb48 100644 --- a/packages/core/src/v3/apps/friendlyId.ts +++ b/packages/core/src/v3/apps/friendlyId.ts @@ -1,7 +1,83 @@ import { customAlphabet } from "nanoid"; +import cuid from "@bugsnag/cuid"; const idGenerator = customAlphabet("123456789abcdefghijkmnopqrstuvwxyz", 21); export function generateFriendlyId(prefix: string, size?: number) { return `${prefix}_${idGenerator(size)}`; } + +export function generateInternalId() { + return cuid(); +} + +/** Convert an internal ID to a friendly ID */ +export function toFriendlyId(entityName: string, internalId: string): string { + if (!entityName) { + throw new Error("Entity name cannot be empty"); + } + + if (!internalId) { + throw new Error("Internal ID cannot be empty"); + } + + return `${entityName}_${internalId}`; +} + +/** Convert a friendly ID to an internal ID */ +export function fromFriendlyId(friendlyId: string, expectedEntityName?: string): string { + if (!friendlyId) { + throw new Error("Friendly ID cannot be empty"); + } + + const parts = friendlyId.split("_"); + + if (parts.length !== 2) { + throw new Error("Invalid friendly ID format"); + } + + const [entityName, internalId] = parts; + + if (!entityName) { + throw new Error("Entity name cannot be empty"); + } + + if (!internalId) { + throw new Error("Internal ID cannot be empty"); + } + + if (expectedEntityName && entityName !== expectedEntityName) { + throw new Error(`Invalid entity name: ${entityName}`); + } + + return internalId; +} + +export class IdUtil { + constructor(private entityName: string) {} + + generate() { + const internalId = generateInternalId(); + + return { + id: internalId, + friendlyId: this.toFriendlyId(internalId), + }; + } + + toFriendlyId(internalId: string) { + return toFriendlyId(this.entityName, internalId); + } + + fromFriendlyId(friendlyId: string) { + return fromFriendlyId(friendlyId); + } +} + +export const BackgroundWorkerId = new IdUtil("worker"); +export const CheckpointId = new IdUtil("checkpoint"); +export const QueueId = new IdUtil("queue"); +export const RunId = new IdUtil("run"); +export const SnapshotId = new IdUtil("snapshot"); +export const WaitpointId = new IdUtil("waitpoint"); +export const BatchId = new IdUtil("batch"); diff --git a/packages/core/src/v3/apps/http.ts b/packages/core/src/v3/apps/http.ts index 70de522c00..03d852630a 100644 --- a/packages/core/src/v3/apps/http.ts +++ b/packages/core/src/v3/apps/http.ts @@ -14,6 +14,33 @@ export const getTextBody = (req: IncomingMessage) => }); }); +export async function getJsonBody(req: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + let body = ""; + + req.on("data", (chunk) => { + body += chunk.toString(); + }); + + req.on("end", () => { + resolve(safeJsonParse(body)); + }); + }); +} + +function safeJsonParse(text: string) { + if (!text) { + return null; + } + + try { + return JSON.parse(text); + } catch (error) { + console.error("Failed to parse JSON", { error, text }); + return null; + } +} + export class HttpReply { constructor(private response: Parameters[1]) {} diff --git a/packages/core/src/v3/apps/index.ts b/packages/core/src/v3/apps/index.ts index 97266f729f..c4028cf4b2 100644 --- a/packages/core/src/v3/apps/index.ts +++ b/packages/core/src/v3/apps/index.ts @@ -7,3 +7,6 @@ export * from "./provider.js"; export * from "./isExecaChildProcess.js"; export * from "./friendlyId.js"; export * from "./duration.js"; +export * from "./maxDuration.js"; +export * from "./queueName.js"; +export * from "./consts.js"; diff --git a/packages/core/src/v3/apps/maxDuration.ts b/packages/core/src/v3/apps/maxDuration.ts new file mode 100644 index 0000000000..b19d2786fd --- /dev/null +++ b/packages/core/src/v3/apps/maxDuration.ts @@ -0,0 +1,22 @@ +const MINIMUM_MAX_DURATION = 5; +const MAXIMUM_MAX_DURATION = 2_147_483_647; // largest 32-bit signed integer + +export function clampMaxDuration(maxDuration: number): number { + return Math.min(Math.max(maxDuration, MINIMUM_MAX_DURATION), MAXIMUM_MAX_DURATION); +} + +export function getMaxDuration( + maxDuration?: number | null, + defaultMaxDuration?: number | null +): number | undefined { + if (!maxDuration) { + return defaultMaxDuration ?? undefined; + } + + // Setting the maxDuration to MAXIMUM_MAX_DURATION means we don't want to use the default maxDuration + if (maxDuration === MAXIMUM_MAX_DURATION) { + return; + } + + return maxDuration; +} diff --git a/packages/core/src/v3/apps/queueName.ts b/packages/core/src/v3/apps/queueName.ts new file mode 100644 index 0000000000..1416148978 --- /dev/null +++ b/packages/core/src/v3/apps/queueName.ts @@ -0,0 +1,4 @@ +// Only allow alphanumeric characters, underscores, hyphens, and slashes (and only the first 128 characters) +export function sanitizeQueueName(queueName: string) { + return queueName.replace(/[^a-zA-Z0-9_\-\/]/g, "").substring(0, 128); +} diff --git a/packages/core/src/v3/build/resolvedConfig.ts b/packages/core/src/v3/build/resolvedConfig.ts index 674a7fce14..06caa8d256 100644 --- a/packages/core/src/v3/build/resolvedConfig.ts +++ b/packages/core/src/v3/build/resolvedConfig.ts @@ -1,6 +1,6 @@ import { type Defu } from "defu"; import type { Prettify } from "ts-essentials"; -import { TriggerConfig } from "../config.js"; +import { CompatibilityFlag, CompatibilityFlagFeatures, TriggerConfig } from "../config.js"; import { BuildRuntime } from "../schemas/build.js"; import { ResolveEnvironmentVariablesFunction } from "../types/index.js"; @@ -16,6 +16,8 @@ export type ResolvedConfig = Prettify< build: { jsx: { factory: string; fragment: string; automatic: true }; } & Omit, "jsx">; + compatibilityFlags: CompatibilityFlag[]; + features: CompatibilityFlagFeatures; }, ] > & { diff --git a/packages/core/src/v3/build/runtime.ts b/packages/core/src/v3/build/runtime.ts index d473f0f7f8..d03a49c484 100644 --- a/packages/core/src/v3/build/runtime.ts +++ b/packages/core/src/v3/build/runtime.ts @@ -2,11 +2,12 @@ import { join } from "node:path"; import { pathToFileURL } from "url"; import { BuildRuntime } from "../schemas/build.js"; -export const DEFAULT_RUNTIME: BuildRuntime = "node"; +export const DEFAULT_RUNTIME = "node" satisfies BuildRuntime; export function binaryForRuntime(runtime: BuildRuntime): string { switch (runtime) { case "node": + case "node-22": return "node"; case "bun": return "bun"; @@ -18,6 +19,7 @@ export function binaryForRuntime(runtime: BuildRuntime): string { export function execPathForRuntime(runtime: BuildRuntime): string { switch (runtime) { case "node": + case "node-22": return process.execPath; case "bun": if (typeof process.env.BUN_INSTALL === "string") { @@ -41,7 +43,8 @@ export type ExecOptions = { export function execOptionsForRuntime(runtime: BuildRuntime, options: ExecOptions): string { switch (runtime) { - case "node": { + case "node": + case "node-22": { const importEntryPoint = options.loaderEntryPoint ? `--import=${pathToFileURL(options.loaderEntryPoint).href}` : undefined; diff --git a/packages/core/src/v3/config.ts b/packages/core/src/v3/config.ts index 42ca53c616..a21534df73 100644 --- a/packages/core/src/v3/config.ts +++ b/packages/core/src/v3/config.ts @@ -10,6 +10,12 @@ import type { } from "./types/index.js"; import type { BuildRuntime, RetryOptions } from "./index.js"; +export type CompatibilityFlag = "run_engine_v2"; + +export type CompatibilityFlagFeatures = { + [key in CompatibilityFlag]: boolean; +}; + export type TriggerConfig = { /** * @default "node" @@ -23,6 +29,7 @@ export type TriggerConfig = { enabledInDev?: boolean; default?: RetryOptions; }; + compatibilityFlags?: Array; /** * The default machine preset to use for your deployed trigger.dev tasks. You can override this on a per-task basis. * @default "small-1x" diff --git a/packages/core/src/v3/errors.ts b/packages/core/src/v3/errors.ts index 232bb8aa94..285a753724 100644 --- a/packages/core/src/v3/errors.ts +++ b/packages/core/src/v3/errors.ts @@ -166,10 +166,19 @@ export function shouldRetryError(error: TaskRunError): boolean { case "TASK_RUN_CANCELLED": case "MAX_DURATION_EXCEEDED": case "DISK_SPACE_EXCEEDED": - case "TASK_RUN_HEARTBEAT_TIMEOUT": case "OUTDATED_SDK_VERSION": + case "TASK_RUN_HEARTBEAT_TIMEOUT": + // run engine errors + case "TASK_DEQUEUED_INVALID_STATE": + case "TASK_DEQUEUED_QUEUE_NOT_FOUND": + case "TASK_HAS_N0_EXECUTION_SNAPSHOT": + case "TASK_RUN_DEQUEUED_MAX_RETRIES": return false; + //new heartbeat error + //todo + case "TASK_RUN_STALLED_EXECUTING": + case "TASK_RUN_STALLED_EXECUTING_WITH_WAITPOINTS": case "GRACEFUL_EXIT_TIMEOUT": case "HANDLE_ERROR_ERROR": case "TASK_INPUT_ERROR": diff --git a/packages/core/src/v3/index.ts b/packages/core/src/v3/index.ts index 12fbc8b2d2..2d8eaed951 100644 --- a/packages/core/src/v3/index.ts +++ b/packages/core/src/v3/index.ts @@ -60,6 +60,9 @@ export { type IOPacket, } from "./utils/ioSerialization.js"; +export * from "./utils/imageRef.js"; +export * from "./utils/heartbeat.js"; + export * from "./config.js"; export { getSchemaParseFn, type AnySchemaParseFn, type SchemaParseFn } from "./types/schemas.js"; diff --git a/packages/core/src/v3/runtime/managedRuntimeManager.ts b/packages/core/src/v3/runtime/managedRuntimeManager.ts new file mode 100644 index 0000000000..90eddcd5e7 --- /dev/null +++ b/packages/core/src/v3/runtime/managedRuntimeManager.ts @@ -0,0 +1,147 @@ +import { + BatchTaskRunExecutionResult, + CompletedWaitpoint, + RuntimeWait, + TaskRunContext, + TaskRunExecutionResult, + TaskRunFailedExecutionResult, + TaskRunSuccessfulExecutionResult, +} from "../schemas/index.js"; +import { ExecutorToWorkerProcessConnection } from "../zodIpc.js"; +import { RuntimeManager } from "./manager.js"; + +type Resolver = (value: CompletedWaitpoint) => void; + +export class ManagedRuntimeManager implements RuntimeManager { + // Maps a resolver ID to a resolver function + private readonly resolversByWaitId: Map = new Map(); + // Maps a waitpoint ID to a wait ID + private readonly resolversByWaitpoint: Map = new Map(); + + constructor(private ipc: ExecutorToWorkerProcessConnection) { + setTimeout(() => { + console.log("Runtime status", { + resolversbyWaitId: this.resolversByWaitId.keys(), + resolversByWaitpoint: this.resolversByWaitpoint.keys(), + }); + }, 1000); + } + + disable(): void { + // do nothing + } + + async waitForDuration(ms: number): Promise { + const wait = { + type: "DATETIME", + id: crypto.randomUUID(), + date: new Date(Date.now() + ms), + } satisfies RuntimeWait; + + const promise = new Promise((resolve) => { + this.resolversByWaitId.set(wait.id, resolve); + }); + + // Send wait to parent process + this.ipc.send("WAIT", { wait }); + + await promise; + } + + async waitUntil(date: Date): Promise { + return this.waitForDuration(date.getTime() - Date.now()); + } + + async waitForTask(params: { id: string; ctx: TaskRunContext }): Promise { + const promise = new Promise((resolve) => { + this.resolversByWaitId.set(params.id, resolve); + }); + + const waitpoint = await promise; + const result = this.waitpointToTaskRunExecutionResult(waitpoint); + + return result; + } + + async waitForBatch(params: { + id: string; + runs: string[]; + ctx: TaskRunContext; + }): Promise { + if (!params.runs.length) { + return Promise.resolve({ id: params.id, items: [] }); + } + + const promise = Promise.all( + params.runs.map((runId) => { + return new Promise((resolve, reject) => { + this.resolversByWaitId.set(runId, resolve); + }); + }) + ); + + const waitpoints = await promise; + + return { + id: params.id, + items: waitpoints.map(this.waitpointToTaskRunExecutionResult), + }; + } + + associateWaitWithWaitpoint(waitId: string, waitpointId: string) { + this.resolversByWaitpoint.set(waitpointId, waitId); + } + + async completeWaitpoints(waitpoints: CompletedWaitpoint[]): Promise { + await Promise.all(waitpoints.map((waitpoint) => this.completeWaitpoint(waitpoint))); + } + + private completeWaitpoint(waitpoint: CompletedWaitpoint): void { + console.log("completeWaitpoint", waitpoint); + + const waitId = + waitpoint.completedByTaskRun?.friendlyId ?? this.resolversByWaitpoint.get(waitpoint.id); + + if (!waitId) { + // TODO: Handle failures better + console.log("No waitId found for waitpoint", waitpoint); + return; + } + + const resolve = this.resolversByWaitId.get(waitId); + + if (!resolve) { + // TODO: Handle failures better + console.log("No resolver found for waitId", waitId); + return; + } + + console.log("Resolving waitpoint", waitpoint); + + resolve(waitpoint); + + this.resolversByWaitId.delete(waitId); + } + + private waitpointToTaskRunExecutionResult(waitpoint: CompletedWaitpoint): TaskRunExecutionResult { + if (waitpoint.outputIsError) { + return { + ok: false, + id: waitpoint.id, + error: waitpoint.output + ? JSON.parse(waitpoint.output) + : { + type: "STRING_ERROR", + message: "Missing error output", + }, + } satisfies TaskRunFailedExecutionResult; + } else { + return { + ok: true, + id: waitpoint.id, + output: waitpoint.output, + outputType: waitpoint.outputType ?? "application/json", + } satisfies TaskRunSuccessfulExecutionResult; + } + } +} diff --git a/packages/core/src/v3/runtime/unmanagedRuntimeManager.ts b/packages/core/src/v3/runtime/unmanagedRuntimeManager.ts new file mode 100644 index 0000000000..88b0350590 --- /dev/null +++ b/packages/core/src/v3/runtime/unmanagedRuntimeManager.ts @@ -0,0 +1,81 @@ +import { + BatchTaskRunExecutionResult, + TaskRunContext, + TaskRunExecutionResult, +} from "../schemas/index.js"; +import { RuntimeManager } from "./manager.js"; +import { unboundedTimeout } from "../utils/timers.js"; + +type Waitpoint = any; + +export class UnmanagedRuntimeManager implements RuntimeManager { + private readonly waitpoints: Map = new Map(); + + _taskWaits: Map void }> = new Map(); + + _batchWaits: Map< + string, + { resolve: (value: BatchTaskRunExecutionResult) => void; reject: (err?: any) => void } + > = new Map(); + + disable(): void { + // do nothing + } + + async waitForDuration(ms: number): Promise { + await unboundedTimeout(ms); + } + + async waitUntil(date: Date): Promise { + return this.waitForDuration(date.getTime() - Date.now()); + } + + async waitForTask(params: { id: string; ctx: TaskRunContext }): Promise { + const promise = new Promise((resolve) => { + this._taskWaits.set(params.id, { resolve }); + }); + + return await promise; + } + + async waitForBatch(params: { + id: string; + runs: string[]; + ctx: TaskRunContext; + }): Promise { + if (!params.runs.length) { + return Promise.resolve({ id: params.id, items: [] }); + } + + const promise = Promise.all( + params.runs.map((runId) => { + return new Promise((resolve, reject) => { + this._taskWaits.set(runId, { resolve }); + }); + }) + ); + + const results = await promise; + + return { + id: params.id, + items: results, + }; + } + + async completeWaitpoints(waitpoints: Waitpoint[]): Promise { + await Promise.all(waitpoints.map((waitpoint) => this.completeWaitpoint(waitpoint))); + } + + private completeWaitpoint(waitpoint: Waitpoint): void { + const wait = this._taskWaits.get(waitpoint.id); + + if (!wait) { + return; + } + + wait.resolve(waitpoint.completion); + + this._taskWaits.delete(waitpoint.id); + } +} diff --git a/packages/core/src/v3/schemas/api.ts b/packages/core/src/v3/schemas/api.ts index 55189d8140..609f7c8e15 100644 --- a/packages/core/src/v3/schemas/api.ts +++ b/packages/core/src/v3/schemas/api.ts @@ -1,4 +1,4 @@ -import { z } from "zod"; +import { coerce, date, z } from "zod"; import { DeserializedJsonSchema } from "../../schemas/json.js"; import { SerializedError, TaskRunError } from "./common.js"; import { BackgroundWorkerMetadata } from "./resources.js"; @@ -68,10 +68,30 @@ export const TriggerTaskRequestBody = z.object({ context: z.any(), options: z .object({ + /** @deprecated engine v1 only */ dependentAttempt: z.string().optional(), + /** @deprecated engine v1 only */ parentAttempt: z.string().optional(), + /** @deprecated engine v1 only */ dependentBatch: z.string().optional(), + /** + * If triggered in a batch, this is the BatchTaskRun id + */ parentBatch: z.string().optional(), + /** + * RunEngine v2 + * If triggered inside another run, the parentRunId is the friendly ID of the parent run. + */ + parentRunId: z.string().optional(), + /** + * RunEngine v2 + * Should be `true` if `triggerAndWait` or `batchTriggerAndWait` + */ + resumeParentOnCompletion: z.boolean().optional(), + /** + * Locks the version to the passed value. + * Automatically set when using `triggerAndWait` or `batchTriggerAndWait` + */ lockToVersion: z.string().optional(), queue: QueueOptions.optional(), concurrencyKey: z.string().optional(), @@ -134,7 +154,18 @@ export type BatchTriggerTaskItem = z.infer; export const BatchTriggerTaskV2RequestBody = z.object({ items: BatchTriggerTaskItem.array(), + /** @deprecated engine v1 only */ dependentAttempt: z.string().optional(), + /** + * RunEngine v2 + * If triggered inside another run, the parentRunId is the friendly ID of the parent run. + */ + parentRunId: z.string().optional(), + /** + * RunEngine v2 + * Should be `true` if `triggerAndWait` or `batchTriggerAndWait` + */ + resumeParentOnCompletion: z.boolean().optional(), }); export type BatchTriggerTaskV2RequestBody = z.infer; @@ -244,6 +275,7 @@ export const InitializeDeploymentRequestBody = z.object({ registryHost: z.string().optional(), selfHosted: z.boolean().optional(), namespace: z.string().optional(), + type: z.enum(["MANAGED", "UNMANAGED", "V1"]).optional(), }); export type InitializeDeploymentRequestBody = z.infer; @@ -303,10 +335,45 @@ export const GetDeploymentResponseBody = z.object({ export type GetDeploymentResponseBody = z.infer; +export const GetLatestDeploymentResponseBody = GetDeploymentResponseBody.omit({ + worker: true, +}); +export type GetLatestDeploymentResponseBody = z.infer; + export const CreateUploadPayloadUrlResponseBody = z.object({ presignedUrl: z.string(), }); +export const WorkersListResponseBody = z + .object({ + type: z.string(), + name: z.string(), + description: z.string().nullish(), + latestVersion: z.string().nullish(), + lastHeartbeatAt: z.string().nullish(), + isDefault: z.boolean(), + updatedAt: z.coerce.date(), + }) + .array(); +export type WorkersListResponseBody = z.infer; + +export const WorkersCreateRequestBody = z.object({ + name: z.string().optional(), + description: z.string().optional(), +}); +export type WorkersCreateRequestBody = z.infer; + +export const WorkersCreateResponseBody = z.object({ + workerGroup: z.object({ + name: z.string(), + description: z.string().nullish(), + }), + token: z.object({ + plaintext: z.string(), + }), +}); +export type WorkersCreateResponseBody = z.infer; + export type CreateUploadPayloadUrlResponseBody = z.infer; export const ReplayRunResponse = z.object({ diff --git a/packages/core/src/v3/schemas/build.ts b/packages/core/src/v3/schemas/build.ts index 83518b5341..d8a8d8545b 100644 --- a/packages/core/src/v3/schemas/build.ts +++ b/packages/core/src/v3/schemas/build.ts @@ -9,11 +9,11 @@ export const BuildExternal = z.object({ export type BuildExternal = z.infer; -export const BuildTarget = z.enum(["dev", "deploy"]); +export const BuildTarget = z.enum(["dev", "deploy", "managed", "unmanaged"]); export type BuildTarget = z.infer; -export const BuildRuntime = z.enum(["node", "bun"]); +export const BuildRuntime = z.enum(["node", "node-22", "bun"]); export type BuildRuntime = z.infer; diff --git a/packages/core/src/v3/schemas/common.ts b/packages/core/src/v3/schemas/common.ts index 270d87e6df..1f9f18f4c3 100644 --- a/packages/core/src/v3/schemas/common.ts +++ b/packages/core/src/v3/schemas/common.ts @@ -104,6 +104,12 @@ export const TaskRunInternalError = z.object({ "DISK_SPACE_EXCEEDED", "POD_EVICTED", "POD_UNKNOWN_ERROR", + "TASK_HAS_N0_EXECUTION_SNAPSHOT", + "TASK_DEQUEUED_INVALID_STATE", + "TASK_DEQUEUED_QUEUE_NOT_FOUND", + "TASK_RUN_DEQUEUED_MAX_RETRIES", + "TASK_RUN_STALLED_EXECUTING", + "TASK_RUN_STALLED_EXECUTING_WITH_WAITPOINTS", "OUTDATED_SDK_VERSION", ]), message: z.string().optional(), @@ -127,19 +133,32 @@ export const TaskRun = z.object({ id: z.string(), payload: z.string(), payloadType: z.string(), - context: z.any(), tags: z.array(z.string()), isTest: z.boolean().default(false), createdAt: z.coerce.date(), startedAt: z.coerce.date().default(() => new Date()), idempotencyKey: z.string().optional(), maxAttempts: z.number().optional(), - durationMs: z.number().default(0), - costInCents: z.number().default(0), - baseCostInCents: z.number().default(0), version: z.string().optional(), metadata: z.record(DeserializedJsonSchema).optional(), maxDuration: z.number().optional(), + /** @deprecated */ + context: z.any(), + /** + * @deprecated For live values use the `usage` SDK functions + * @link https://trigger.dev/docs/run-usage + */ + durationMs: z.number().default(0), + /** + * @deprecated For live values use the `usage` SDK functions + * @link https://trigger.dev/docs/run-usage + */ + costInCents: z.number().default(0), + /** + * @deprecated For live values use the `usage` SDK functions + * @link https://trigger.dev/docs/run-usage + */ + baseCostInCents: z.number().default(0), }); export type TaskRun = z.infer; @@ -153,11 +172,15 @@ export const TaskRunExecutionTask = z.object({ export type TaskRunExecutionTask = z.infer; export const TaskRunExecutionAttempt = z.object({ - id: z.string(), number: z.number(), startedAt: z.coerce.date(), + /** @deprecated */ + id: z.string(), + /** @deprecated */ backgroundWorkerId: z.string(), + /** @deprecated */ backgroundWorkerTaskId: z.string(), + /** @deprecated */ status: z.string(), }); @@ -202,7 +225,11 @@ export const TaskRunExecutionBatch = z.object({ export const TaskRunExecution = z.object({ task: TaskRunExecutionTask, attempt: TaskRunExecutionAttempt, - run: TaskRun, + run: TaskRun.and( + z.object({ + traceContext: z.record(z.unknown()).optional(), + }) + ), queue: TaskRunExecutionQueue, environment: TaskRunExecutionEnvironment, organization: TaskRunExecutionOrganization, @@ -232,6 +259,7 @@ export type TaskRunContext = z.infer; export const TaskRunExecutionRetry = z.object({ timestamp: z.number(), + /** Retry delay in milliseconds */ delay: z.number(), error: z.unknown().optional(), }); diff --git a/packages/core/src/v3/schemas/index.ts b/packages/core/src/v3/schemas/index.ts index 6f8b74e64e..7a99fd1575 100644 --- a/packages/core/src/v3/schemas/index.ts +++ b/packages/core/src/v3/schemas/index.ts @@ -10,3 +10,4 @@ export * from "./eventFilter.js"; export * from "./openTelemetry.js"; export * from "./config.js"; export * from "./build.js"; +export * from "./runEngine.js"; diff --git a/packages/core/src/v3/schemas/messages.ts b/packages/core/src/v3/schemas/messages.ts index 31d6dc3e4f..9da770646a 100644 --- a/packages/core/src/v3/schemas/messages.ts +++ b/packages/core/src/v3/schemas/messages.ts @@ -12,9 +12,12 @@ import { EnvironmentType, ProdTaskRunExecution, ProdTaskRunExecutionPayload, + RunEngineVersionSchema, + RuntimeWait, TaskRunExecutionLazyAttemptPayload, WaitReason, } from "./schemas.js"; +import { CompletedWaitpoint } from "./runEngine.js"; const ackCallbackResult = z.discriminatedUnion("success", [ z.object({ @@ -101,6 +104,7 @@ export const ServerBackgroundWorker = z.object({ id: z.string(), version: z.string(), contentHash: z.string(), + engine: RunEngineVersionSchema.optional(), }); export type ServerBackgroundWorker = z.infer; @@ -191,6 +195,12 @@ export const ExecutorToWorkerMessageCatalog = { UNCAUGHT_EXCEPTION: { message: UncaughtExceptionMessage, }, + WAIT: { + message: z.object({ + version: z.literal("v1").default("v1"), + wait: RuntimeWait, + }), + }, }; export const WorkerToExecutorMessageCatalog = { @@ -226,6 +236,23 @@ export const WorkerToExecutorMessageCatalog = { }), callback: z.void(), }, + WAITPOINT_CREATED: { + message: z.object({ + version: z.literal("v1").default("v1"), + wait: z.object({ + id: z.string(), + }), + waitpoint: z.object({ + id: z.string(), + }), + }), + }, + WAITPOINT_COMPLETED: { + message: z.object({ + version: z.literal("v1").default("v1"), + waitpoint: CompletedWaitpoint, + }), + }, }; export const ProviderToPlatformMessages = { diff --git a/packages/core/src/v3/schemas/runEngine.ts b/packages/core/src/v3/schemas/runEngine.ts new file mode 100644 index 0000000000..2eb5c52680 --- /dev/null +++ b/packages/core/src/v3/schemas/runEngine.ts @@ -0,0 +1,193 @@ +import { z } from "zod"; +import { MachinePreset, TaskRunExecution } from "./common.js"; +import { EnvironmentType } from "./schemas.js"; +import type * as DB_TYPES from "@trigger.dev/database"; + +type Enum = { [K in T]: K }; + +export const TaskRunExecutionStatus = { + RUN_CREATED: "RUN_CREATED", + QUEUED: "QUEUED", + PENDING_EXECUTING: "PENDING_EXECUTING", + EXECUTING: "EXECUTING", + EXECUTING_WITH_WAITPOINTS: "EXECUTING_WITH_WAITPOINTS", + BLOCKED_BY_WAITPOINTS: "BLOCKED_BY_WAITPOINTS", + PENDING_CANCEL: "PENDING_CANCEL", + FINISHED: "FINISHED", +} satisfies Enum; + +export type TaskRunExecutionStatus = + (typeof TaskRunExecutionStatus)[keyof typeof TaskRunExecutionStatus]; + +export const TaskRunStatus = { + DELAYED: "DELAYED", + PENDING: "PENDING", + WAITING_FOR_DEPLOY: "WAITING_FOR_DEPLOY", + EXECUTING: "EXECUTING", + WAITING_TO_RESUME: "WAITING_TO_RESUME", + RETRYING_AFTER_FAILURE: "RETRYING_AFTER_FAILURE", + PAUSED: "PAUSED", + CANCELED: "CANCELED", + INTERRUPTED: "INTERRUPTED", + COMPLETED_SUCCESSFULLY: "COMPLETED_SUCCESSFULLY", + COMPLETED_WITH_ERRORS: "COMPLETED_WITH_ERRORS", + SYSTEM_FAILURE: "SYSTEM_FAILURE", + CRASHED: "CRASHED", + EXPIRED: "EXPIRED", + TIMED_OUT: "TIMED_OUT", +} satisfies Enum; + +export type TaskRunStatus = (typeof TaskRunStatus)[keyof typeof TaskRunStatus]; + +export const WaitpointType = { + RUN: "RUN", + DATETIME: "DATETIME", + MANUAL: "MANUAL", +} satisfies Enum; + +export type WaitpointType = (typeof WaitpointType)[keyof typeof WaitpointType]; + +export const CompletedWaitpoint = z.object({ + id: z.string(), + friendlyId: z.string(), + type: z.enum(Object.values(WaitpointType) as [WaitpointType]), + completedAt: z.coerce.date(), + idempotencyKey: z.string().optional(), + /** For type === "RUN" */ + completedByTaskRun: z + .object({ + id: z.string(), + friendlyId: z.string(), + }) + .optional(), + /** For type === "DATETIME" */ + completedAfter: z.coerce.date().optional(), + output: z.string().optional(), + outputType: z.string().optional(), + outputIsError: z.boolean(), +}); + +export type CompletedWaitpoint = z.infer; + +const ExecutionSnapshot = z.object({ + id: z.string(), + friendlyId: z.string(), + executionStatus: z.enum(Object.values(TaskRunExecutionStatus) as [TaskRunExecutionStatus]), + description: z.string(), +}); + +const BaseRunMetadata = z.object({ + id: z.string(), + friendlyId: z.string(), + status: z.enum(Object.values(TaskRunStatus) as [TaskRunStatus]), + attemptNumber: z.number().nullish(), +}); + +export const ExecutionResult = z.object({ + snapshot: ExecutionSnapshot, + run: BaseRunMetadata, +}); + +export type ExecutionResult = z.infer; + +/** This is sent to a Worker when a run is dequeued (a new run or continuing run) */ +export const DequeuedMessage = z.object({ + version: z.literal("1"), + snapshot: ExecutionSnapshot, + image: z.string().optional(), + checkpoint: z + .object({ + id: z.string(), + type: z.string(), + location: z.string(), + reason: z.string().nullish(), + }) + .optional(), + completedWaitpoints: z.array(CompletedWaitpoint), + backgroundWorker: z.object({ + id: z.string(), + friendlyId: z.string(), + version: z.string(), + }), + deployment: z.object({ + id: z.string().optional(), + friendlyId: z.string().optional(), + }), + run: z.object({ + id: z.string(), + friendlyId: z.string(), + isTest: z.boolean(), + machine: MachinePreset, + attemptNumber: z.number(), + masterQueue: z.string(), + traceContext: z.record(z.unknown()), + }), + environment: z.object({ + id: z.string(), + type: EnvironmentType, + }), + organization: z.object({ + id: z.string(), + }), + project: z.object({ + id: z.string(), + }), +}); +export type DequeuedMessage = z.infer; + +/** The response to the Worker when starting an attempt */ +export const StartRunAttemptResult = ExecutionResult.and( + z.object({ + execution: TaskRunExecution, + }) +); +export type StartRunAttemptResult = z.infer; + +/** The response to the Worker when completing an attempt */ +const CompleteAttemptStatus = z.enum([ + "RUN_FINISHED", + "RUN_PENDING_CANCEL", + "RETRY_QUEUED", + "RETRY_IMMEDIATELY", +]); +export type CompleteAttemptStatus = z.infer; + +export const CompleteRunAttemptResult = z + .object({ + attemptStatus: CompleteAttemptStatus, + }) + .and(ExecutionResult); +export type CompleteRunAttemptResult = z.infer; + +/** The response when a Worker asks for the latest execution state */ +export const RunExecutionData = z.object({ + version: z.literal("1"), + snapshot: ExecutionSnapshot, + run: BaseRunMetadata, + checkpoint: z + .object({ + id: z.string(), + friendlyId: z.string(), + type: z.string(), + location: z.string(), + imageRef: z.string(), + reason: z.string().optional(), + }) + .optional(), + completedWaitpoints: z.array(CompletedWaitpoint), +}); +export type RunExecutionData = z.infer; + +export const WaitForDurationResult = z + .object({ + /** + If you pass an idempotencyKey, you may actually not need to wait. + Use this date to determine when to continue. + */ + waitUntil: z.coerce.date(), + waitpoint: z.object({ + id: z.string(), + }), + }) + .and(ExecutionResult); +export type WaitForDurationResult = z.infer; diff --git a/packages/core/src/v3/schemas/schemas.ts b/packages/core/src/v3/schemas/schemas.ts index 42edf8602e..c4494f3dde 100644 --- a/packages/core/src/v3/schemas/schemas.ts +++ b/packages/core/src/v3/schemas/schemas.ts @@ -8,6 +8,8 @@ import { MachineConfig, MachinePreset, TaskRunExecution } from "./common.js"; export const EnvironmentType = z.enum(["PRODUCTION", "STAGING", "DEVELOPMENT", "PREVIEW"]); export type EnvironmentType = z.infer; +export const RunEngineVersionSchema = z.enum(["V1", "V2"]); + export const TaskRunExecutionPayload = z.object({ execution: TaskRunExecution, traceContext: z.record(z.unknown()), @@ -25,6 +27,7 @@ export const ProdTaskRunExecution = TaskRunExecution.extend({ id: z.string(), contentHash: z.string(), version: z.string(), + type: RunEngineVersionSchema.optional(), }), machine: MachinePreset.default({ name: "small-1x", cpu: 1, memory: 1, centsPerMs: 0 }), }); @@ -240,3 +243,17 @@ export const TaskRunExecutionLazyAttemptPayload = z.object({ }); export type TaskRunExecutionLazyAttemptPayload = z.infer; + +export const RuntimeWait = z.discriminatedUnion("type", [ + z.object({ + type: z.literal("DATETIME"), + id: z.string(), + date: z.coerce.date(), + }), + z.object({ + type: z.literal("MANUAL"), + id: z.string(), + }), +]); + +export type RuntimeWait = z.infer; diff --git a/apps/webapp/app/v3/services/heartbeatService.server.ts b/packages/core/src/v3/utils/heartbeat.ts similarity index 51% rename from apps/webapp/app/v3/services/heartbeatService.server.ts rename to packages/core/src/v3/utils/heartbeat.ts index 8db36b2beb..295d8ebc04 100644 --- a/apps/webapp/app/v3/services/heartbeatService.server.ts +++ b/packages/core/src/v3/utils/heartbeat.ts @@ -1,23 +1,34 @@ type HeartbeatServiceOptions = { heartbeat: () => Promise; - pingIntervalInMs?: number; + intervalMs?: number; leadingEdge?: boolean; + onError?: (error: unknown) => Promise; }; export class HeartbeatService { private _heartbeat: () => Promise; - private _heartbeatIntervalInMs: number; + private _intervalMs: number; private _nextHeartbeat: NodeJS.Timeout | undefined; private _leadingEdge: boolean; + private _isHeartbeating: boolean; + private _onError?: (error: unknown) => Promise; constructor(opts: HeartbeatServiceOptions) { this._heartbeat = opts.heartbeat; - this._heartbeatIntervalInMs = opts.pingIntervalInMs ?? 45_000; + this._intervalMs = opts.intervalMs ?? 45_000; this._nextHeartbeat = undefined; this._leadingEdge = opts.leadingEdge ?? false; + this._isHeartbeating = false; + this._onError = opts.onError; } start() { + if (this._isHeartbeating) { + return; + } + + this._isHeartbeating = true; + if (this._leadingEdge) { this.#doHeartbeat(); } else { @@ -26,13 +37,28 @@ export class HeartbeatService { } stop() { + if (!this._isHeartbeating) { + return; + } + + this._isHeartbeating = false; this.#clearNextHeartbeat(); } #doHeartbeat = async () => { this.#clearNextHeartbeat(); - await this._heartbeat(); + try { + await this._heartbeat(); + } catch (error) { + if (this._onError) { + try { + await this._onError(error); + } catch (error) { + console.error("Error handling heartbeat error", error); + } + } + } this.#scheduleNextHeartbeat(); }; @@ -44,6 +70,6 @@ export class HeartbeatService { } #scheduleNextHeartbeat() { - this._nextHeartbeat = setTimeout(this.#doHeartbeat, this._heartbeatIntervalInMs); + this._nextHeartbeat = setTimeout(this.#doHeartbeat, this._intervalMs); } } diff --git a/packages/core/src/v3/utils/imageRef.ts b/packages/core/src/v3/utils/imageRef.ts new file mode 100644 index 0000000000..88efba0fdf --- /dev/null +++ b/packages/core/src/v3/utils/imageRef.ts @@ -0,0 +1,69 @@ +export type DockerImageParts = { + registry?: string; + repo: string; + tag?: string; + digest?: string; +}; + +export function parseDockerImageReference(imageReference: string): DockerImageParts { + const parts: DockerImageParts = { repo: "" }; // Initialize with an empty repo + + // Splitting by '@' to separate the digest (if exists) + const atSplit = imageReference.split("@"); + if (atSplit.length > 1) { + parts.digest = atSplit[1]; + imageReference = atSplit[0] as string; + } + + // Splitting by ':' to separate the tag (if exists) and to ensure it's not part of a port + let colonSplit = imageReference.split(":"); + if ( + colonSplit.length > 2 || + (colonSplit.length === 2 && !(colonSplit[1] as string).includes("/")) + ) { + // It's a tag if there's no '/' in the second part (after colon), or there are more than 2 parts (implying a port number in registry) + parts.tag = colonSplit.pop(); // The last part is the tag + imageReference = colonSplit.join(":"); // Join back in case it was a port number + } + + // Check for registry + let slashIndex = imageReference.indexOf("/"); + if (slashIndex !== -1) { + let potentialRegistry = imageReference.substring(0, slashIndex); + // Validate if the first part is a valid hostname-like string (registry), otherwise treat the entire string as the repo + if ( + potentialRegistry.includes(".") || + potentialRegistry === "localhost" || + potentialRegistry.includes(":") + ) { + parts.registry = potentialRegistry; + parts.repo = imageReference.substring(slashIndex + 1); + } else { + parts.repo = imageReference; // No valid registry found, treat as repo + } + } else { + parts.repo = imageReference; // Only repo is present + } + + return parts; +} + +export function rebuildDockerImageReference(parts: DockerImageParts): string { + let imageReference = ""; + + if (parts.registry) { + imageReference += `${parts.registry}/`; + } + + imageReference += parts.repo; // Repo is now guaranteed to be defined + + if (parts.tag) { + imageReference += `:${parts.tag}`; + } + + if (parts.digest) { + imageReference += `@${parts.digest}`; + } + + return imageReference; +} diff --git a/packages/core/src/v3/workers/index.ts b/packages/core/src/v3/workers/index.ts index 504302dde2..2912d69387 100644 --- a/packages/core/src/v3/workers/index.ts +++ b/packages/core/src/v3/workers/index.ts @@ -16,3 +16,5 @@ export { ProdUsageManager, type ProdUsageManagerOptions } from "../usage/prodUsa export { UsageTimeoutManager } from "../timeout/usageTimeoutManager.js"; export { StandardMetadataManager } from "../runMetadata/manager.js"; export { StandardWaitUntilManager } from "../waitUntil/manager.js"; +export { ManagedRuntimeManager } from "../runtime/managedRuntimeManager.js"; +export { UnmanagedRuntimeManager } from "../runtime/unmanagedRuntimeManager.js"; diff --git a/packages/react-hooks/package.json b/packages/react-hooks/package.json index 94a8c3bffd..b1296c2fe5 100644 --- a/packages/react-hooks/package.json +++ b/packages/react-hooks/package.json @@ -29,7 +29,7 @@ ] }, "scripts": { - "clean": "rimraf dist", + "clean": "rimraf dist .tshy .tshy-build .turbo", "build": "tshy && pnpm run update-version", "dev": "tshy --watch", "typecheck": "tsc --noEmit", @@ -42,13 +42,11 @@ }, "devDependencies": { "@arethetypeswrong/cli": "^0.15.4", - "@types/node": "^20.14.14", "@types/react": "*", "@types/react-dom": "*", "rimraf": "^3.0.2", "tshy": "^3.0.2", - "tsx": "4.17.0", - "typescript": "^5.5.4" + "tsx": "4.17.0" }, "peerDependencies": { "react": "^18.0 || ^19.0 || ^19.0.0-rc", @@ -74,4 +72,4 @@ "main": "./dist/commonjs/index.js", "types": "./dist/commonjs/index.d.ts", "module": "./dist/esm/index.js" -} \ No newline at end of file +} diff --git a/packages/rsc/package.json b/packages/rsc/package.json index 15f5910bf5..68731484da 100644 --- a/packages/rsc/package.json +++ b/packages/rsc/package.json @@ -29,7 +29,7 @@ ] }, "scripts": { - "clean": "rimraf dist", + "clean": "rimraf dist .tshy .tshy-build .turbo", "build": "tshy && pnpm run update-version", "dev": "tshy --watch", "typecheck": "tsc --noEmit", @@ -45,13 +45,11 @@ "devDependencies": { "@arethetypeswrong/cli": "^0.15.4", "@trigger.dev/build": "workspace:^3.3.7", - "@types/node": "^20.14.14", "@types/react": "*", "@types/react-dom": "*", "rimraf": "^3.0.2", "tshy": "^3.0.2", - "tsx": "4.17.0", - "typescript": "^5.5.4" + "tsx": "4.17.0" }, "engines": { "node": ">=18.20.0" diff --git a/packages/trigger-sdk/package.json b/packages/trigger-sdk/package.json index 832d724221..b33d0e95c8 100644 --- a/packages/trigger-sdk/package.json +++ b/packages/trigger-sdk/package.json @@ -37,7 +37,7 @@ } }, "scripts": { - "clean": "rimraf dist", + "clean": "rimraf dist .tshy .tshy-build .turbo", "build": "tshy && pnpm run update-version", "dev": "tshy --watch", "typecheck": "tsc --noEmit", @@ -62,7 +62,6 @@ "devDependencies": { "@arethetypeswrong/cli": "^0.15.4", "@types/debug": "^4.1.7", - "@types/node": "20.14.14", "@types/slug": "^5.0.3", "@types/uuid": "^9.0.0", "@types/ws": "^8.5.3", @@ -72,7 +71,6 @@ "tshy": "^3.0.2", "tsx": "4.17.0", "typed-emitter": "^2.1.0", - "typescript": "^5.5.4", "zod": "3.23.8" }, "peerDependencies": { diff --git a/packages/trigger-sdk/src/v3/shared.ts b/packages/trigger-sdk/src/v3/shared.ts index f7513f8c49..86e0175662 100644 --- a/packages/trigger-sdk/src/v3/shared.ts +++ b/packages/trigger-sdk/src/v3/shared.ts @@ -617,6 +617,7 @@ export async function batchTriggerById( }; }) ), + parentRunId: taskContext.ctx?.run.id, }, { spanParentAsLink: true, @@ -791,6 +792,8 @@ export async function batchTriggerByIdAndWait( }) ), dependentAttempt: ctx.attempt.id, + parentRunId: ctx.run.id, + resumeParentOnCompletion: true, }, { processingStrategy: options?.triggerSequentially ? "sequential" : undefined, @@ -951,6 +954,7 @@ export async function batchTriggerTasks( }; }) ), + parentRunId: taskContext.ctx?.run.id, }, { spanParentAsLink: true, @@ -1127,6 +1131,8 @@ export async function batchTriggerAndWaitTasks( parentAttempt: taskContext.ctx?.attempt.id, metadata: options?.metadata, maxDuration: options?.maxDuration, + parentRunId: taskContext.ctx?.run.id, }, }, { @@ -1234,6 +1241,8 @@ async function batchTrigger_internal( ): Promise> { const apiClient = apiClientManager.clientOrThrow(); + const ctx = taskContext.ctx; + const response = await apiClient.batchTriggerV2( { items: await Promise.all( @@ -1259,6 +1268,7 @@ async function batchTrigger_internal( parentAttempt: taskContext.ctx?.attempt.id, metadata: item.options?.metadata, maxDuration: item.options?.maxDuration, + parentRunId: ctx?.run.id, }, }; }) @@ -1352,6 +1362,8 @@ async function triggerAndWait_internal=18.20.0" + }, + "exports": { + "./package.json": "./package.json", + ".": { + "import": { + "@triggerdotdev/source": "./src/index.ts", + "types": "./dist/esm/index.d.ts", + "default": "./dist/esm/index.js" + }, + "require": { + "types": "./dist/commonjs/index.d.ts", + "default": "./dist/commonjs/index.js" + } + } + }, + "main": "./dist/commonjs/index.js", + "types": "./dist/commonjs/index.d.ts", + "module": "./dist/esm/index.js" +} diff --git a/packages/worker/src/consts.ts b/packages/worker/src/consts.ts new file mode 100644 index 0000000000..7f1c475301 --- /dev/null +++ b/packages/worker/src/consts.ts @@ -0,0 +1,9 @@ +export const HEADER_NAME = { + WORKER_INSTANCE_NAME: "x-trigger-worker-instance-name", + WORKER_DEPLOYMENT_ID: "x-trigger-worker-deployment-id", + WORKER_MANAGED_SECRET: "x-trigger-worker-managed-secret", +}; + +export const WORKLOAD_HEADER_NAME = { + WORKLOAD_DEPLOYMENT_ID: "x-trigger-workload-deployment-id", +}; diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts new file mode 100644 index 0000000000..ae89268885 --- /dev/null +++ b/packages/worker/src/index.ts @@ -0,0 +1,8 @@ +export { VERSION as WORKER_VERSION } from "./version.js"; +export * from "./consts.js"; +export * from "./supervisor/http.js"; +export * from "./supervisor/schemas.js"; +export * from "./supervisor/session.js"; +export * from "./workload/http.js"; +export * from "./workload/schemas.js"; +export * from "./types.js"; diff --git a/packages/worker/src/supervisor/events.ts b/packages/worker/src/supervisor/events.ts new file mode 100644 index 0000000000..b08c330a37 --- /dev/null +++ b/packages/worker/src/supervisor/events.ts @@ -0,0 +1,54 @@ +import { + DequeuedMessage, + StartRunAttemptResult, + TaskRunExecutionResult, +} from "@trigger.dev/core/v3"; + +export type WorkerEvents = { + runQueueMessage: [ + { + time: Date; + message: DequeuedMessage; + }, + ]; + requestRunAttemptStart: [ + { + time: Date; + run: { + friendlyId: string; + }; + snapshot: { + friendlyId: string; + }; + }, + ]; + runAttemptStarted: [ + { + time: Date; + } & StartRunAttemptResult & { + envVars: Record; + }, + ]; + runAttemptCompleted: [ + { + time: Date; + run: { + friendlyId: string; + }; + snapshot: { + friendlyId: string; + }; + completion: TaskRunExecutionResult; + }, + ]; + runNotification: [ + { + time: Date; + run: { + friendlyId: string; + }; + }, + ]; +}; + +export type WorkerEventArgs = WorkerEvents[T]; diff --git a/packages/worker/src/supervisor/http.ts b/packages/worker/src/supervisor/http.ts new file mode 100644 index 0000000000..c5f50f4bf2 --- /dev/null +++ b/packages/worker/src/supervisor/http.ts @@ -0,0 +1,232 @@ +import { z } from "zod"; +import { zodfetch, ApiError } from "@trigger.dev/core/v3/zodfetch"; +import { + WorkerApiConnectRequestBody, + WorkerApiConnectResponseBody, + WorkerApiDequeueResponseBody, + WorkerApiHeartbeatRequestBody, + WorkerApiHeartbeatResponseBody, + WorkerApiRunAttemptCompleteRequestBody, + WorkerApiRunAttemptCompleteResponseBody, + WorkerApiRunAttemptStartRequestBody, + WorkerApiRunAttemptStartResponseBody, + WorkerApiRunHeartbeatRequestBody, + WorkerApiRunHeartbeatResponseBody, + WorkerApiRunLatestSnapshotResponseBody, + WorkerApiWaitForDurationRequestBody, + WorkerApiWaitForDurationResponseBody, +} from "./schemas.js"; +import { SupervisorClientCommonOptions } from "./types.js"; +import { getDefaultWorkerHeaders } from "./util.js"; + +type SupervisorHttpClientOptions = SupervisorClientCommonOptions; + +export class SupervisorHttpClient { + private readonly apiUrl: string; + private readonly workerToken: string; + private readonly instanceName: string; + private readonly defaultHeaders: Record; + + constructor(opts: SupervisorHttpClientOptions) { + this.apiUrl = opts.apiUrl.replace(/\/$/, ""); + this.workerToken = opts.workerToken; + this.instanceName = opts.instanceName; + this.defaultHeaders = getDefaultWorkerHeaders(opts); + + if (!this.apiUrl) { + throw new Error("apiURL is required and needs to be a non-empty string"); + } + + if (!this.workerToken) { + throw new Error("workerToken is required and needs to be a non-empty string"); + } + + if (!this.instanceName) { + throw new Error("instanceName is required and needs to be a non-empty string"); + } + } + + async connect(body: WorkerApiConnectRequestBody) { + return wrapZodFetch( + WorkerApiConnectResponseBody, + `${this.apiUrl}/api/v1/worker-actions/connect`, + { + method: "POST", + headers: { + ...this.defaultHeaders, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + } + ); + } + + async dequeue() { + return wrapZodFetch( + WorkerApiDequeueResponseBody, + `${this.apiUrl}/api/v1/worker-actions/dequeue`, + { + headers: { + ...this.defaultHeaders, + }, + } + ); + } + + async dequeueFromVersion(deploymentId: string) { + return wrapZodFetch( + WorkerApiDequeueResponseBody, + `${this.apiUrl}/api/v1/worker-actions/deployments/${deploymentId}/dequeue`, + { + headers: { + ...this.defaultHeaders, + }, + } + ); + } + + async heartbeatWorker(body: WorkerApiHeartbeatRequestBody) { + return wrapZodFetch( + WorkerApiHeartbeatResponseBody, + `${this.apiUrl}/api/v1/worker-actions/heartbeat`, + { + method: "POST", + headers: { + ...this.defaultHeaders, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + } + ); + } + + async heartbeatRun(runId: string, snapshotId: string, body: WorkerApiRunHeartbeatRequestBody) { + return wrapZodFetch( + WorkerApiRunHeartbeatResponseBody, + `${this.apiUrl}/api/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/heartbeat`, + { + method: "POST", + headers: { + ...this.defaultHeaders, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + } + ); + } + + async startRunAttempt( + runId: string, + snapshotId: string, + body: WorkerApiRunAttemptStartRequestBody + ) { + return wrapZodFetch( + WorkerApiRunAttemptStartResponseBody, + `${this.apiUrl}/api/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/attempts/start`, + { + method: "POST", + headers: { + ...this.defaultHeaders, + }, + body: JSON.stringify(body), + } + ); + } + + async completeRunAttempt( + runId: string, + snapshotId: string, + body: WorkerApiRunAttemptCompleteRequestBody + ) { + return wrapZodFetch( + WorkerApiRunAttemptCompleteResponseBody, + `${this.apiUrl}/api/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/attempts/complete`, + { + method: "POST", + headers: { + ...this.defaultHeaders, + }, + body: JSON.stringify(body), + } + ); + } + + async getLatestSnapshot(runId: string) { + return wrapZodFetch( + WorkerApiRunLatestSnapshotResponseBody, + `${this.apiUrl}/api/v1/worker-actions/runs/${runId}/snapshots/latest`, + { + method: "GET", + headers: { + ...this.defaultHeaders, + }, + } + ); + } + + async waitForDuration( + runId: string, + snapshotId: string, + body: WorkerApiWaitForDurationRequestBody + ) { + return wrapZodFetch( + WorkerApiWaitForDurationResponseBody, + `${this.apiUrl}/api/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/wait/duration`, + { + method: "POST", + headers: { + ...this.defaultHeaders, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + } + ); + } +} + +type ApiResult = + | { success: true; data: TSuccessResult } + | { + success: false; + error: string; + }; + +async function wrapZodFetch( + schema: T, + url: string, + requestInit?: RequestInit +): Promise>> { + try { + const response = await zodfetch(schema, url, requestInit, { + retry: { + minTimeoutInMs: 500, + maxTimeoutInMs: 5000, + maxAttempts: 5, + factor: 2, + randomize: false, + }, + }); + + return { + success: true, + data: response, + }; + } catch (error) { + if (error instanceof ApiError) { + return { + success: false, + error: error.message, + }; + } else if (error instanceof Error) { + return { + success: false, + error: error.message, + }; + } else { + return { + success: false, + error: String(error), + }; + } + } +} diff --git a/packages/worker/src/supervisor/queueConsumer.ts b/packages/worker/src/supervisor/queueConsumer.ts new file mode 100644 index 0000000000..c9cdc8cc58 --- /dev/null +++ b/packages/worker/src/supervisor/queueConsumer.ts @@ -0,0 +1,71 @@ +import { SupervisorHttpClient } from "./http.js"; +import { WorkerApiDequeueResponseBody } from "./schemas.js"; + +type RunQueueConsumerOptions = { + client: SupervisorHttpClient; + intervalMs?: number; + onDequeue: (messages: WorkerApiDequeueResponseBody) => Promise; +}; + +export class RunQueueConsumer { + private readonly client: SupervisorHttpClient; + private readonly onDequeue: (messages: WorkerApiDequeueResponseBody) => Promise; + + private intervalMs: number; + private isEnabled: boolean; + + constructor(opts: RunQueueConsumerOptions) { + this.isEnabled = false; + this.intervalMs = opts.intervalMs ?? 5_000; + this.onDequeue = opts.onDequeue; + this.client = opts.client; + } + + start() { + if (this.isEnabled) { + return; + } + + this.isEnabled = true; + this.dequeue(); + } + + stop() { + if (!this.isEnabled) { + return; + } + + this.isEnabled = false; + } + + private async dequeue() { + // Incredibly verbose logging for debugging purposes + // console.debug("[RunQueueConsumer] dequeue()", { enabled: this.isEnabled }); + + if (!this.isEnabled) { + return; + } + + try { + const response = await this.client.dequeue(); + + if (!response.success) { + console.error("[RunQueueConsumer] Failed to dequeue", { error: response.error }); + } else { + try { + await this.onDequeue(response.data); + } catch (handlerError) { + console.error("[RunQueueConsumer] onDequeue error", { error: handlerError }); + } + } + } catch (clientError) { + console.error("[RunQueueConsumer] client.dequeue error", { error: clientError }); + } + + this.scheduleNextDequeue(); + } + + scheduleNextDequeue(delay: number = this.intervalMs) { + setTimeout(this.dequeue.bind(this), delay); + } +} diff --git a/packages/worker/src/supervisor/schemas.ts b/packages/worker/src/supervisor/schemas.ts new file mode 100644 index 0000000000..6fc646576e --- /dev/null +++ b/packages/worker/src/supervisor/schemas.ts @@ -0,0 +1,109 @@ +import { z } from "zod"; +import { + CompleteRunAttemptResult, + DequeuedMessage, + RunExecutionData, + StartRunAttemptResult, + TaskRunExecutionResult, + WaitForDurationResult, +} from "@trigger.dev/core/v3"; + +export const WorkerApiHeartbeatRequestBody = z.object({ + cpu: z.object({ + used: z.number(), + available: z.number(), + }), + memory: z.object({ + used: z.number(), + available: z.number(), + }), + tasks: z.array(z.string()), +}); +export type WorkerApiHeartbeatRequestBody = z.infer; + +export const WorkerApiHeartbeatResponseBody = z.object({ + ok: z.literal(true), +}); +export type WorkerApiHeartbeatResponseBody = z.infer; + +export const WorkerApiConnectRequestBody = z.object({ + metadata: z.record(z.any()), +}); +export type WorkerApiConnectRequestBody = z.infer; + +export const WorkerApiConnectResponseBody = z.object({ + ok: z.literal(true), + workerGroup: z.object({ + type: z.string(), + name: z.string(), + }), +}); +export type WorkerApiConnectResponseBody = z.infer; + +export const WorkerApiDequeueResponseBody = DequeuedMessage.array(); +export type WorkerApiDequeueResponseBody = z.infer; + +export const WorkerApiRunHeartbeatRequestBody = z.object({ + cpu: z.number(), + memory: z.number(), +}); +export type WorkerApiRunHeartbeatRequestBody = z.infer; + +export const WorkerApiRunHeartbeatResponseBody = z.object({ + ok: z.literal(true), +}); +export type WorkerApiRunHeartbeatResponseBody = z.infer; + +export const WorkerApiRunAttemptStartRequestBody = z.object({ + isWarmStart: z.boolean().optional(), +}); +export type WorkerApiRunAttemptStartRequestBody = z.infer< + typeof WorkerApiRunAttemptStartRequestBody +>; + +export const WorkerApiRunAttemptStartResponseBody = StartRunAttemptResult.and( + z.object({ + envVars: z.record(z.string()), + }) +); +export type WorkerApiRunAttemptStartResponseBody = z.infer< + typeof WorkerApiRunAttemptStartResponseBody +>; + +export const WorkerApiRunAttemptCompleteRequestBody = z.object({ + completion: TaskRunExecutionResult, +}); +export type WorkerApiRunAttemptCompleteRequestBody = z.infer< + typeof WorkerApiRunAttemptCompleteRequestBody +>; + +export const WorkerApiRunAttemptCompleteResponseBody = z.object({ + result: CompleteRunAttemptResult, +}); +export type WorkerApiRunAttemptCompleteResponseBody = z.infer< + typeof WorkerApiRunAttemptCompleteResponseBody +>; + +export const WorkerApiRunLatestSnapshotResponseBody = z.object({ + execution: RunExecutionData, +}); +export type WorkerApiRunLatestSnapshotResponseBody = z.infer< + typeof WorkerApiRunLatestSnapshotResponseBody +>; + +export const WorkerApiDequeueFromVersionResponseBody = DequeuedMessage.array(); +export type WorkerApiDequeueFromVersionResponseBody = z.infer< + typeof WorkerApiDequeueFromVersionResponseBody +>; + +export const WorkerApiWaitForDurationRequestBody = z.object({ + date: z.coerce.date(), +}); +export type WorkerApiWaitForDurationRequestBody = z.infer< + typeof WorkerApiWaitForDurationRequestBody +>; + +export const WorkerApiWaitForDurationResponseBody = WaitForDurationResult; +export type WorkerApiWaitForDurationResponseBody = z.infer< + typeof WorkerApiWaitForDurationResponseBody +>; diff --git a/packages/worker/src/supervisor/session.ts b/packages/worker/src/supervisor/session.ts new file mode 100644 index 0000000000..c21ebff139 --- /dev/null +++ b/packages/worker/src/supervisor/session.ts @@ -0,0 +1,159 @@ +import { HeartbeatService } from "@trigger.dev/core/v3"; +import { SupervisorHttpClient } from "./http.js"; +import { SupervisorClientCommonOptions } from "./types.js"; +import { WorkerApiDequeueResponseBody, WorkerApiHeartbeatRequestBody } from "./schemas.js"; +import { RunQueueConsumer } from "./queueConsumer.js"; +import { WorkerEvents } from "./events.js"; +import EventEmitter from "events"; +import { VERSION } from "../version.js"; +import { io, Socket } from "socket.io-client"; +import { WorkerClientToServerEvents, WorkerServerToClientEvents } from "../types.js"; +import { getDefaultWorkerHeaders } from "./util.js"; + +type SupervisorSessionOptions = SupervisorClientCommonOptions & { + heartbeatIntervalSeconds?: number; + dequeueIntervalMs?: number; +}; + +export class SupervisorSession extends EventEmitter { + public readonly httpClient: SupervisorHttpClient; + + private socket?: Socket; + + private readonly queueConsumer: RunQueueConsumer; + private readonly heartbeatService: HeartbeatService; + private readonly heartbeatIntervalSeconds: number; + + constructor(private opts: SupervisorSessionOptions) { + super(); + + this.httpClient = new SupervisorHttpClient(opts); + this.queueConsumer = new RunQueueConsumer({ + client: this.httpClient, + onDequeue: this.onDequeue.bind(this), + intervalMs: opts.dequeueIntervalMs, + }); + + // TODO: This should be dynamic and set by (or at least overridden by) the platform + this.heartbeatIntervalSeconds = opts.heartbeatIntervalSeconds || 30; + this.heartbeatService = new HeartbeatService({ + heartbeat: async () => { + console.debug("[WorkerSession] Sending heartbeat"); + + const body = this.getHeartbeatBody(); + const response = await this.httpClient.heartbeatWorker(body); + + if (!response.success) { + console.error("[WorkerSession] Heartbeat failed", { error: response.error }); + } + }, + intervalMs: this.heartbeatIntervalSeconds * 1000, + leadingEdge: false, + onError: async (error) => { + console.error("[WorkerSession] Failed to send heartbeat", { error }); + }, + }); + } + + private async onDequeue(messages: WorkerApiDequeueResponseBody): Promise { + // Incredibly verbose logging for debugging purposes + // console.log("[WorkerSession] Dequeued messages", { count: messages.length }); + // console.debug("[WorkerSession] Dequeued messages with contents", messages); + + for (const message of messages) { + console.log("[WorkerSession] Emitting message", { message }); + this.emit("runQueueMessage", { + time: new Date(), + message, + }); + } + } + + subscribeToRunNotifications(runFriendlyIds: string[]) { + console.log("[WorkerSession] Subscribing to run notifications", { runFriendlyIds }); + + if (!this.socket) { + console.error("[WorkerSession] Socket not connected"); + return; + } + + this.socket.emit("run:subscribe", { version: "1", runFriendlyIds }); + } + + unsubscribeFromRunNotifications(runFriendlyIds: string[]) { + console.log("[WorkerSession] Unsubscribing from run notifications", { runFriendlyIds }); + + if (!this.socket) { + console.error("[WorkerSession] Socket not connected"); + return; + } + + this.socket.emit("run:unsubscribe", { version: "1", runFriendlyIds }); + } + + private createSocket() { + const wsUrl = new URL(this.opts.apiUrl); + wsUrl.pathname = "/worker"; + + this.socket = io(wsUrl.href, { + transports: ["websocket"], + extraHeaders: getDefaultWorkerHeaders(this.opts), + }); + this.socket.on("run:notify", ({ version, run }) => { + console.log("[WorkerSession][WS] Received run notification", { version, run }); + this.emit("runNotification", { time: new Date(), run }); + }); + this.socket.on("connect", () => { + console.log("[WorkerSession][WS] Connected to platform"); + }); + this.socket.on("connect_error", (error) => { + console.error("[WorkerSession][WS] Connection error", { error }); + }); + this.socket.on("disconnect", (reason, description) => { + console.log("[WorkerSession][WS] Disconnected from platform", { reason, description }); + }); + } + + async start() { + const connect = await this.httpClient.connect({ + metadata: { + workerVersion: VERSION, + }, + }); + + if (!connect.success) { + console.error("[WorkerSession][HTTP] Failed to connect", { error: connect.error }); + throw new Error("[WorkerSession][HTTP] Failed to connect"); + } + + const { workerGroup } = connect.data; + + console.log("[WorkerSession][HTTP] Connected to platform", { + type: workerGroup.type, + name: workerGroup.name, + }); + + this.queueConsumer.start(); + this.heartbeatService.start(); + this.createSocket(); + } + + async stop() { + this.heartbeatService.stop(); + this.socket?.disconnect(); + } + + private getHeartbeatBody(): WorkerApiHeartbeatRequestBody { + return { + cpu: { + used: 0.5, + available: 0.5, + }, + memory: { + used: 0.5, + available: 0.5, + }, + tasks: [], + }; + } +} diff --git a/packages/worker/src/supervisor/types.ts b/packages/worker/src/supervisor/types.ts new file mode 100644 index 0000000000..dfc3d21ed0 --- /dev/null +++ b/packages/worker/src/supervisor/types.ts @@ -0,0 +1,7 @@ +export type SupervisorClientCommonOptions = { + apiUrl: string; + workerToken: string; + instanceName: string; + deploymentId?: string; + managedWorkerSecret?: string; +}; diff --git a/packages/worker/src/supervisor/util.ts b/packages/worker/src/supervisor/util.ts new file mode 100644 index 0000000000..2ed2b41c8c --- /dev/null +++ b/packages/worker/src/supervisor/util.ts @@ -0,0 +1,40 @@ +import { HEADER_NAME } from "../consts.js"; +import { createHeaders } from "../util.js"; +import { SupervisorClientCommonOptions } from "./types.js"; + +export function getDefaultWorkerHeaders( + options: SupervisorClientCommonOptions +): Record { + return createHeaders({ + Authorization: `Bearer ${options.workerToken}`, + [HEADER_NAME.WORKER_INSTANCE_NAME]: options.instanceName, + [HEADER_NAME.WORKER_DEPLOYMENT_ID]: options.deploymentId, + [HEADER_NAME.WORKER_MANAGED_SECRET]: options.managedWorkerSecret, + }); +} + +function redactString(value: string, end = 10) { + return value.slice(0, end) + "*".repeat(value.length - end); +} + +function redactNumber(value: number, end = 10) { + const str = String(value); + const redacted = redactString(str, end); + return Number(redacted); +} + +export function redactKeys>(obj: T, keys: Array): T { + const redacted = { ...obj }; + for (const key of keys) { + const value = obj[key]; + + if (typeof value === "number") { + redacted[key] = redactNumber(value) as any; + } else if (typeof value === "string") { + redacted[key] = redactString(value) as any; + } else { + continue; + } + } + return redacted; +} diff --git a/packages/worker/src/types.ts b/packages/worker/src/types.ts new file mode 100644 index 0000000000..928b03c86d --- /dev/null +++ b/packages/worker/src/types.ts @@ -0,0 +1,26 @@ +export interface WorkerServerToClientEvents { + "run:notify": (message: { version: "1"; run: { friendlyId: string } }) => void; +} + +export interface WorkerClientToServerEvents { + "run:subscribe": (message: { version: "1"; runFriendlyIds: string[] }) => void; + "run:unsubscribe": (message: { version: "1"; runFriendlyIds: string[] }) => void; +} + +export interface WorkloadServerToClientEvents { + "run:notify": (message: { version: "1"; run: { friendlyId: string } }) => void; +} + +export interface WorkloadClientToServerEvents { + "run:start": (message: { + version: "1"; + run: { friendlyId: string }; + snapshot: { friendlyId: string }; + }) => void; +} + +export type WorkloadClientSocketData = { + deploymentId: string; + runFriendlyId?: string; + snapshotId?: string; +}; diff --git a/packages/worker/src/util.ts b/packages/worker/src/util.ts new file mode 100644 index 0000000000..c957231bdf --- /dev/null +++ b/packages/worker/src/util.ts @@ -0,0 +1,13 @@ +/** Will ignore headers with falsey values */ +export function createHeaders(headersInit: Record) { + const headers = new Headers(); + + for (const [key, value] of Object.entries(headersInit)) { + if (!value) { + continue; + } + headers.set(key, value); + } + + return Object.fromEntries(headers.entries()); +} diff --git a/packages/worker/src/version.ts b/packages/worker/src/version.ts new file mode 100644 index 0000000000..2e47a88682 --- /dev/null +++ b/packages/worker/src/version.ts @@ -0,0 +1 @@ +export const VERSION = "0.0.0"; diff --git a/packages/worker/src/workload/http.ts b/packages/worker/src/workload/http.ts new file mode 100644 index 0000000000..f64248ba58 --- /dev/null +++ b/packages/worker/src/workload/http.ts @@ -0,0 +1,181 @@ +import { z } from "zod"; +import { zodfetch, ApiError } from "@trigger.dev/core/v3/zodfetch"; +import { + WorkloadHeartbeatRequestBody, + WorkloadHeartbeatResponseBody, + WorkloadRunAttemptCompleteRequestBody, + WorkloadRunAttemptCompleteResponseBody, + WorkloadRunAttemptStartResponseBody, + WorkloadRunLatestSnapshotResponseBody, + WorkloadDequeueFromVersionResponseBody, + WorkloadRunAttemptStartRequestBody, + WorkloadWaitForDurationRequestBody, + WorkloadWaitForDurationResponseBody, +} from "./schemas.js"; +import { WorkloadClientCommonOptions } from "./types.js"; +import { getDefaultWorkloadHeaders } from "./util.js"; + +type WorkloadHttpClientOptions = WorkloadClientCommonOptions; + +export class WorkloadHttpClient { + private readonly apiUrl: string; + private readonly deploymentId: string; + private readonly defaultHeaders: Record; + + constructor(opts: WorkloadHttpClientOptions) { + this.apiUrl = opts.workerApiUrl.replace(/\/$/, ""); + this.defaultHeaders = getDefaultWorkloadHeaders(opts); + this.deploymentId = opts.deploymentId; + + if (!this.apiUrl) { + throw new Error("apiURL is required and needs to be a non-empty string"); + } + + if (!this.deploymentId) { + throw new Error("deploymentId is required and needs to be a non-empty string"); + } + } + + async heartbeatRun(runId: string, snapshotId: string, body: WorkloadHeartbeatRequestBody) { + return wrapZodFetch( + WorkloadHeartbeatResponseBody, + `${this.apiUrl}/api/v1/workload-actions/runs/${runId}/snapshots/${snapshotId}/heartbeat`, + { + method: "POST", + headers: { + ...this.defaultHeaders, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + } + ); + } + + async startRunAttempt( + runId: string, + snapshotId: string, + body: WorkloadRunAttemptStartRequestBody + ) { + return wrapZodFetch( + WorkloadRunAttemptStartResponseBody, + `${this.apiUrl}/api/v1/workload-actions/runs/${runId}/snapshots/${snapshotId}/attempts/start`, + { + method: "POST", + headers: { + ...this.defaultHeaders, + }, + body: JSON.stringify(body), + } + ); + } + + async completeRunAttempt( + runId: string, + snapshotId: string, + body: WorkloadRunAttemptCompleteRequestBody + ) { + return wrapZodFetch( + WorkloadRunAttemptCompleteResponseBody, + `${this.apiUrl}/api/v1/workload-actions/runs/${runId}/snapshots/${snapshotId}/attempts/complete`, + { + method: "POST", + headers: { + ...this.defaultHeaders, + }, + body: JSON.stringify(body), + } + ); + } + + async getRunExecutionData(runId: string) { + return wrapZodFetch( + WorkloadRunLatestSnapshotResponseBody, + `${this.apiUrl}/api/v1/workload-actions/runs/${runId}/snapshots/latest`, + { + method: "GET", + headers: { + ...this.defaultHeaders, + }, + } + ); + } + + async waitForDuration( + runId: string, + snapshotId: string, + body: WorkloadWaitForDurationRequestBody + ) { + return wrapZodFetch( + WorkloadWaitForDurationResponseBody, + `${this.apiUrl}/api/v1/workload-actions/runs/${runId}/snapshots/${snapshotId}/wait/duration`, + { + method: "POST", + headers: { + ...this.defaultHeaders, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + } + ); + } + + async dequeue() { + return wrapZodFetch( + WorkloadDequeueFromVersionResponseBody, + `${this.apiUrl}/api/v1/workload-actions/deployments/${this.deploymentId}/dequeue`, + { + method: "GET", + headers: { + ...this.defaultHeaders, + }, + } + ); + } +} + +type ApiResult = + | { success: true; data: TSuccessResult } + | { + success: false; + error: string; + }; + +async function wrapZodFetch( + schema: T, + url: string, + requestInit?: RequestInit +): Promise>> { + try { + const response = await zodfetch(schema, url, requestInit, { + retry: { + minTimeoutInMs: 500, + maxTimeoutInMs: 5000, + maxAttempts: 5, + factor: 2, + randomize: false, + }, + }); + + return { + success: true, + data: response, + }; + } catch (error) { + if (error instanceof ApiError) { + return { + success: false, + error: error.message, + }; + } else if (error instanceof Error) { + return { + success: false, + error: error.message, + }; + } else { + return { + success: false, + error: String(error), + }; + } + } +} diff --git a/packages/worker/src/workload/schemas.ts b/packages/worker/src/workload/schemas.ts new file mode 100644 index 0000000000..83971a1707 --- /dev/null +++ b/packages/worker/src/workload/schemas.ts @@ -0,0 +1,55 @@ +import { z } from "zod"; +import { + WorkerApiRunHeartbeatRequestBody, + WorkerApiHeartbeatResponseBody, + WorkerApiRunAttemptCompleteRequestBody, + WorkerApiRunAttemptCompleteResponseBody, + WorkerApiRunAttemptStartRequestBody, + WorkerApiRunAttemptStartResponseBody, + WorkerApiRunLatestSnapshotResponseBody, + WorkerApiDequeueFromVersionResponseBody, + WorkerApiWaitForDurationRequestBody, + WorkerApiWaitForDurationResponseBody, +} from "../supervisor/schemas.js"; + +export const WorkloadHeartbeatRequestBody = WorkerApiRunHeartbeatRequestBody; +export type WorkloadHeartbeatRequestBody = z.infer; + +export const WorkloadHeartbeatResponseBody = WorkerApiHeartbeatResponseBody; +export type WorkloadHeartbeatResponseBody = z.infer; + +export const WorkloadRunAttemptCompleteRequestBody = WorkerApiRunAttemptCompleteRequestBody; +export type WorkloadRunAttemptCompleteRequestBody = z.infer< + typeof WorkloadRunAttemptCompleteRequestBody +>; + +export const WorkloadRunAttemptCompleteResponseBody = WorkerApiRunAttemptCompleteResponseBody; +export type WorkloadRunAttemptCompleteResponseBody = z.infer< + typeof WorkloadRunAttemptCompleteResponseBody +>; + +export const WorkloadRunAttemptStartRequestBody = WorkerApiRunAttemptStartRequestBody; +export type WorkloadRunAttemptStartRequestBody = z.infer; + +export const WorkloadRunAttemptStartResponseBody = WorkerApiRunAttemptStartResponseBody; +export type WorkloadRunAttemptStartResponseBody = z.infer< + typeof WorkloadRunAttemptStartResponseBody +>; + +export const WorkloadRunLatestSnapshotResponseBody = WorkerApiRunLatestSnapshotResponseBody; +export type WorkloadRunLatestSnapshotResponseBody = z.infer< + typeof WorkloadRunLatestSnapshotResponseBody +>; + +export const WorkloadDequeueFromVersionResponseBody = WorkerApiDequeueFromVersionResponseBody; +export type WorkloadDequeueFromVersionResponseBody = z.infer< + typeof WorkloadDequeueFromVersionResponseBody +>; + +export const WorkloadWaitForDurationRequestBody = WorkerApiWaitForDurationRequestBody; +export type WorkloadWaitForDurationRequestBody = z.infer; + +export const WorkloadWaitForDurationResponseBody = WorkerApiWaitForDurationResponseBody; +export type WorkloadWaitForDurationResponseBody = z.infer< + typeof WorkloadWaitForDurationResponseBody +>; diff --git a/packages/worker/src/workload/types.ts b/packages/worker/src/workload/types.ts new file mode 100644 index 0000000000..6930550e0b --- /dev/null +++ b/packages/worker/src/workload/types.ts @@ -0,0 +1,4 @@ +export type WorkloadClientCommonOptions = { + workerApiUrl: string; + deploymentId: string; +}; diff --git a/packages/worker/src/workload/util.ts b/packages/worker/src/workload/util.ts new file mode 100644 index 0000000000..3457a0fbcb --- /dev/null +++ b/packages/worker/src/workload/util.ts @@ -0,0 +1,11 @@ +import { WORKLOAD_HEADER_NAME } from "../consts.js"; +import { createHeaders } from "../util.js"; +import { WorkloadClientCommonOptions } from "./types.js"; + +export function getDefaultWorkloadHeaders( + options: WorkloadClientCommonOptions +): Record { + return createHeaders({ + [WORKLOAD_HEADER_NAME.WORKLOAD_DEPLOYMENT_ID]: options.deploymentId, + }); +} diff --git a/packages/worker/tsconfig.json b/packages/worker/tsconfig.json new file mode 100644 index 0000000000..16881b51b6 --- /dev/null +++ b/packages/worker/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../.configs/tsconfig.base.json", + "references": [ + { + "path": "./tsconfig.src.json" + } + ] +} diff --git a/packages/worker/tsconfig.src.json b/packages/worker/tsconfig.src.json new file mode 100644 index 0000000000..db06c53317 --- /dev/null +++ b/packages/worker/tsconfig.src.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "include": ["./src/**/*.ts"], + "compilerOptions": { + "isolatedDeclarations": false, + "composite": true, + "sourceMap": true, + "customConditions": ["@triggerdotdev/source"] + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f208eb3d2a..a3d20b7e3c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,9 @@ patchedDependencies: graphile-worker@0.16.6: hash: hdpetta7btqcc7xb5wfkcnanoa path: patches/graphile-worker@0.16.6.patch + redlock@5.0.0-beta.2: + hash: rwyegdki7iserrd7fgjwxkhnlu + path: patches/redlock@5.0.0-beta.2.patch importers: @@ -60,7 +63,7 @@ importers: specifier: ^1.10.3 version: 1.10.3 typescript: - specifier: ^5.5.4 + specifier: 5.5.4 version: 5.5.4 vite: specifier: ^4.1.1 @@ -90,9 +93,6 @@ importers: specifier: ^0.3.0 version: 0.3.0 devDependencies: - '@types/node': - specifier: ^18 - version: 18.17.1 dotenv: specifier: ^16.4.2 version: 16.4.4 @@ -102,9 +102,6 @@ importers: tsx: specifier: ^4.7.0 version: 4.7.1 - typescript: - specifier: ^5.3.3 - version: 5.3.3 apps/docker-provider: dependencies: @@ -115,9 +112,6 @@ importers: specifier: ^8.0.1 version: 8.0.1 devDependencies: - '@types/node': - specifier: ^18.19.8 - version: 18.19.20 dotenv: specifier: ^16.4.2 version: 16.4.4 @@ -127,9 +121,6 @@ importers: tsx: specifier: ^4.7.0 version: 4.7.1 - typescript: - specifier: ^5.3.3 - version: 5.3.3 apps/kubernetes-provider: dependencies: @@ -152,9 +143,6 @@ importers: tsx: specifier: ^4.7.0 version: 4.7.1 - typescript: - specifier: ^5.3.3 - version: 5.3.3 apps/proxy: dependencies: @@ -177,9 +165,6 @@ importers: '@cloudflare/workers-types': specifier: ^4.20240512.0 version: 4.20240512.0 - typescript: - specifier: ^5.0.4 - version: 5.2.2 wrangler: specifier: ^3.57.1 version: 3.57.1(@cloudflare/workers-types@4.20240512.0) @@ -240,6 +225,9 @@ importers: '@heroicons/react': specifier: ^2.0.12 version: 2.0.13(react@18.2.0) + '@internal/run-engine': + specifier: workspace:* + version: link:../../internal-packages/run-engine '@internal/zod-worker': specifier: workspace:* version: link:../../internal-packages/zod-worker @@ -335,22 +323,22 @@ importers: version: 3.7.1(react@18.2.0) '@remix-run/express': specifier: 2.1.0 - version: 2.1.0(express@4.18.2)(typescript@5.2.2) + version: 2.1.0(express@4.18.2)(typescript@5.5.4) '@remix-run/node': specifier: 2.1.0 - version: 2.1.0(typescript@5.2.2) + version: 2.1.0(typescript@5.5.4) '@remix-run/react': specifier: 2.1.0 - version: 2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.2.2) + version: 2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.5.4) '@remix-run/router': specifier: ^1.15.3 version: 1.15.3 '@remix-run/serve': specifier: 2.1.0 - version: 2.1.0(typescript@5.2.2) + version: 2.1.0(typescript@5.5.4) '@remix-run/server-runtime': specifier: 2.1.0 - version: 2.1.0(typescript@5.2.2) + version: 2.1.0(typescript@5.5.4) '@remix-run/v1-meta': specifier: ^0.1.3 version: 0.1.3(@remix-run/react@2.1.0)(@remix-run/server-runtime@2.1.0) @@ -393,6 +381,9 @@ importers: '@trigger.dev/sdk': specifier: workspace:* version: link:../../packages/trigger-sdk + '@trigger.dev/worker': + specifier: workspace:* + version: link:../../packages/worker '@trigger.dev/yalt': specifier: npm:@trigger.dev/yalt version: 2.3.19 @@ -422,7 +413,7 @@ importers: version: 1.0.18 class-variance-authority: specifier: ^0.5.2 - version: 0.5.2(typescript@5.2.2) + version: 0.5.2(typescript@5.5.4) clsx: specifier: ^1.2.1 version: 1.2.1 @@ -461,7 +452,7 @@ importers: version: 10.12.11(react-dom@18.2.0)(react@18.2.0) graphile-worker: specifier: 0.16.6 - version: 0.16.6(patch_hash=hdpetta7btqcc7xb5wfkcnanoa)(typescript@5.2.2) + version: 0.16.6(patch_hash=hdpetta7btqcc7xb5wfkcnanoa)(typescript@5.5.4) highlight.run: specifier: ^7.3.4 version: 7.3.4 @@ -654,13 +645,13 @@ importers: version: link:../../internal-packages/testcontainers '@remix-run/dev': specifier: 2.1.0 - version: 2.1.0(@remix-run/serve@2.1.0)(@types/node@18.11.18)(ts-node@10.9.1)(typescript@5.2.2) + version: 2.1.0(@remix-run/serve@2.1.0)(@types/node@20.14.14)(ts-node@10.9.1)(typescript@5.5.4) '@remix-run/eslint-config': specifier: 2.1.0 - version: 2.1.0(eslint@8.31.0)(react@18.2.0)(typescript@5.2.2) + version: 2.1.0(eslint@8.31.0)(react@18.2.0)(typescript@5.5.4) '@remix-run/testing': specifier: ^2.1.0 - version: 2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.2.2) + version: 2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.5.4) '@swc/core': specifier: ^1.3.4 version: 1.3.26 @@ -706,9 +697,6 @@ importers: '@types/morgan': specifier: ^1.9.3 version: 1.9.4 - '@types/node': - specifier: ^18.11.15 - version: 18.11.18 '@types/node-fetch': specifier: ^2.6.2 version: 2.6.2 @@ -753,10 +741,10 @@ importers: version: 8.5.4 '@typescript-eslint/eslint-plugin': specifier: ^5.59.6 - version: 5.59.6(@typescript-eslint/parser@5.59.6)(eslint@8.31.0)(typescript@5.2.2) + version: 5.59.6(@typescript-eslint/parser@5.59.6)(eslint@8.31.0)(typescript@5.5.4) '@typescript-eslint/parser': specifier: ^5.59.6 - version: 5.59.6(eslint@8.31.0)(typescript@5.2.2) + version: 5.59.6(eslint@8.31.0)(typescript@5.5.4) autoprefixer: specifier: ^10.4.13 version: 10.4.13(postcss@8.4.44) @@ -801,7 +789,7 @@ importers: version: 16.0.1(postcss@8.4.44) postcss-loader: specifier: ^8.1.1 - version: 8.1.1(postcss@8.4.44)(typescript@5.2.2)(webpack@5.88.2) + version: 8.1.1(postcss@8.4.44)(typescript@5.5.4)(webpack@5.88.2) prettier: specifier: ^2.8.8 version: 2.8.8 @@ -828,19 +816,16 @@ importers: version: 3.4.1(ts-node@10.9.1) ts-node: specifier: ^10.7.0 - version: 10.9.1(@swc/core@1.3.26)(@types/node@18.11.18)(typescript@5.2.2) + version: 10.9.1(@swc/core@1.3.26)(@types/node@20.14.14)(typescript@5.5.4) tsconfig-paths: specifier: ^3.14.1 version: 3.14.1 - typescript: - specifier: ^5.1.6 - version: 5.2.2 vite-tsconfig-paths: specifier: ^4.0.5 - version: 4.0.5(typescript@5.2.2) + version: 4.0.5(typescript@5.5.4) vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.18) + version: 1.4.0(@types/node@20.14.14) docs: {} @@ -849,9 +834,6 @@ importers: '@prisma/client': specifier: 5.4.1 version: 5.4.1(prisma@5.4.1) - typescript: - specifier: ^4.8.4 - version: 4.9.5 devDependencies: prisma: specifier: 5.4.1 @@ -887,18 +869,12 @@ importers: specifier: 3.23.8 version: 3.23.8 devDependencies: - '@types/node': - specifier: ^18 - version: 18.19.20 '@types/nodemailer': specifier: ^6.4.17 version: 6.4.17 '@types/react': specifier: 18.2.69 version: 18.2.69 - typescript: - specifier: ^4.9.4 - version: 4.9.5 internal-packages/otlp-importer: dependencies: @@ -918,9 +894,6 @@ importers: ts-proto: specifier: ^1.167.3 version: 1.167.3 - typescript: - specifier: ^5.5.0 - version: 5.5.4 internal-packages/redis-worker: dependencies: @@ -939,9 +912,6 @@ importers: nanoid: specifier: ^5.0.7 version: 5.0.7 - typescript: - specifier: ^5.5.4 - version: 5.5.4 zod: specifier: 3.23.8 version: 3.23.8 @@ -956,6 +926,46 @@ importers: specifier: ^1.4.0 version: 1.6.0(@types/node@20.14.14) + internal-packages/run-engine: + dependencies: + '@internal/redis-worker': + specifier: workspace:* + version: link:../redis-worker + '@opentelemetry/api': + specifier: ^1.9.0 + version: 1.9.0 + '@opentelemetry/semantic-conventions': + specifier: ^1.27.0 + version: 1.27.0 + '@trigger.dev/core': + specifier: workspace:* + version: link:../../packages/core + '@trigger.dev/database': + specifier: workspace:* + version: link:../database + assert-never: + specifier: ^1.2.1 + version: 1.2.1 + ioredis: + specifier: ^5.3.2 + version: 5.3.2 + nanoid: + specifier: ^3.3.4 + version: 3.3.7 + redlock: + specifier: 5.0.0-beta.2 + version: 5.0.0-beta.2(patch_hash=rwyegdki7iserrd7fgjwxkhnlu) + zod: + specifier: 3.23.8 + version: 3.23.8 + devDependencies: + '@internal/testcontainers': + specifier: workspace:* + version: link:../testcontainers + vitest: + specifier: ^1.4.0 + version: 1.6.0(@types/node@20.14.14) + internal-packages/testcontainers: dependencies: '@opentelemetry/api': @@ -967,9 +977,6 @@ importers: ioredis: specifier: ^5.3.2 version: 5.3.2 - typescript: - specifier: ^4.8.4 - version: 4.9.5 devDependencies: '@testcontainers/postgresql': specifier: ^10.13.1 @@ -977,6 +984,9 @@ importers: '@testcontainers/redis': specifier: ^10.13.1 version: 10.13.1 + '@trigger.dev/core': + specifier: workspace:* + version: link:../../packages/core testcontainers: specifier: ^10.13.1 version: 10.13.1 @@ -1004,9 +1014,6 @@ importers: lodash.omit: specifier: ^4.5.0 version: 4.5.0 - typescript: - specifier: ^5.5.4 - version: 5.5.4 zod: specifier: 3.23.8 version: 3.23.8 @@ -1039,9 +1046,6 @@ importers: '@arethetypeswrong/cli': specifier: ^0.15.4 version: 0.15.4 - '@types/node': - specifier: 20.14.14 - version: 20.14.14 esbuild: specifier: ^0.23.0 version: 0.23.0 @@ -1054,9 +1058,6 @@ importers: tsx: specifier: 4.17.0 version: 4.17.0 - typescript: - specifier: ^5.5.4 - version: 5.5.4 packages/cli-v3: dependencies: @@ -1108,6 +1109,9 @@ importers: '@trigger.dev/core': specifier: workspace:3.3.7 version: link:../core + '@trigger.dev/worker': + specifier: workspace:3.3.7 + version: link:../worker c12: specifier: ^1.11.1 version: 1.11.1(magicast@0.3.4) @@ -1183,6 +1187,9 @@ importers: signal-exit: specifier: ^4.1.0 version: 4.1.0 + socket.io-client: + specifier: 4.7.5 + version: 4.7.5 source-map-support: specifier: 0.5.21 version: 0.5.21 @@ -1217,9 +1224,6 @@ importers: '@types/gradient-string': specifier: ^1.1.2 version: 1.1.2 - '@types/node': - specifier: 20.14.14 - version: 20.14.14 '@types/object-hash': specifier: 3.0.6 version: 3.0.6 @@ -1262,15 +1266,15 @@ importers: tsx: specifier: 4.17.0 version: 4.17.0 - typescript: - specifier: ^5.5.4 - version: 5.5.4 vitest: specifier: ^2.0.5 version: 2.0.5(@types/node@20.14.14) packages/core: dependencies: + '@bugsnag/cuid': + specifier: ^3.1.1 + version: 3.1.1 '@electric-sql/client': specifier: 1.0.0-beta.1 version: 1.0.0-beta.1 @@ -1356,12 +1360,12 @@ importers: '@epic-web/test-server': specifier: ^0.1.0 version: 0.1.0 + '@trigger.dev/database': + specifier: workspace:* + version: link:../../internal-packages/database '@types/humanize-duration': specifier: ^3.27.1 version: 3.27.1 - '@types/node': - specifier: 20.14.14 - version: 20.14.14 '@types/readable-stream': specifier: ^4.0.14 version: 4.0.14 @@ -1389,9 +1393,6 @@ importers: tsx: specifier: 4.17.0 version: 4.17.0 - typescript: - specifier: ^5.5.4 - version: 5.5.4 vitest: specifier: ^1.6.0 version: 1.6.0(@types/node@20.14.14) @@ -1414,9 +1415,6 @@ importers: '@arethetypeswrong/cli': specifier: ^0.15.4 version: 0.15.4 - '@types/node': - specifier: ^20.14.14 - version: 20.14.14 '@types/react': specifier: '*' version: 18.3.1 @@ -1432,9 +1430,6 @@ importers: tsx: specifier: 4.17.0 version: 4.17.0 - typescript: - specifier: ^5.5.4 - version: 5.5.4 packages/rsc: dependencies: @@ -1457,9 +1452,6 @@ importers: '@trigger.dev/build': specifier: workspace:^3.3.7 version: link:../build - '@types/node': - specifier: ^20.14.14 - version: 20.14.14 '@types/react': specifier: '*' version: 18.3.1 @@ -1475,9 +1467,6 @@ importers: tsx: specifier: 4.17.0 version: 4.17.0 - typescript: - specifier: ^5.5.4 - version: 5.5.4 packages/trigger-sdk: dependencies: @@ -1527,9 +1516,6 @@ importers: '@types/debug': specifier: ^4.1.7 version: 4.1.7 - '@types/node': - specifier: 20.14.14 - version: 20.14.14 '@types/slug': specifier: ^5.0.3 version: 5.0.3 @@ -1557,13 +1543,38 @@ importers: typed-emitter: specifier: ^2.1.0 version: 2.1.0 - typescript: - specifier: ^5.5.4 - version: 5.5.4 zod: specifier: 3.23.8 version: 3.23.8 + packages/worker: + dependencies: + '@trigger.dev/core': + specifier: workspace:* + version: link:../core + socket.io: + specifier: 4.7.4 + version: 4.7.4 + socket.io-client: + specifier: 4.7.5 + version: 4.7.5 + zod: + specifier: 3.23.8 + version: 3.23.8 + devDependencies: + '@arethetypeswrong/cli': + specifier: ^0.15.4 + version: 0.15.4 + rimraf: + specifier: 6.0.1 + version: 6.0.1 + tshy: + specifier: ^3.0.2 + version: 3.0.2 + tsx: + specifier: 4.17.0 + version: 4.17.0 + references/bun-catalog: dependencies: '@trigger.dev/sdk': @@ -1576,9 +1587,6 @@ importers: trigger.dev: specifier: workspace:* version: link:../../packages/cli-v3 - typescript: - specifier: ^5.5.4 - version: 5.5.4 references/hello-world: dependencies: @@ -1589,18 +1597,12 @@ importers: trigger.dev: specifier: workspace:* version: link:../../packages/cli-v3 - typescript: - specifier: ^5.5.4 - version: 5.5.4 references/init-shell: devDependencies: trigger.dev: specifier: workspace:* version: link:../../packages/cli-v3 - typescript: - specifier: ^5.5.4 - version: 5.5.4 references/init-shell-js: devDependencies: @@ -1686,9 +1688,6 @@ importers: '@trigger.dev/rsc': specifier: workspace:^3 version: link:../../packages/rsc - '@types/node': - specifier: ^20 - version: 20.14.14 '@types/react': specifier: ^18 version: 18.3.1 @@ -1704,9 +1703,6 @@ importers: trigger.dev: specifier: workspace:^3 version: link:../../packages/cli-v3 - typescript: - specifier: ^5 - version: 5.5.4 references/v3-catalog: dependencies: @@ -1879,9 +1875,6 @@ importers: '@types/fluent-ffmpeg': specifier: ^2.1.26 version: 2.1.26 - '@types/node': - specifier: 20.4.2 - version: 20.4.2 '@types/react': specifier: ^18.3.1 version: 18.3.1 @@ -1903,9 +1896,6 @@ importers: tsconfig-paths: specifier: ^4.2.0 version: 4.2.0 - typescript: - specifier: ^5.5.4 - version: 5.5.4 packages: @@ -4944,6 +4934,10 @@ packages: resolution: {integrity: sha512-QDdVFLoN93Zjg36NoQPZfsVH9tZew7wKDKyV5qRdj8ntT4wQCOradQjRaTdwMhWUYsgKsvCINKKm87FdEk96Ag==} dev: false + /@bugsnag/cuid@3.1.1: + resolution: {integrity: sha512-d2z4b0rEo3chI07FNN1Xds8v25CNeekecU6FC/2Fs9MxY2EipkZTThVcV2YinMn8dvRUlViKOyC50evoUxg8tw==} + dev: false + /@bundled-es-modules/cookie@2.0.0: resolution: {integrity: sha512-Or6YHg/kamKHpxULAdSqhGqnWFneIXu1NKvvfBBzKGwpVsYuFIQ5aBPHDnnoR3ghW1nvSkALd+EF9iMtY7Vjxw==} dependencies: @@ -7248,7 +7242,7 @@ packages: engines: {node: ^8.13.0 || >=10.10.0} dependencies: '@grpc/proto-loader': 0.7.7 - '@types/node': 18.19.20 + '@types/node': 20.14.14 /@grpc/proto-loader@0.7.7: resolution: {integrity: sha512-1TIeXOi8TuSCQprPItwoMymZXxWT0CPxUhkrkeCUH+D8U7QDwQ6b7SUz2MaLuWM2llT+J/TVFLmQI5KtML3BhQ==} @@ -9064,7 +9058,7 @@ packages: engines: {node: '>=16'} hasBin: true dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 playwright-core: 1.37.0 optionalDependencies: fsevents: 2.3.2 @@ -10833,7 +10827,7 @@ packages: react: ^16.8 || ^17.0 || ^18.0 react-dom: ^16.8 || ^17.0 || ^18.0 dependencies: - '@babel/runtime': 7.20.7 + '@babel/runtime': 7.24.5 '@radix-ui/primitive': 1.0.0 '@radix-ui/react-context': 1.0.0(react@18.2.0) '@radix-ui/react-direction': 1.0.0(react@18.2.0) @@ -14392,7 +14386,7 @@ packages: - encoding dev: false - /@remix-run/dev@2.1.0(@remix-run/serve@2.1.0)(@types/node@18.11.18)(ts-node@10.9.1)(typescript@5.2.2): + /@remix-run/dev@2.1.0(@remix-run/serve@2.1.0)(@types/node@20.14.14)(ts-node@10.9.1)(typescript@5.5.4): resolution: {integrity: sha512-Hn5lw46F+a48dp5uHKe68ckaHgdStW4+PmLod+LMFEqrMbkF0j4XD1ousebxlv989o0Uy/OLgfRMgMy4cBOvHg==} engines: {node: '>=18.0.0'} hasBin: true @@ -14414,10 +14408,10 @@ packages: '@babel/traverse': 7.22.17 '@mdx-js/mdx': 2.3.0 '@npmcli/package-json': 4.0.1 - '@remix-run/serve': 2.1.0(typescript@5.2.2) - '@remix-run/server-runtime': 2.1.0(typescript@5.2.2) + '@remix-run/serve': 2.1.0(typescript@5.5.4) + '@remix-run/server-runtime': 2.1.0(typescript@5.5.4) '@types/mdx': 2.0.5 - '@vanilla-extract/integration': 6.2.1(@types/node@18.11.18) + '@vanilla-extract/integration': 6.2.1(@types/node@20.14.14) arg: 5.0.2 cacache: 17.1.4 chalk: 4.1.2 @@ -14453,7 +14447,7 @@ packages: semver: 7.6.3 tar-fs: 2.1.1 tsconfig-paths: 4.2.0 - typescript: 5.2.2 + typescript: 5.5.4 ws: 7.5.9 transitivePeerDependencies: - '@types/node' @@ -14471,7 +14465,7 @@ packages: - utf-8-validate dev: true - /@remix-run/eslint-config@2.1.0(eslint@8.31.0)(react@18.2.0)(typescript@5.2.2): + /@remix-run/eslint-config@2.1.0(eslint@8.31.0)(react@18.2.0)(typescript@5.5.4): resolution: {integrity: sha512-yfeUnHpUG+XveujMi6QODKMGhs5CvKWCKzASU397BPXiPWbMv6r2acfODSWK64ZdBMu9hcLbOb42GBFydVQeHA==} engines: {node: '>=18.0.0'} peerDependencies: @@ -14486,28 +14480,28 @@ packages: '@babel/eslint-parser': 7.21.8(@babel/core@7.22.17)(eslint@8.31.0) '@babel/preset-react': 7.18.6(@babel/core@7.22.17) '@rushstack/eslint-patch': 1.2.0 - '@typescript-eslint/eslint-plugin': 5.59.6(@typescript-eslint/parser@5.59.6)(eslint@8.31.0)(typescript@5.2.2) - '@typescript-eslint/parser': 5.59.6(eslint@8.31.0)(typescript@5.2.2) + '@typescript-eslint/eslint-plugin': 5.59.6(@typescript-eslint/parser@5.59.6)(eslint@8.31.0)(typescript@5.5.4) + '@typescript-eslint/parser': 5.59.6(eslint@8.31.0)(typescript@5.5.4) eslint: 8.31.0 eslint-import-resolver-node: 0.3.7 eslint-import-resolver-typescript: 3.5.5(@typescript-eslint/parser@5.59.6)(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.29.1)(eslint@8.31.0) eslint-plugin-import: 2.29.1(@typescript-eslint/parser@5.59.6)(eslint-import-resolver-typescript@3.5.5)(eslint@8.31.0) - eslint-plugin-jest: 26.9.0(@typescript-eslint/eslint-plugin@5.59.6)(eslint@8.31.0)(typescript@5.2.2) + eslint-plugin-jest: 26.9.0(@typescript-eslint/eslint-plugin@5.59.6)(eslint@8.31.0)(typescript@5.5.4) eslint-plugin-jest-dom: 4.0.3(eslint@8.31.0) eslint-plugin-jsx-a11y: 6.7.1(eslint@8.31.0) eslint-plugin-node: 11.1.0(eslint@8.31.0) eslint-plugin-react: 7.32.2(eslint@8.31.0) eslint-plugin-react-hooks: 4.6.2(eslint@8.31.0) - eslint-plugin-testing-library: 5.11.0(eslint@8.31.0)(typescript@5.2.2) + eslint-plugin-testing-library: 5.11.0(eslint@8.31.0)(typescript@5.5.4) react: 18.2.0 - typescript: 5.2.2 + typescript: 5.5.4 transitivePeerDependencies: - eslint-import-resolver-webpack - jest - supports-color dev: true - /@remix-run/express@2.1.0(express@4.18.2)(typescript@5.2.2): + /@remix-run/express@2.1.0(express@4.18.2)(typescript@5.5.4): resolution: {integrity: sha512-R5myPowQx6LYWY3+EqP42q19MOCT3+ZGwb2f0UKNs9a34R8U3nFpGWL7saXryC+To+EasujEScc8rTQw5Pftog==} engines: {node: '>=18.0.0'} peerDependencies: @@ -14517,11 +14511,11 @@ packages: typescript: optional: true dependencies: - '@remix-run/node': 2.1.0(typescript@5.2.2) + '@remix-run/node': 2.1.0(typescript@5.5.4) express: 4.18.2 - typescript: 5.2.2 + typescript: 5.5.4 - /@remix-run/node@2.1.0(typescript@5.2.2): + /@remix-run/node@2.1.0(typescript@5.5.4): resolution: {integrity: sha512-TeSgjXnZUUlmw5FVpBVnXY7MLpracjdnwFNwoJE5NQkiUEFnGD/Yhvk4F2fOCkszqc2Z25KRclc5noweyiFu6Q==} engines: {node: '>=18.0.0'} peerDependencies: @@ -14530,7 +14524,7 @@ packages: typescript: optional: true dependencies: - '@remix-run/server-runtime': 2.1.0(typescript@5.2.2) + '@remix-run/server-runtime': 2.1.0(typescript@5.5.4) '@remix-run/web-fetch': 4.4.1 '@remix-run/web-file': 3.1.0 '@remix-run/web-stream': 1.1.0 @@ -14538,9 +14532,9 @@ packages: cookie-signature: 1.2.0 source-map-support: 0.5.21 stream-slice: 0.1.2 - typescript: 5.2.2 + typescript: 5.5.4 - /@remix-run/react@2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.2.2): + /@remix-run/react@2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.5.4): resolution: {integrity: sha512-DeYgfsvNxHqNn29sGA3XsZCciMKo2EFTQ9hHkuVPTsJXC4ipHr6Dja1j6UzZYPe/ZuKppiuTjueWCQlE2jOe1w==} engines: {node: '>=18.0.0'} peerDependencies: @@ -14552,11 +14546,11 @@ packages: optional: true dependencies: '@remix-run/router': 1.10.0 - '@remix-run/server-runtime': 2.1.0(typescript@5.2.2) + '@remix-run/server-runtime': 2.1.0(typescript@5.5.4) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) react-router-dom: 6.17.0(react-dom@18.2.0)(react@18.2.0) - typescript: 5.2.2 + typescript: 5.5.4 /@remix-run/router@1.10.0: resolution: {integrity: sha512-Lm+fYpMfZoEucJ7cMxgt4dYt8jLfbpwRCzAjm9UgSLOkmlqo9gupxt6YX3DY0Fk155NT9l17d/ydi+964uS9Lw==} @@ -14567,13 +14561,13 @@ packages: engines: {node: '>=14.0.0'} dev: false - /@remix-run/serve@2.1.0(typescript@5.2.2): + /@remix-run/serve@2.1.0(typescript@5.5.4): resolution: {integrity: sha512-XHI+vPYz217qrg1QcV38TTPlEBTzMJzAt0SImPutyF0S2IBrZGZIFMEsspI0i0wNvdcdQz1IqmSx+mTghzW8eQ==} engines: {node: '>=18.0.0'} hasBin: true dependencies: - '@remix-run/express': 2.1.0(express@4.18.2)(typescript@5.2.2) - '@remix-run/node': 2.1.0(typescript@5.2.2) + '@remix-run/express': 2.1.0(express@4.18.2)(typescript@5.5.4) + '@remix-run/node': 2.1.0(typescript@5.5.4) chokidar: 3.5.3 compression: 1.7.4 express: 4.18.2 @@ -14584,7 +14578,7 @@ packages: - supports-color - typescript - /@remix-run/server-runtime@2.1.0(typescript@5.2.2): + /@remix-run/server-runtime@2.1.0(typescript@5.5.4): resolution: {integrity: sha512-Uz69yF4Gu6F3VYQub3JgDo9godN8eDMeZclkadBTAWN7bYLonu0ChR/GlFxS35OLeF7BDgudxOSZob0nE1WHNg==} engines: {node: '>=18.0.0'} peerDependencies: @@ -14599,9 +14593,9 @@ packages: cookie: 0.4.2 set-cookie-parser: 2.6.0 source-map: 0.7.4 - typescript: 5.2.2 + typescript: 5.5.4 - /@remix-run/testing@2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.2.2): + /@remix-run/testing@2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.5.4): resolution: {integrity: sha512-eLPx4Bmjt243kyRpQTong1eFo6nkvSfCr65bb5PfoF172DKnsSSCYWAmBmB72VwtAPESHxBm3g6AUbhwphkU6A==} engines: {node: '>=18.0.0'} peerDependencies: @@ -14611,12 +14605,12 @@ packages: typescript: optional: true dependencies: - '@remix-run/node': 2.1.0(typescript@5.2.2) - '@remix-run/react': 2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.2.2) + '@remix-run/node': 2.1.0(typescript@5.5.4) + '@remix-run/react': 2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.5.4) '@remix-run/router': 1.10.0 react: 18.2.0 react-router-dom: 6.17.0(react-dom@18.2.0)(react@18.2.0) - typescript: 5.2.2 + typescript: 5.5.4 transitivePeerDependencies: - react-dom dev: true @@ -14627,8 +14621,8 @@ packages: '@remix-run/react': ^1.15.0 || ^2.0.0 '@remix-run/server-runtime': ^1.15.0 || ^2.0.0 dependencies: - '@remix-run/react': 2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.2.2) - '@remix-run/server-runtime': 2.1.0(typescript@5.2.2) + '@remix-run/react': 2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.5.4) + '@remix-run/server-runtime': 2.1.0(typescript@5.5.4) dev: false /@remix-run/web-blob@3.1.0: @@ -14974,7 +14968,7 @@ packages: resolution: {integrity: sha512-DTuBFbqu4gGfajREEMrkq5jBhcnskinhr4+AnfJEk48zhVeEv3XnUKGIX98B74kxhYsIMfApGGySTn7V3b5yBA==} engines: {node: '>= 12.13.0', npm: '>= 6.12.0'} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 dev: false /@slack/types@2.8.0: @@ -14989,7 +14983,7 @@ packages: '@slack/logger': 3.0.0 '@slack/types': 2.8.0 '@types/is-stream': 1.1.0 - '@types/node': 18.19.20 + '@types/node': 20.14.14 axios: 0.27.2 eventemitter3: 3.1.2 form-data: 2.5.1 @@ -16265,7 +16259,7 @@ packages: resolution: {integrity: sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==} dependencies: '@types/connect': 3.4.35 - '@types/node': 18.19.20 + '@types/node': 20.14.14 dev: true /@types/bun@1.1.6: @@ -16301,7 +16295,7 @@ packages: /@types/connect@3.4.35: resolution: {integrity: sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 dev: true /@types/cookie@0.4.1: @@ -16317,12 +16311,12 @@ packages: /@types/cors@2.8.17: resolution: {integrity: sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 /@types/cross-spawn@6.0.2: resolution: {integrity: sha512-KuwNhp3eza+Rhu8IFI5HUXRP0LIhqH5cAjubUvGXXthh4YYBuP2ntwEX+Cz8GJoZUHlKo247wPWOfA9LYEq4cw==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 dev: true /@types/d3-array@3.0.8: @@ -16390,7 +16384,7 @@ packages: /@types/docker-modem@3.0.6: resolution: {integrity: sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/ssh2': 1.15.1 dev: true @@ -16398,7 +16392,7 @@ packages: resolution: {integrity: sha512-42R9eoVqJDSvVspV89g7RwRqfNExgievLNWoHkg7NoWIqAmavIbgQBb4oc0qRtHkxE+I3Xxvqv7qVXFABKPBTg==} dependencies: '@types/docker-modem': 3.0.6 - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/ssh2': 1.15.1 dev: true @@ -16433,7 +16427,7 @@ packages: /@types/express-serve-static-core@4.17.32: resolution: {integrity: sha512-aI5h/VOkxOF2Z1saPy0Zsxs5avets/iaiAJYznQFm5By/pamU31xWKL//epiF4OfUA2qTOc9PV6tCUjhO8wlZA==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/qs': 6.9.7 '@types/range-parser': 1.2.4 dev: true @@ -16450,7 +16444,7 @@ packages: /@types/fluent-ffmpeg@2.1.26: resolution: {integrity: sha512-0JVF3wdQG+pN0ImwWD0bNgJiKF2OHg/7CDBHw5UIbRTvlnkgGHK6V5doE54ltvhud4o31/dEiHm23CAlxFiUQg==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 dev: true /@types/gradient-string@1.1.2: @@ -16472,7 +16466,7 @@ packages: /@types/interpret@1.1.3: resolution: {integrity: sha512-uBaBhj/BhilG58r64mtDb/BEdH51HIQLgP5bmWzc5qCtFMja8dCk/IOJmk36j0lbi9QHwI6sbtUNGuqXdKCAtQ==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 dev: false /@types/invariant@2.2.37: @@ -16488,7 +16482,7 @@ packages: /@types/is-stream@1.1.0: resolution: {integrity: sha512-jkZatu4QVbR60mpIzjINmtS1ZF4a/FqdTUTBeQDVOQ2PYyidtwFKr0B5G6ERukKwliq+7mIXvxyppwzG5EgRYg==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 dev: false /@types/js-cookie@2.2.7: @@ -16516,7 +16510,7 @@ packages: /@types/keyv@3.1.4: resolution: {integrity: sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 dev: true /@types/lodash.omit@4.5.7: @@ -16561,7 +16555,7 @@ packages: /@types/morgan@1.9.4: resolution: {integrity: sha512-cXoc4k+6+YAllH3ZHmx4hf7La1dzUk6keTR4bF4b4Sc0mZxU/zK4wO7l+ZzezXm/jkYj/qC+uYGZrarZdIVvyQ==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 dev: true /@types/ms@0.7.31: @@ -16570,38 +16564,31 @@ packages: /@types/mute-stream@0.0.4: resolution: {integrity: sha512-CPM9nzrCPPJHQNA9keH9CVkVI+WR5kMa+7XEs5jcGQ0VoAGnLv242w8lIVgwAEfmE4oufJRaTc9PNLQl0ioAow==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 dev: false /@types/node-fetch@2.6.2: resolution: {integrity: sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 form-data: 3.0.1 dev: true /@types/node-fetch@2.6.4: resolution: {integrity: sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 form-data: 3.0.1 dev: false /@types/node-forge@1.3.10: resolution: {integrity: sha512-y6PJDYN4xYBxwd22l+OVH35N+1fCYWiuC3aiP2SlXVE6Lo7SS+rSx9r89hLxrP4pn6n1lBGhHJ12pj3F3Mpttw==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 /@types/node@12.20.55: resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} - /@types/node@18.11.18: - resolution: {integrity: sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==} - - /@types/node@18.17.1: - resolution: {integrity: sha512-xlR1jahfizdplZYRU59JlUx9uzF1ARa8jbhM11ccpCJya8kvos5jwdm2ZAgxSCwOl0fq21svP18EVwPBXMQudw==} - dev: true - /@types/node@18.19.20: resolution: {integrity: sha512-SKXZvI375jkpvAj8o+5U2518XQv76mAsixqfXiVyWyXZbVWQK25RurFovYpVIxVzul0rZoH58V/3SkEnm7s3qA==} dependencies: @@ -16647,7 +16634,7 @@ packages: /@types/pg@8.11.6: resolution: {integrity: sha512-/2WmmBXHLsfRqzfHW7BNZ8SbYzE8OSk7i3WjFYvfgRHj7S1xj+16Je5fUKv3lVdVzk/zn9TXOqf+avFCFIE0yQ==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 pg-protocol: 1.6.1 pg-types: 4.0.2 dev: false @@ -16655,7 +16642,7 @@ packages: /@types/pg@8.6.6: resolution: {integrity: sha512-O2xNmXebtwVekJDD+02udOncjVcMZQuTEQEMpKJ0ZRf5E7/9JJX3izhKUcUifBkyKpljyUM6BTgy2trmviKlpw==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 pg-protocol: 1.6.1 pg-types: 2.2.0 @@ -16708,7 +16695,7 @@ packages: /@types/readable-stream@4.0.14: resolution: {integrity: sha512-xZn/AuUbCMShGsqH/ehZtGDwQtbx00M9rZ2ENLe4tOjFZ/JFeWMhEZkk2fEe1jAUqqEAURIkFJ7Az/go8mM1/w==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 safe-buffer: 5.1.2 dev: true @@ -16720,7 +16707,7 @@ packages: resolution: {integrity: sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==} dependencies: '@types/caseless': 0.12.5 - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/tough-cookie': 4.0.5 form-data: 2.5.1 dev: false @@ -16732,7 +16719,7 @@ packages: /@types/responselike@1.0.0: resolution: {integrity: sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 dev: true /@types/retry@0.12.0: @@ -16767,7 +16754,7 @@ packages: resolution: {integrity: sha512-z5xyF6uh8CbjAu9760KDKsH2FcDxZ2tFCsA4HIMWE6IkiYMXfVoa+4f9KX+FN0ZLsaMw1WNG2ETLA6N+/YA+cg==} dependencies: '@types/mime': 3.0.1 - '@types/node': 18.19.20 + '@types/node': 20.14.14 dev: true /@types/shimmer@1.0.2: @@ -16790,13 +16777,13 @@ packages: /@types/ssh2-streams@0.1.12: resolution: {integrity: sha512-Sy8tpEmCce4Tq0oSOYdfqaBpA3hDM8SoxoFh5vzFsu2oL+znzGz8oVWW7xb4K920yYMUY+PIG31qZnFMfPWNCg==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 dev: true /@types/ssh2@0.5.52: resolution: {integrity: sha512-lbLLlXxdCZOSJMCInKH2+9V/77ET2J6NPQHpFI0kda61Dd1KglJs+fPQBchizmzYSOJBgdTajhPqBO1xxLywvg==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/ssh2-streams': 0.1.12 dev: true @@ -16815,7 +16802,7 @@ packages: dependencies: '@types/cookiejar': 2.1.5 '@types/methods': 1.1.4 - '@types/node': 18.19.20 + '@types/node': 20.14.14 form-data: 4.0.0 dev: true @@ -16829,7 +16816,7 @@ packages: /@types/tar@6.1.4: resolution: {integrity: sha512-Cp4oxpfIzWt7mr2pbhHT2OTXGMAL0szYCzuf8lRWyIMCgsx6/Hfc3ubztuhvzXHXgraTQxyOCmmg7TDGIMIJJQ==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 minipass: 4.0.0 dev: true @@ -16851,7 +16838,7 @@ packages: /@types/webpack@5.28.5(@swc/core@1.3.101)(esbuild@0.19.11): resolution: {integrity: sha512-wR87cgvxj3p6D0Crt1r5avwqffqPXUkNlnQ1mjU93G7gCuFjufZR4I6j8cz5g1F1tTYpfOOFvly+cmIQwL9wvw==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 tapable: 2.2.1 webpack: 5.88.2(@swc/core@1.3.101)(esbuild@0.19.11) transitivePeerDependencies: @@ -16868,25 +16855,25 @@ packages: /@types/ws@8.5.10: resolution: {integrity: sha512-vmQSUcfalpIq0R9q7uTo2lXs6eGIpt9wtnLdMv9LVpIjCA/+ufZRozlVoVelIYixx1ugCBKDhn89vnsEGOCx9A==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 /@types/ws@8.5.12: resolution: {integrity: sha512-3tPRkv1EtkDpzlgyKyI8pGsGZAGPEaXeu0DOj5DI25Ja91bdAYddYHbADRYVrZMRbfW+1l5YwXVDKohDJNQxkQ==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 dev: true /@types/ws@8.5.4: resolution: {integrity: sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 dev: true /@types/yauzl@2.10.3: resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==} requiresBuild: true dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 dev: false optional: true @@ -16913,7 +16900,7 @@ packages: - '@types/json-schema' dev: false - /@typescript-eslint/eslint-plugin@5.59.6(@typescript-eslint/parser@5.59.6)(eslint@8.31.0)(typescript@5.2.2): + /@typescript-eslint/eslint-plugin@5.59.6(@typescript-eslint/parser@5.59.6)(eslint@8.31.0)(typescript@5.5.4): resolution: {integrity: sha512-sXtOgJNEuRU5RLwPUb1jxtToZbgvq3M6FPpY4QENxoOggK+UpTxUBpj6tD8+Qh2g46Pi9We87E+eHnUw8YcGsw==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: @@ -16925,23 +16912,23 @@ packages: optional: true dependencies: '@eslint-community/regexpp': 4.5.1 - '@typescript-eslint/parser': 5.59.6(eslint@8.31.0)(typescript@5.2.2) + '@typescript-eslint/parser': 5.59.6(eslint@8.31.0)(typescript@5.5.4) '@typescript-eslint/scope-manager': 5.59.6 - '@typescript-eslint/type-utils': 5.59.6(eslint@8.31.0)(typescript@5.2.2) - '@typescript-eslint/utils': 5.59.6(eslint@8.31.0)(typescript@5.2.2) + '@typescript-eslint/type-utils': 5.59.6(eslint@8.31.0)(typescript@5.5.4) + '@typescript-eslint/utils': 5.59.6(eslint@8.31.0)(typescript@5.5.4) debug: 4.3.4 eslint: 8.31.0 grapheme-splitter: 1.0.4 ignore: 5.2.4 natural-compare-lite: 1.4.0 semver: 7.6.3 - tsutils: 3.21.0(typescript@5.2.2) - typescript: 5.2.2 + tsutils: 3.21.0(typescript@5.5.4) + typescript: 5.5.4 transitivePeerDependencies: - supports-color dev: true - /@typescript-eslint/parser@5.59.6(eslint@8.31.0)(typescript@5.2.2): + /@typescript-eslint/parser@5.59.6(eslint@8.31.0)(typescript@5.5.4): resolution: {integrity: sha512-7pCa6al03Pv1yf/dUg/s1pXz/yGMUBAw5EeWqNTFiSueKvRNonze3hma3lhdsOrQcaOXhbk5gKu2Fludiho9VA==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: @@ -16953,10 +16940,10 @@ packages: dependencies: '@typescript-eslint/scope-manager': 5.59.6 '@typescript-eslint/types': 5.59.6 - '@typescript-eslint/typescript-estree': 5.59.6(typescript@5.2.2) + '@typescript-eslint/typescript-estree': 5.59.6(typescript@5.5.4) debug: 4.3.4 eslint: 8.31.0 - typescript: 5.2.2 + typescript: 5.5.4 transitivePeerDependencies: - supports-color dev: true @@ -16969,7 +16956,7 @@ packages: '@typescript-eslint/visitor-keys': 5.59.6 dev: true - /@typescript-eslint/type-utils@5.59.6(eslint@8.31.0)(typescript@5.2.2): + /@typescript-eslint/type-utils@5.59.6(eslint@8.31.0)(typescript@5.5.4): resolution: {integrity: sha512-A4tms2Mp5yNvLDlySF+kAThV9VTBPCvGf0Rp8nl/eoDX9Okun8byTKoj3fJ52IJitjWOk0fKPNQhXEB++eNozQ==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: @@ -16979,12 +16966,12 @@ packages: typescript: optional: true dependencies: - '@typescript-eslint/typescript-estree': 5.59.6(typescript@5.2.2) - '@typescript-eslint/utils': 5.59.6(eslint@8.31.0)(typescript@5.2.2) + '@typescript-eslint/typescript-estree': 5.59.6(typescript@5.5.4) + '@typescript-eslint/utils': 5.59.6(eslint@8.31.0)(typescript@5.5.4) debug: 4.3.7 eslint: 8.31.0 - tsutils: 3.21.0(typescript@5.2.2) - typescript: 5.2.2 + tsutils: 3.21.0(typescript@5.5.4) + typescript: 5.5.4 transitivePeerDependencies: - supports-color dev: true @@ -16994,7 +16981,7 @@ packages: engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dev: true - /@typescript-eslint/typescript-estree@5.59.6(typescript@5.2.2): + /@typescript-eslint/typescript-estree@5.59.6(typescript@5.5.4): resolution: {integrity: sha512-vW6JP3lMAs/Tq4KjdI/RiHaaJSO7IUsbkz17it/Rl9Q+WkQ77EOuOnlbaU8kKfVIOJxMhnRiBG+olE7f3M16DA==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: @@ -17009,13 +16996,13 @@ packages: globby: 11.1.0 is-glob: 4.0.3 semver: 7.6.3 - tsutils: 3.21.0(typescript@5.2.2) - typescript: 5.2.2 + tsutils: 3.21.0(typescript@5.5.4) + typescript: 5.5.4 transitivePeerDependencies: - supports-color dev: true - /@typescript-eslint/utils@5.59.6(eslint@8.31.0)(typescript@5.2.2): + /@typescript-eslint/utils@5.59.6(eslint@8.31.0)(typescript@5.5.4): resolution: {integrity: sha512-vzaaD6EXbTS29cVH0JjXBdzMt6VBlv+hE31XktDRMX1j3462wZCJa7VzO2AxXEXcIl8GQqZPcOPuW/Z1tZVogg==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: @@ -17026,7 +17013,7 @@ packages: '@types/semver': 7.5.1 '@typescript-eslint/scope-manager': 5.59.6 '@typescript-eslint/types': 5.59.6 - '@typescript-eslint/typescript-estree': 5.59.6(typescript@5.2.2) + '@typescript-eslint/typescript-estree': 5.59.6(typescript@5.5.4) eslint: 8.31.0 eslint-scope: 5.1.1 semver: 7.6.3 @@ -17176,7 +17163,7 @@ packages: outdent: 0.8.0 dev: true - /@vanilla-extract/integration@6.2.1(@types/node@18.11.18): + /@vanilla-extract/integration@6.2.1(@types/node@20.14.14): resolution: {integrity: sha512-+xYJz07G7TFAMZGrOqArOsURG+xcYvqctujEkANjw2McCBvGEK505RxQqOuNiA9Mi9hgGdNp2JedSa94f3eoLg==} dependencies: '@babel/core': 7.22.17 @@ -17190,8 +17177,8 @@ packages: lodash: 4.17.21 mlly: 1.7.1 outdent: 0.8.0 - vite: 4.4.9(@types/node@18.11.18) - vite-node: 0.28.5(@types/node@18.11.18) + vite: 4.4.9(@types/node@20.14.14) + vite-node: 0.28.5(@types/node@20.14.14) transitivePeerDependencies: - '@types/node' - less @@ -18892,7 +18879,7 @@ packages: assertion-error: 1.1.0 check-error: 1.0.3 deep-eql: 4.1.3 - get-func-name: 2.0.0 + get-func-name: 2.0.2 loupe: 2.3.7 pathval: 1.1.1 type-detect: 4.0.8 @@ -19078,7 +19065,7 @@ packages: /cjs-module-lexer@1.2.3: resolution: {integrity: sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==} - /class-variance-authority@0.5.2(typescript@5.2.2): + /class-variance-authority@0.5.2(typescript@5.5.4): resolution: {integrity: sha512-j7Qqw3NPbs4IpO80gvdACWmVvHiLLo5MECacUBLnJG17CrLpWaQ7/4OaWX6P0IO1j2nvZ7AuSfBS/ImtEUZJGA==} peerDependencies: typescript: '>= 4.5.5 < 6' @@ -19086,7 +19073,7 @@ packages: typescript: optional: true dependencies: - typescript: 5.2.2 + typescript: 5.5.4 dev: false /class-variance-authority@0.7.0: @@ -19489,22 +19476,6 @@ packages: yaml: 1.10.2 dev: true - /cosmiconfig@8.3.6(typescript@5.2.2): - resolution: {integrity: sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==} - engines: {node: '>=14'} - peerDependencies: - typescript: '>=4.9.5' - peerDependenciesMeta: - typescript: - optional: true - dependencies: - import-fresh: 3.3.0 - js-yaml: 4.1.0 - parse-json: 5.2.0 - path-type: 4.0.0 - typescript: 5.2.2 - dev: false - /cosmiconfig@8.3.6(typescript@5.5.4): resolution: {integrity: sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==} engines: {node: '>=14'} @@ -19521,22 +19492,6 @@ packages: typescript: 5.5.4 dev: false - /cosmiconfig@9.0.0(typescript@5.2.2): - resolution: {integrity: sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==} - engines: {node: '>=14'} - peerDependencies: - typescript: '>=4.9.5' - peerDependenciesMeta: - typescript: - optional: true - dependencies: - env-paths: 2.2.1 - import-fresh: 3.3.0 - js-yaml: 4.1.0 - parse-json: 5.2.0 - typescript: 5.2.2 - dev: true - /cosmiconfig@9.0.0(typescript@5.5.4): resolution: {integrity: sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==} engines: {node: '>=14'} @@ -19551,7 +19506,6 @@ packages: js-yaml: 4.1.0 parse-json: 5.2.0 typescript: 5.5.4 - dev: false /cp-file@10.0.0: resolution: {integrity: sha512-vy2Vi1r2epK5WqxOLnskeKeZkdZvTKfFZQCplE3XWsP+SUJyd5XAUFC9lFgTjjXJF2GMne/UML14iEmkAaDfFg==} @@ -20448,7 +20402,7 @@ packages: dependencies: '@types/cookie': 0.4.1 '@types/cors': 2.8.17 - '@types/node': 18.19.20 + '@types/node': 20.14.14 accepts: 1.3.8 base64id: 2.0.0 cookie: 0.4.2 @@ -21204,7 +21158,7 @@ packages: eslint-import-resolver-webpack: optional: true dependencies: - '@typescript-eslint/parser': 5.59.6(eslint@8.31.0)(typescript@5.2.2) + '@typescript-eslint/parser': 5.59.6(eslint@8.31.0)(typescript@5.5.4) debug: 3.2.7 eslint: 8.31.0 eslint-import-resolver-node: 0.3.7 @@ -21234,7 +21188,7 @@ packages: eslint-import-resolver-webpack: optional: true dependencies: - '@typescript-eslint/parser': 5.59.6(eslint@8.31.0)(typescript@5.2.2) + '@typescript-eslint/parser': 5.59.6(eslint@8.31.0)(typescript@5.5.4) debug: 3.2.7 eslint: 8.31.0 eslint-import-resolver-node: 0.3.9 @@ -21264,7 +21218,7 @@ packages: '@typescript-eslint/parser': optional: true dependencies: - '@typescript-eslint/parser': 5.59.6(eslint@8.31.0)(typescript@5.2.2) + '@typescript-eslint/parser': 5.59.6(eslint@8.31.0)(typescript@5.5.4) array-includes: 3.1.8 array.prototype.findlastindex: 1.2.5 array.prototype.flat: 1.3.2 @@ -21301,7 +21255,7 @@ packages: requireindex: 1.2.0 dev: true - /eslint-plugin-jest@26.9.0(@typescript-eslint/eslint-plugin@5.59.6)(eslint@8.31.0)(typescript@5.2.2): + /eslint-plugin-jest@26.9.0(@typescript-eslint/eslint-plugin@5.59.6)(eslint@8.31.0)(typescript@5.5.4): resolution: {integrity: sha512-TWJxWGp1J628gxh2KhaH1H1paEdgE2J61BBF1I59c6xWeL5+D1BzMxGDN/nXAfX+aSkR5u80K+XhskK6Gwq9ng==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: @@ -21314,8 +21268,8 @@ packages: jest: optional: true dependencies: - '@typescript-eslint/eslint-plugin': 5.59.6(@typescript-eslint/parser@5.59.6)(eslint@8.31.0)(typescript@5.2.2) - '@typescript-eslint/utils': 5.59.6(eslint@8.31.0)(typescript@5.2.2) + '@typescript-eslint/eslint-plugin': 5.59.6(@typescript-eslint/parser@5.59.6)(eslint@8.31.0)(typescript@5.5.4) + '@typescript-eslint/utils': 5.59.6(eslint@8.31.0)(typescript@5.5.4) eslint: 8.31.0 transitivePeerDependencies: - supports-color @@ -21395,13 +21349,13 @@ packages: string.prototype.matchall: 4.0.8 dev: true - /eslint-plugin-testing-library@5.11.0(eslint@8.31.0)(typescript@5.2.2): + /eslint-plugin-testing-library@5.11.0(eslint@8.31.0)(typescript@5.5.4): resolution: {integrity: sha512-ELY7Gefo+61OfXKlQeXNIDVVLPcvKTeiQOoMZG9TeuWa7Ln4dUNRv8JdRWBQI9Mbb427XGlVB1aa1QPZxBJM8Q==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0, npm: '>=6'} peerDependencies: eslint: ^7.5.0 || ^8.0.0 dependencies: - '@typescript-eslint/utils': 5.59.6(eslint@8.31.0)(typescript@5.2.2) + '@typescript-eslint/utils': 5.59.6(eslint@8.31.0)(typescript@5.5.4) eslint: 8.31.0 transitivePeerDependencies: - supports-color @@ -22369,10 +22323,6 @@ packages: resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} engines: {node: 6.* || 8.* || >= 10.*} - /get-func-name@2.0.0: - resolution: {integrity: sha512-Hm0ixYtaSZ/V7C8FJrtZIuBBI+iSgL+1Aq82zSu8VQNB4S3Gk8e7Qs3VwBDJAhmRZcFqkl3tQu36g/Foh5I5ig==} - dev: true - /get-func-name@2.0.2: resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==} dev: true @@ -22693,27 +22643,6 @@ packages: - supports-color dev: false - /graphile-worker@0.16.6(patch_hash=hdpetta7btqcc7xb5wfkcnanoa)(typescript@5.2.2): - resolution: {integrity: sha512-e7gGYDmGqzju2l83MpzX8vNG/lOtVJiSzI3eZpAFubSxh/cxs7sRrRGBGjzBP1kNG0H+c95etPpNRNlH65PYhw==} - engines: {node: '>=14.0.0'} - hasBin: true - dependencies: - '@graphile/logger': 0.2.0 - '@types/debug': 4.1.12 - '@types/pg': 8.11.6 - cosmiconfig: 8.3.6(typescript@5.2.2) - graphile-config: 0.0.1-beta.8 - json5: 2.2.3 - pg: 8.11.5 - tslib: 2.6.2 - yargs: 17.7.2 - transitivePeerDependencies: - - pg-native - - supports-color - - typescript - dev: false - patched: true - /graphile-worker@0.16.6(patch_hash=hdpetta7btqcc7xb5wfkcnanoa)(typescript@5.5.4): resolution: {integrity: sha512-e7gGYDmGqzju2l83MpzX8vNG/lOtVJiSzI3eZpAFubSxh/cxs7sRrRGBGjzBP1kNG0H+c95etPpNRNlH65PYhw==} engines: {node: '>=14.0.0'} @@ -23668,7 +23597,7 @@ packages: resolution: {integrity: sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==} engines: {node: '>= 10.13.0'} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -25339,6 +25268,10 @@ packages: resolution: {integrity: sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==} dev: true + /node-abort-controller@3.1.1: + resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} + dev: false + /node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -26578,7 +26511,7 @@ packages: dependencies: lilconfig: 2.1.0 postcss: 8.4.29 - ts-node: 10.9.1(@swc/core@1.3.26)(@types/node@18.11.18)(typescript@5.2.2) + ts-node: 10.9.1(@swc/core@1.3.26)(@types/node@20.14.14)(typescript@5.5.4) yaml: 2.3.1 dev: true @@ -26596,10 +26529,10 @@ packages: dependencies: lilconfig: 2.1.0 postcss: 8.4.44 - ts-node: 10.9.1(@swc/core@1.3.26)(@types/node@18.11.18)(typescript@5.2.2) + ts-node: 10.9.1(@swc/core@1.3.26)(@types/node@20.14.14)(typescript@5.5.4) yaml: 2.3.1 - /postcss-loader@8.1.1(postcss@8.4.44)(typescript@5.2.2)(webpack@5.88.2): + /postcss-loader@8.1.1(postcss@8.4.44)(typescript@5.5.4)(webpack@5.88.2): resolution: {integrity: sha512-0IeqyAsG6tYiDRCYKQJLAmgQr47DX6N7sFSWvQxt6AcupX8DIdmykuk/o/tx0Lze3ErGHJEp5OSRxrelC6+NdQ==} engines: {node: '>= 18.12.0'} peerDependencies: @@ -26612,7 +26545,7 @@ packages: webpack: optional: true dependencies: - cosmiconfig: 9.0.0(typescript@5.2.2) + cosmiconfig: 9.0.0(typescript@5.5.4) jiti: 1.21.0 postcss: 8.4.44 semver: 7.6.3 @@ -27156,7 +27089,7 @@ packages: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 18.19.20 + '@types/node': 20.14.14 long: 5.2.3 /proxy-addr@2.0.7: @@ -28059,6 +27992,14 @@ packages: redis-errors: 1.2.0 dev: false + /redlock@5.0.0-beta.2(patch_hash=rwyegdki7iserrd7fgjwxkhnlu): + resolution: {integrity: sha512-2RDWXg5jgRptDrB1w9O/JgSZC0j7y4SlaXnor93H/UJm/QyDiFgBKNtrh0TI6oCXqYSaSoXxFh6Sd3VtYfhRXw==} + engines: {node: '>=12'} + dependencies: + node-abort-controller: 3.1.1 + dev: false + patched: true + /reduce-css-calc@2.1.8: resolution: {integrity: sha512-8liAVezDmUcH+tdzoEGrhfbGcP7nOV4NkGE3a74+qqvE7nt9i4sKLGBuZNOnpI4WiGksiNPklZxva80061QiPg==} dependencies: @@ -28231,7 +28172,7 @@ packages: '@remix-run/server-runtime': ^1.1.1 remix-auth: ^3.2.1 dependencies: - '@remix-run/server-runtime': 2.1.0(typescript@5.2.2) + '@remix-run/server-runtime': 2.1.0(typescript@5.5.4) crypto-js: 4.1.1 remix-auth: 3.6.0(@remix-run/react@2.1.0)(@remix-run/server-runtime@2.1.0) dev: false @@ -28242,7 +28183,7 @@ packages: '@remix-run/server-runtime': ^1.0.0 remix-auth: ^3.4.0 dependencies: - '@remix-run/server-runtime': 2.1.0(typescript@5.2.2) + '@remix-run/server-runtime': 2.1.0(typescript@5.5.4) remix-auth: 3.6.0(@remix-run/react@2.1.0)(@remix-run/server-runtime@2.1.0) remix-auth-oauth2: 1.11.0(@remix-run/server-runtime@2.1.0)(remix-auth@3.6.0) transitivePeerDependencies: @@ -28255,7 +28196,7 @@ packages: '@remix-run/server-runtime': ^1.0.0 || ^2.0.0 remix-auth: ^3.6.0 dependencies: - '@remix-run/server-runtime': 2.1.0(typescript@5.2.2) + '@remix-run/server-runtime': 2.1.0(typescript@5.5.4) debug: 4.3.7 remix-auth: 3.6.0(@remix-run/react@2.1.0)(@remix-run/server-runtime@2.1.0) transitivePeerDependencies: @@ -28268,8 +28209,8 @@ packages: '@remix-run/react': ^1.0.0 || ^2.0.0 '@remix-run/server-runtime': ^1.0.0 || ^2.0.0 dependencies: - '@remix-run/react': 2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.2.2) - '@remix-run/server-runtime': 2.1.0(typescript@5.2.2) + '@remix-run/react': 2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.5.4) + '@remix-run/server-runtime': 2.1.0(typescript@5.5.4) uuid: 8.3.2 dev: false @@ -28280,8 +28221,8 @@ packages: '@remix-run/server-runtime': ^1.16.0 || ^2.0 react: ^17.0.2 || ^18.0.0 dependencies: - '@remix-run/react': 2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.2.2) - '@remix-run/server-runtime': 2.1.0(typescript@5.2.2) + '@remix-run/react': 2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.5.4) + '@remix-run/server-runtime': 2.1.0(typescript@5.5.4) react: 18.2.0 dev: false @@ -28321,8 +28262,8 @@ packages: zod: optional: true dependencies: - '@remix-run/node': 2.1.0(typescript@5.2.2) - '@remix-run/react': 2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.2.2) + '@remix-run/node': 2.1.0(typescript@5.5.4) + '@remix-run/react': 2.1.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.5.4) '@remix-run/router': 1.15.3 intl-parse-accept-language: 1.0.0 react: 18.2.0 @@ -28998,7 +28939,7 @@ packages: engines: {node: '>=10.0.0'} dependencies: '@socket.io/component-emitter': 3.1.0 - debug: 4.3.4 + debug: 4.3.7 engine.io-client: 6.5.3 socket.io-parser: 4.2.4 transitivePeerDependencies: @@ -29040,7 +28981,7 @@ packages: accepts: 1.3.8 base64id: 2.0.0 cors: 2.8.5 - debug: 4.3.4 + debug: 4.3.7 engine.io: 6.5.4 socket.io-adapter: 2.5.4 socket.io-parser: 4.2.4 @@ -29515,7 +29456,7 @@ packages: resolution: {integrity: sha512-cYjgBM2SY/dTm8Lr6eMyyONaHTZHA/QjHxFUIW5WH8FevSRIGAVtXEmBkUXF1fsqe7QvvRgQSGSJZmjDacegGg==} engines: {node: '>=12.*'} dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 qs: 6.11.0 dev: false @@ -30321,7 +30262,7 @@ packages: /ts-interface-checker@0.1.13: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} - /ts-node@10.9.1(@swc/core@1.3.26)(@types/node@18.11.18)(typescript@5.2.2): + /ts-node@10.9.1(@swc/core@1.3.26)(@types/node@20.14.14)(typescript@5.5.4): resolution: {integrity: sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==} hasBin: true peerDependencies: @@ -30341,14 +30282,14 @@ packages: '@tsconfig/node12': 1.0.11 '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.3 - '@types/node': 18.11.18 + '@types/node': 20.14.14 acorn: 8.10.0 acorn-walk: 8.2.0 arg: 4.1.3 create-require: 1.1.1 diff: 4.0.2 make-error: 1.3.6 - typescript: 5.2.2 + typescript: 5.5.4 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 @@ -30413,19 +30354,6 @@ packages: resolution: {integrity: sha512-3IDBalvf6SyvHFS14UiwCWzqdSdo+Q0k2J7DZyJYaHW/iraW9DJpaBKDJpry3yQs3o/t/A+oGaRW3iVt2lKxzA==} dev: false - /tsconfck@2.1.2(typescript@5.2.2): - resolution: {integrity: sha512-ghqN1b0puy3MhhviwO2kGF8SeMDNhEbnKxjK7h6+fvY9JAxqvXi8y5NAHSQv687OVboS2uZIByzGd45/YxrRHg==} - engines: {node: ^14.13.1 || ^16 || >=18} - hasBin: true - peerDependencies: - typescript: ^4.3.5 || ^5.0.0 - peerDependenciesMeta: - typescript: - optional: true - dependencies: - typescript: 5.2.2 - dev: true - /tsconfck@2.1.2(typescript@5.5.4): resolution: {integrity: sha512-ghqN1b0puy3MhhviwO2kGF8SeMDNhEbnKxjK7h6+fvY9JAxqvXi8y5NAHSQv687OVboS2uZIByzGd45/YxrRHg==} engines: {node: ^14.13.1 || ^16 || >=18} @@ -30510,14 +30438,14 @@ packages: /tslib@2.6.2: resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} - /tsutils@3.21.0(typescript@5.2.2): + /tsutils@3.21.0(typescript@5.5.4): resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==} engines: {node: '>= 6'} peerDependencies: typescript: '>=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta' dependencies: tslib: 1.14.1 - typescript: 5.2.2 + typescript: 5.5.4 dev: true /tsx@3.12.2: @@ -30846,22 +30774,12 @@ packages: - supports-color dev: false - /typescript@4.9.5: - resolution: {integrity: sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==} - engines: {node: '>=4.2.0'} - hasBin: true - /typescript@5.1.6: resolution: {integrity: sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==} engines: {node: '>=14.17'} hasBin: true dev: false - /typescript@5.2.2: - resolution: {integrity: sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==} - engines: {node: '>=14.17'} - hasBin: true - /typescript@5.3.3: resolution: {integrity: sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==} engines: {node: '>=14.17'} @@ -31390,31 +31308,7 @@ packages: d3-timer: 3.0.1 dev: false - /vite-node@0.28.5(@types/node@18.11.18): - resolution: {integrity: sha512-LmXb9saMGlrMZbXTvOveJKwMTBTNUH66c8rJnQ0ZPNX+myPEol64+szRzXtV5ORb0Hb/91yq+/D3oERoyAt6LA==} - engines: {node: '>=v14.16.0'} - hasBin: true - dependencies: - cac: 6.7.14 - debug: 4.3.7 - mlly: 1.7.1 - pathe: 1.1.2 - picocolors: 1.0.1 - source-map: 0.6.1 - source-map-support: 0.5.21 - vite: 4.4.9(@types/node@18.11.18) - transitivePeerDependencies: - - '@types/node' - - less - - lightningcss - - sass - - stylus - - sugarss - - supports-color - - terser - dev: true - - /vite-node@0.28.5(@types/node@18.19.20): + /vite-node@0.28.5(@types/node@20.14.14): resolution: {integrity: sha512-LmXb9saMGlrMZbXTvOveJKwMTBTNUH66c8rJnQ0ZPNX+myPEol64+szRzXtV5ORb0Hb/91yq+/D3oERoyAt6LA==} engines: {node: '>=v14.16.0'} hasBin: true @@ -31426,7 +31320,7 @@ packages: picocolors: 1.0.1 source-map: 0.6.1 source-map-support: 0.5.21 - vite: 4.4.9(@types/node@18.19.20) + vite: 4.4.9(@types/node@20.14.14) transitivePeerDependencies: - '@types/node' - less @@ -31438,7 +31332,7 @@ packages: - terser dev: true - /vite-node@1.4.0(@types/node@18.11.18): + /vite-node@1.4.0(@types/node@20.14.14): resolution: {integrity: sha512-VZDAseqjrHgNd4Kh8icYHWzTKSCZMhia7GyHfhtzLW33fZlG9SwsB6CEhgyVOWkJfJ2pFLrp/Gj1FSfAiqH9Lw==} engines: {node: ^18.0.0 || >=20.0.0} hasBin: true @@ -31447,7 +31341,7 @@ packages: debug: 4.3.7 pathe: 1.1.2 picocolors: 1.0.1 - vite: 5.2.7(@types/node@18.11.18) + vite: 5.2.7(@types/node@20.14.14) transitivePeerDependencies: - '@types/node' - less @@ -31501,17 +31395,6 @@ packages: - terser dev: true - /vite-tsconfig-paths@4.0.5(typescript@5.2.2): - resolution: {integrity: sha512-/L/eHwySFYjwxoYt1WRJniuK/jPv+WGwgRGBYx3leciR5wBeqntQpUE6Js6+TJemChc+ter7fDBKieyEWDx4yQ==} - dependencies: - debug: 4.3.7 - globrex: 0.1.2 - tsconfck: 2.1.2(typescript@5.2.2) - transitivePeerDependencies: - - supports-color - - typescript - dev: true - /vite-tsconfig-paths@4.0.5(typescript@5.5.4): resolution: {integrity: sha512-/L/eHwySFYjwxoYt1WRJniuK/jPv+WGwgRGBYx3leciR5wBeqntQpUE6Js6+TJemChc+ter7fDBKieyEWDx4yQ==} dependencies: @@ -31523,40 +31406,6 @@ packages: - typescript dev: true - /vite@4.1.4(@types/node@18.19.20): - resolution: {integrity: sha512-3knk/HsbSTKEin43zHu7jTwYWv81f8kgAL99G5NWBcA1LKvtvcVAC4JjBH1arBunO9kQka+1oGbrMKOjk4ZrBg==} - engines: {node: ^14.18.0 || >=16.0.0} - hasBin: true - peerDependencies: - '@types/node': '>= 14' - less: '*' - sass: '*' - stylus: '*' - sugarss: '*' - terser: ^5.4.0 - peerDependenciesMeta: - '@types/node': - optional: true - less: - optional: true - sass: - optional: true - stylus: - optional: true - sugarss: - optional: true - terser: - optional: true - dependencies: - '@types/node': 18.19.20 - esbuild: 0.16.17 - postcss: 8.4.44 - resolve: 1.22.8 - rollup: 3.10.0 - optionalDependencies: - fsevents: 2.3.3 - dev: true - /vite@4.1.4(@types/node@20.14.14): resolution: {integrity: sha512-3knk/HsbSTKEin43zHu7jTwYWv81f8kgAL99G5NWBcA1LKvtvcVAC4JjBH1arBunO9kQka+1oGbrMKOjk4ZrBg==} engines: {node: ^14.18.0 || >=16.0.0} @@ -31591,7 +31440,7 @@ packages: fsevents: 2.3.3 dev: true - /vite@4.4.9(@types/node@18.11.18): + /vite@4.4.9(@types/node@20.14.14): resolution: {integrity: sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==} engines: {node: ^14.18.0 || >=16.0.0} hasBin: true @@ -31619,43 +31468,7 @@ packages: terser: optional: true dependencies: - '@types/node': 18.11.18 - esbuild: 0.18.11 - postcss: 8.4.44 - rollup: 3.29.1 - optionalDependencies: - fsevents: 2.3.3 - dev: true - - /vite@4.4.9(@types/node@18.19.20): - resolution: {integrity: sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==} - engines: {node: ^14.18.0 || >=16.0.0} - hasBin: true - peerDependencies: - '@types/node': '>= 14' - less: '*' - lightningcss: ^1.21.0 - sass: '*' - stylus: '*' - sugarss: '*' - terser: ^5.4.0 - peerDependenciesMeta: - '@types/node': - optional: true - less: - optional: true - lightningcss: - optional: true - sass: - optional: true - stylus: - optional: true - sugarss: - optional: true - terser: - optional: true - dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 esbuild: 0.18.11 postcss: 8.4.44 rollup: 3.29.1 @@ -31663,42 +31476,6 @@ packages: fsevents: 2.3.3 dev: true - /vite@5.2.7(@types/node@18.11.18): - resolution: {integrity: sha512-k14PWOKLI6pMaSzAuGtT+Cf0YmIx12z9YGon39onaJNy8DLBfBJrzg9FQEmkAM5lpHBZs9wksWAsyF/HkpEwJA==} - engines: {node: ^18.0.0 || >=20.0.0} - hasBin: true - peerDependencies: - '@types/node': ^18.0.0 || >=20.0.0 - less: '*' - lightningcss: ^1.21.0 - sass: '*' - stylus: '*' - sugarss: '*' - terser: ^5.4.0 - peerDependenciesMeta: - '@types/node': - optional: true - less: - optional: true - lightningcss: - optional: true - sass: - optional: true - stylus: - optional: true - sugarss: - optional: true - terser: - optional: true - dependencies: - '@types/node': 18.11.18 - esbuild: 0.20.2 - postcss: 8.4.44 - rollup: 4.13.2 - optionalDependencies: - fsevents: 2.3.3 - dev: true - /vite@5.2.7(@types/node@20.14.14): resolution: {integrity: sha512-k14PWOKLI6pMaSzAuGtT+Cf0YmIx12z9YGon39onaJNy8DLBfBJrzg9FQEmkAM5lpHBZs9wksWAsyF/HkpEwJA==} engines: {node: ^18.0.0 || >=20.0.0} @@ -31759,7 +31536,7 @@ packages: dependencies: '@types/chai': 4.3.4 '@types/chai-subset': 1.3.3 - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@vitest/expect': 0.28.5 '@vitest/runner': 0.28.5 '@vitest/spy': 0.28.5 @@ -31778,8 +31555,8 @@ packages: tinybench: 2.3.1 tinypool: 0.3.1 tinyspy: 1.0.2 - vite: 4.1.4(@types/node@18.19.20) - vite-node: 0.28.5(@types/node@18.19.20) + vite: 4.1.4(@types/node@20.14.14) + vite-node: 0.28.5(@types/node@20.14.14) why-is-node-running: 2.2.2 transitivePeerDependencies: - less @@ -31791,7 +31568,7 @@ packages: - terser dev: true - /vitest@1.4.0(@types/node@18.11.18): + /vitest@1.4.0(@types/node@20.14.14): resolution: {integrity: sha512-gujzn0g7fmwf83/WzrDTnncZt2UiXP41mHuFYFrdwaLRVQ6JYQEiME2IfEjU3vcFL3VKa75XhI3lFgn+hfVsQw==} engines: {node: ^18.0.0 || >=20.0.0} hasBin: true @@ -31816,7 +31593,7 @@ packages: jsdom: optional: true dependencies: - '@types/node': 18.11.18 + '@types/node': 20.14.14 '@vitest/expect': 1.4.0 '@vitest/runner': 1.4.0 '@vitest/snapshot': 1.4.0 @@ -31834,8 +31611,8 @@ packages: strip-literal: 2.1.0 tinybench: 2.6.0 tinypool: 0.8.3 - vite: 5.2.7(@types/node@18.11.18) - vite-node: 1.4.0(@types/node@18.11.18) + vite: 5.2.7(@types/node@20.14.14) + vite-node: 1.4.0(@types/node@20.14.14) why-is-node-running: 2.2.2 transitivePeerDependencies: - less diff --git a/references/bun-catalog/package.json b/references/bun-catalog/package.json index 483cfe2c54..519296c6ca 100644 --- a/references/bun-catalog/package.json +++ b/references/bun-catalog/package.json @@ -11,7 +11,6 @@ }, "devDependencies": { "@types/bun": "^1.1.6", - "trigger.dev": "workspace:*", - "typescript": "^5.5.4" + "trigger.dev": "workspace:*" } } \ No newline at end of file diff --git a/references/hello-world/package.json b/references/hello-world/package.json index c38f17fa7e..b8c3a8ad1b 100644 --- a/references/hello-world/package.json +++ b/references/hello-world/package.json @@ -3,8 +3,7 @@ "private": true, "type": "module", "devDependencies": { - "trigger.dev": "workspace:*", - "typescript": "^5.5.4" + "trigger.dev": "workspace:*" }, "dependencies": { "@trigger.dev/sdk": "workspace:*" diff --git a/references/hello-world/src/trigger/example.ts b/references/hello-world/src/trigger/example.ts index 50d76069b0..18cb388b25 100644 --- a/references/hello-world/src/trigger/example.ts +++ b/references/hello-world/src/trigger/example.ts @@ -1,4 +1,4 @@ -import { logger, task, timeout, usage, wait } from "@trigger.dev/sdk/v3"; +import { batch, logger, task, timeout, usage, wait } from "@trigger.dev/sdk/v3"; import { setTimeout } from "timers/promises"; export const helloWorldTask = task({ @@ -26,20 +26,62 @@ export const parentTask = task({ }, }); +export const batchParentTask = task({ + id: "batch-parent", + run: async (payload: any, { ctx }) => { + logger.log("Hello, world from the parent", { payload }); + + const results = await childTask.batchTriggerAndWait([ + { payload: { message: "Hello, world!" } }, + { payload: { message: "Hello, world 2!" } }, + ]); + logger.log("Results", { results }); + + const results2 = await batch.triggerAndWait([ + { id: "child", payload: { message: "Hello, world !" } }, + { id: "child", payload: { message: "Hello, world 2!" } }, + ]); + logger.log("Results 2", { results2 }); + + const results3 = await batch.triggerByTask([ + { task: childTask, payload: { message: "Hello, world !" } }, + { task: childTask, payload: { message: "Hello, world 2!" } }, + ]); + logger.log("Results 3", { results3 }); + + const results4 = await batch.triggerByTaskAndWait([ + { task: childTask, payload: { message: "Hello, world !" } }, + { task: childTask, payload: { message: "Hello, world 2!" } }, + ]); + logger.log("Results 4", { results4 }); + }, +}); + export const childTask = task({ id: "child", - run: async (payload: any, { ctx }) => { - logger.info("Hello, world from the child", { payload }); + run: async ( + { + message, + failureChance = 0.3, + duration = 3_000, + }: { message?: string; failureChance?: number; duration?: number }, + { ctx } + ) => { + logger.info("Hello, world from the child", { message, failureChance }); - if (Math.random() > 0.5) { + if (Math.random() < failureChance) { throw new Error("Random error at start"); } - await setTimeout(10000); + await setTimeout(duration); - if (Math.random() > 0.5) { + if (Math.random() < failureChance) { throw new Error("Random error at end"); } + + return { + message, + }; }, }); diff --git a/references/hello-world/src/trigger/idempotency.ts b/references/hello-world/src/trigger/idempotency.ts new file mode 100644 index 0000000000..9136399cc4 --- /dev/null +++ b/references/hello-world/src/trigger/idempotency.ts @@ -0,0 +1,76 @@ +import { batch, idempotencyKeys, logger, task, timeout, usage, wait } from "@trigger.dev/sdk/v3"; +import { setTimeout } from "timers/promises"; +import { childTask } from "./example.js"; + +export const idempotency = task({ + id: "idempotency", + run: async (payload: any, { ctx }) => { + logger.log("Hello, world from the parent", { payload }); + + const child1Key = await idempotencyKeys.create("a", { scope: "global" }); + + const child1 = await childTask.triggerAndWait( + { message: "Hello, world!", duration: 10_000 }, + { idempotencyKey: child1Key, idempotencyKeyTTL: "60s" } + ); + logger.log("Child 1", { child1 }); + + ctx.attempt.id; + + const child2 = await childTask.triggerAndWait( + { message: "Hello, world!", duration: 10_000 }, + { idempotencyKey: child1Key, idempotencyKeyTTL: "60s" } + ); + logger.log("Child 2", { child2 }); + + // const results = await childTask.batchTriggerAndWait([ + // { + // payload: { message: "Hello, world!" }, + // //@ts-ignore + // options: { idempotencyKey: "1", idempotencyKeyTTL: "60s" }, + // }, + // { + // payload: { message: "Hello, world 2!" }, + // //@ts-ignore + // options: { idempotencyKey: "2", idempotencyKeyTTL: "60s" }, + // }, + // ]); + // logger.log("Results", { results }); + + // const results2 = await batch.triggerAndWait([ + // { + // id: "child", + // payload: { message: "Hello, world !" }, + // //@ts-ignore + // options: { idempotencyKey: "1", idempotencyKeyTTL: "60s" }, + // }, + // { + // id: "child", + // payload: { message: "Hello, world 2!" }, + // //@ts-ignore + // options: { idempotencyKey: "2", idempotencyKeyTTL: "60s" }, + // }, + // ]); + // logger.log("Results 2", { results2 }); + + // const results3 = await batch.triggerByTask([ + // { + // task: childTask, + // payload: { message: "Hello, world !" }, + // options: { idempotencyKey: "1", idempotencyKeyTTL: "60s" }, + // }, + // { + // task: childTask, + // payload: { message: "Hello, world 2!" }, + // options: { idempotencyKey: "2", idempotencyKeyTTL: "60s" }, + // }, + // ]); + // logger.log("Results 3", { results3 }); + + // const results4 = await batch.triggerByTaskAndWait([ + // { task: childTask, payload: { message: "Hello, world !" } }, + // { task: childTask, payload: { message: "Hello, world 2!" } }, + // ]); + // logger.log("Results 4", { results4 }); + }, +}); diff --git a/references/init-shell/package.json b/references/init-shell/package.json index 89c828db71..bf3ef58089 100644 --- a/references/init-shell/package.json +++ b/references/init-shell/package.json @@ -3,7 +3,6 @@ "private": true, "type": "module", "devDependencies": { - "trigger.dev": "workspace:*", - "typescript": "^5.5.4" + "trigger.dev": "workspace:*" } } \ No newline at end of file diff --git a/references/nextjs-realtime/package.json b/references/nextjs-realtime/package.json index 8985fc2a32..cb88e054e9 100644 --- a/references/nextjs-realtime/package.json +++ b/references/nextjs-realtime/package.json @@ -38,12 +38,10 @@ "devDependencies": { "@next/bundle-analyzer": "^15.0.2", "@trigger.dev/rsc": "workspace:^3", - "@types/node": "^20", "@types/react": "^18", "@types/react-dom": "^18", "postcss": "^8", "tailwindcss": "^3.4.1", - "trigger.dev": "workspace:^3", - "typescript": "^5" + "trigger.dev": "workspace:^3" } } \ No newline at end of file diff --git a/references/v3-catalog/package.json b/references/v3-catalog/package.json index c9de9cf913..90977430bc 100644 --- a/references/v3-catalog/package.json +++ b/references/v3-catalog/package.json @@ -75,14 +75,12 @@ "@trigger.dev/build": "workspace:*", "@types/email-reply-parser": "^1.4.2", "@types/fluent-ffmpeg": "^2.1.26", - "@types/node": "20.4.2", "@types/react": "^18.3.1", "esbuild": "^0.19.11", "prisma": "5.19.0", "prisma-kysely": "^1.8.0", "trigger.dev": "workspace:*", "ts-node": "^10.9.2", - "tsconfig-paths": "^4.2.0", - "typescript": "^5.5.4" + "tsconfig-paths": "^4.2.0" } } \ No newline at end of file