v3: small features and fixes for prod infra (#968)

* add checkpoint restore events

* fix retries.enabledInDev

* fix tsconfig paths

* fix provider build and dev

* update kubernetes provider and fix builds again

* upgrade prod base to node 20

* update infra publish workflow

* rethrow k8s errors after logging

* shorten index container names

* fix error type assertion

* rename type assertion

* remove resource limits for now

* add missing run id on create

* add push to deploy command for self-hosting

* checkpointing fixes

* update coordinator image

* ensure valid registry login

* delete checkpoint archive after successful push

* log options on error

* structured logs for socket connections

* fix structured log merge

* exit process after checkpointing

* update restore pull secret name

* append shortcode to restore names

* log handler payload

* disable post start lifecycle hook

* pass in coordinator host via volume

* replace dapi with taskinfo

* add missing restore label

* don't restart restored containers

* remove init container from create

* atomic post-completion checkpoints

* switch to run id for container names

* improve wait accuracy

* measure basic checkpoint perf

* always log disconnect reason

* use system clock to end wait spans

* checkpoint readiness and cancel signals

* restore from checkpoint events and fix statuses

* remove attempt id env var

* restore dependencies from events

* reconnect wip

* lifecycle hooks are back

* fix hooks and improve reconnect

* make docker send postStart hook

* only checkpoint for retry if large delay

* fix a few more resume issues

* lifecycle hook fixes

* skip connection handler when waiting for post start hook

---------

Co-authored-by: Eric Allam <[email protected]>

Author: nicktrn

.github/workflows/publish-dev.yml renamed to .github/workflows/publish-infra.yml

@@ -1,9 +1,11 @@
-name: "🚢 Publish Container Images (dev)"
+name: "🚢 Publish Infra Images"
 
 on:
   push:
     tags:
-      - "dev-*"
+      - "infra-dev-*"
+      - "infra-test-*"
+      - "infra-prod-*"
     paths:
       - ".github/workflows/publish.yml"
       - "packages/**"
@@ -44,12 +46,21 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Generate build ID
+      - name: Generate image reference
         id: prep
+        # WARNING: This step expects the workflow to have been triggered by a specific tag format of: infra-${env}-*
         run: |
+          env=$(echo ${{ github.ref_name }} | cut -d- -f2)
           sha=${GITHUB_SHA::7}
           ts=$(date +%s)
-          echo "BUILD_ID=${{ matrix.package }}-${sha}-${ts}" >> "$GITHUB_OUTPUT"
+          if [[ "${{ matrix.package }}" == *-provider ]]; then
+            provider_type=$(echo ${{ matrix.package }} | cut -d- -f1)
+            repository=provider/${provider_type}
+          else
+            repository=${{ matrix.package }}
+          fi
+          echo "IMAGE_TAG=${env}-${sha}-${ts}" >> "$GITHUB_OUTPUT"
+          echo "REPOSITORY=${repository}" >> "$GITHUB_OUTPUT"
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -63,7 +74,7 @@ jobs:
 
       - name: 🚢 Build Container Image
         run: |
-          docker build -t dev_image -f ./apps/${{ matrix.package }}/Containerfile .
+          docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile .
 
       # ..to push image
       - name: 🐙 Login to GitHub Container Registry
@@ -75,9 +86,9 @@ jobs:
 
       - name: 🐙 Push to GitHub Container Registry
         run: |
-          docker tag dev_image $REGISTRY/$REPOSITORY:$IMAGE_TAG
+          docker tag infra_image $REGISTRY/$REPOSITORY:$IMAGE_TAG
           docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
         env:
           REGISTRY: ghcr.io/triggerdotdev
-          REPOSITORY: dev
-          IMAGE_TAG: ${{ steps.prep.outputs.BUILD_ID }}
+          REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }}
+          IMAGE_TAG: ${{ steps.prep.outputs.IMAGE_TAG }}

apps/coordinator/Containerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:labs
 
-FROM node:18.18.2-bullseye-slim@sha256:21479df46c3173ee0cefc6b264928e10239152c4f74df872ca9369be01a245b7 AS node-18
+FROM node:18-bullseye-slim@sha256:a4edd54dcfdcacc8a4100fee71498e8671d99556a1acf5614539214a70092426 AS node-18
 
 WORKDIR /app
 
@@ -35,7 +35,6 @@ COPY --from=pruner --chown=node:node /app/out/full/ .
 COPY --from=dev-deps --chown=node:node /app/ .
 COPY --chown=node:node turbo.json turbo.json
 
-RUN pnpm run -r --filter '@trigger.dev/core*' build
 RUN pnpm run -r --filter coordinator build:bundle
 
 FROM alpine AS cri-tools
@@ -58,6 +57,4 @@ COPY --from=builder --chown=node:node /app/apps/coordinator/dist/index.mjs ./ind
 
 EXPOSE 8000
 
-USER node
-
 CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]

apps/coordinator/package.json

@@ -6,7 +6,7 @@
   "main": "dist/index.cjs",
   "scripts": {
     "build": "npm run build:bundle",
-    "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"const require = createRequire(import.meta.url);\"",
+    "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
     "build:image": "docker build -f Containerfile . -t coordinator",
     "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
     "start": "tsx src/index.ts",
@@ -19,6 +19,7 @@
     "@trigger.dev/core": "workspace:*",
     "@trigger.dev/core-apps": "workspace:*",
     "execa": "^8.0.1",
+    "nanoid": "^5.0.6",
     "prom-client": "^15.1.0",
     "socket.io": "^4.7.4",
     "socket.io-client": "^4.7.4"