diff --git a/.planning/phases/13-launch-verification/evidence/06-hotfix-alpha24.md b/.planning/phases/13-launch-verification/evidence/06-hotfix-alpha24.md new file mode 100644 index 0000000..2aa3a4b --- /dev/null +++ b/.planning/phases/13-launch-verification/evidence/06-hotfix-alpha24.md @@ -0,0 +1,46 @@ +# Hotfix Release v1.0.0-alpha.24 + +**Date:** 2026-09-29 +**Trigger:** Plan 13-04's LAUNCH-01 verification, retried after v1.0.0-alpha.23's PDF-extraction +fix — the resubmitted job still failed, this time with a generic "An unexpected error occurred" +message and, critically, **zero worker log lines** for the job on the production container. + +## Root cause + +Cross-checked BullMQ's own job history directly against Redis (Upstash) from inside the app +container: + +``` +completed job 4 analysisId= ed338c9f-... ts= 2026-09-29T18:19:10.219Z +completed job 3 analysisId= 8f1bea54-... ts= 2026-09-29T18:07:36.614Z +``` + +Both of this session's test jobs show as **completed** at the BullMQ level, with timestamps +matching exactly when they were submitted — proving a worker process did pick them up and run +them to completion (no exception escaped to mark them "failed" at the queue level). But +`clarify-worker-prod`'s own logs (checked via `docker logs` and the raw +`/var/lib/docker/containers/.../*-json.log` file directly) contain **zero** lines about either +job — not even the unconditional `[Worker] Started processing...` line that is always the very +first thing the processing function logs. + +Found the actual consumer: a local dev Docker container (`clarify-app`, `nuxt dev --port 3001`, +running continuously for 3 days) shares the exact same Upstash Redis instance as production — the +server's `.env` was intentionally copied from Carlos's local dev `.env` during 13-02, including +`REDIS_HOST`/`REDIS_PORT`/`REDIS_TOKEN`. Local dev runs a single combined process (no +`DISABLE_WORKER`), so it also runs the BullMQ worker in-process, competing with production's +dedicated worker container for the same unprefixed `analysis-queue`. Whichever consumer's `BRPOPLPUSH` +wins claims the job; this time it was local dev, which presumably failed it for reasons specific to +that environment (not diagnosed further — out of scope once the collision itself was identified). + +## Fix + +PR #57 (`fix/bullmq-queue-prefix-per-environment` → `develop`): BullMQ's built-in `prefix` option +namespaces all of its Redis keys. Added a shared `queuePrefix` constant (`"prod"` in production, +`"dev"` everywhere else, keyed off `NODE_ENV`) used identically by both the `Queue` +(`server/utils/queue.ts`) and the `Worker` (`server/plugins/worker.ts`) — they still see each +other's jobs within the same environment, but no longer collide across environments. + +## This release + +Fourth same-day hotfix, following the same GitFlow process verified in plan 13-03 and repeated for +v1.0.0-alpha.21, v1.0.0-alpha.22, and v1.0.0-alpha.23. diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d4b9a9..76231cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. +## [1.0.0-alpha.24] - 2026-09-29 + +### Fixed +- Local dev and production shared the same unprefixed BullMQ "analysis-queue" on the same Upstash Redis instance, so a long-running local `nuxt dev` process raced production's worker for the same jobs and sometimes claimed and failed them before production ever saw them — discovered live during LAUNCH-01 verification via BullMQ's own job history showing jobs "completed" that the production worker's logs never mentioned. Fixed by namespacing BullMQ's Redis keys with an environment-derived `prefix` (`prod` vs `dev`). + ## [1.0.0-alpha.23] - 2026-09-29 ### Fixed diff --git a/package.json b/package.json index b255f3b..ffcb430 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clarify", - "version": "1.0.0-alpha.23", + "version": "1.0.0-alpha.24", "private": true, "type": "module", "scripts": { diff --git a/server/plugins/worker.ts b/server/plugins/worker.ts index 98c6237..2a580ba 100644 --- a/server/plugins/worker.ts +++ b/server/plugins/worker.ts @@ -1,5 +1,5 @@ import { Worker } from "bullmq"; -import { getRedisConnection } from "../utils/queue"; +import { getRedisConnection, queuePrefix } from "../utils/queue"; import { extractTextFromPDF } from "../utils/pdf-parser"; import { analyzeContract } from "../utils/openai-client"; import { getWorkerSupabaseClient } from "../utils/worker-supabase"; @@ -166,6 +166,7 @@ export default defineNitroPlugin((_nitroApp) => { }, { connection: getRedisConnection() as any, // Cast to any to resolve ioredis version mismatch + prefix: queuePrefix, // must match getAnalysisQueue()'s prefix in ../utils/queue concurrency: parseInt(process.env.BULLMQ_CONCURRENCY || '2', 10), defaultJobOptions: { attempts: 3, diff --git a/server/utils/queue.ts b/server/utils/queue.ts index c8c22d7..9e36e3e 100644 --- a/server/utils/queue.ts +++ b/server/utils/queue.ts @@ -4,6 +4,16 @@ import { Redis } from "ioredis"; let analysisQueue: Queue | null = null; let redisConnection: Redis | null = null; +// [LAUNCH-01 FIX] Namespaces BullMQ's Redis keys per environment. Local dev +// and production point at the same Upstash instance (same REDIS_HOST/TOKEN +// copied from .env for convenience), so without this both a locally running +// `nuxt dev` and the production worker race to claim jobs from the same +// unprefixed "analysis-queue" — discovered live when a production job was +// silently stolen and failed by a 3-day-old local dev process. Must match +// the prefix used when constructing the Worker in server/plugins/worker.ts. +export const queuePrefix = + process.env.NODE_ENV === "production" ? "prod" : "dev"; + export const getRedisConnection = () => { if (!redisConnection) { const config = useRuntimeConfig(); @@ -31,6 +41,7 @@ export const getAnalysisQueue = () => { if (!analysisQueue) { analysisQueue = new Queue("analysis-queue", { connection: getRedisConnection() as any, // Cast to any to resolve ioredis version mismatch + prefix: queuePrefix, defaultJobOptions: { attempts: 3, backoff: {