From aeab881e0baaf203d378347d173b823c7bcd58e2 Mon Sep 17 00:00:00 2001 From: Amal Date: Fri, 17 Jul 2026 00:45:14 -0700 Subject: [PATCH 1/3] feat: durable BullMQ job queues for document conversion and tabular extraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WHY THIS MATTERS Two workloads in Mike are expensive and can outlive the HTTP request that started them: DOCX -> PDF conversion (LibreOffice) and tabular-review cell extraction (one LLM call per row). Today both run inline on the request thread, so a closed laptop lid, a dropped connection, or a server restart mid-run silently loses the work — the review grid is left with spinners that never resolve, and a large upload blocks its request on LibreOffice. WHAT IS A DURABLE JOB QUEUE A job queue moves work out of the request/response cycle: the request records WHAT should happen (a small JSON payload in Redis) and returns; a worker process picks the job up, runs it, and retries it with exponential backoff if it fails. "Durable" means the job survives the death of the thing that created it — the queue (BullMQ on Redis) holds the job until a worker finishes it, no matter what happens to the original HTTP request or even the server process (BullMQ re-queues jobs whose worker crashed via its stalled-job detection). The classic hazard of queues is the DOUBLE SUBMIT: a client that reconnects and re-POSTs would enqueue the same work twice. This design pushes correctness into the queue's identity model — every job's id is derived deterministically from the work itself (`convert:`, `extract::`), so BullMQ collapses a duplicate submit into the already-in-flight job. Durable STATE lives only in Postgres (documents.status, tabular_cells); jobs re-read that state when they run and skip columns already done, which is what makes retries idempotent. HOW IT WORKS - Both queues are OFF by default and opt-in per deployment: ASYNC_DOCUMENT_CONVERSION / ASYNC_TABULAR_EXTRACTION (default "false"). With the flags off the server never dials Redis — the queue connection is created lazily and only reached via enqueue/startWorkers, so a fresh clone still runs fully synchronously with zero new infrastructure. - lib/queue/: a shared lazy Redis connection (maxRetriesPerRequest: null, which BullMQ's blocking commands require), the two queues with deterministic jobIds + retry/backoff, and runProgress — a Redis pub/sub bridge that carries per-cell progress frames from workers to any HTTP request that is watching. - workers/: conversionWorker (DOCX->PDF off the request thread; conversion failure finalizes the document without a PDF rendition, matching the sync path) and extractionWorker (throws on incomplete extraction so BullMQ retries; after the last retry a permanent-failure handler flips surviving cells to "error" so the grid never shows an eternal spinner). A declarative registry + startWorkers()/stopWorkers() lifecycle, started from index.ts only when a flag is on, with graceful SIGTERM/SIGINT drain. - lib/tabular/: the extraction core factored out of routes/tabular.ts so the synchronous route and the async worker share ONE loop (extractRowColumns). The unit of work is the review ROW — one document, or a folder of source documents extracted together — matching the row model main adopted for folder-grouped reviews. tabular.rows.ts carries the row loaders (loadReviewRows / loadRowDocumentText) that both the routes and the worker need. - POST /:reviewId/generate keeps its exact synchronous behavior by default; with the flag on it enqueues one job per row, subscribes to the review's progress channel BEFORE enqueuing (so a fast worker cannot publish into the void), and forwards the same cell_update SSE frames the sync path emits. A 3-second DB-poll backstop reconciles any missed pub/sub frame, so a dropped message can never hang the stream. A new GET /:reviewId/generate/stream lets a disconnected client reattach to a running generation without re-triggering work. Ported from amal66/mike (upstream-pr/durable-queues, amal66/mike#40) and re-derived against current main: the extraction core is row-based (not document-based) to match the folder-grouped row model (#274) and db pagination (#263) that landed after the original branch, and the moved helper bodies match main's current copies byte-for-byte (multi-document citation prompts, Ollama key exemption). Tests: 510 passing (was 499), including queue jobId determinism, worker idempotency/retry/permanent-failure policy, row extraction core, and the pending-cell targeting used by the reconnectable stream. Co-Authored-By: Claude Fable 5 --- backend/.env.example | 16 + backend/bun.lock | 49 ++ backend/package-lock.json | 278 ++++++- backend/package.json | 2 + backend/src/index.ts | 39 +- backend/src/lib/pdfjs.ts | 48 ++ .../queue/__tests__/conversionQueue.test.ts | 58 ++ .../queue/__tests__/extractionQueue.test.ts | 59 ++ backend/src/lib/queue/connection.ts | 32 + backend/src/lib/queue/conversionQueue.ts | 58 ++ backend/src/lib/queue/extractionQueue.ts | 68 ++ backend/src/lib/queue/runProgress.ts | 46 ++ backend/src/lib/sseHeartbeat.ts | 29 + .../__tests__/tabular.extractRow.test.ts | 166 ++++ .../__tests__/tabular.generateStream.test.ts | 50 ++ backend/src/lib/tabular/tabular.extract.ts | 297 +++++++ backend/src/lib/tabular/tabular.extractRow.ts | 122 +++ backend/src/lib/tabular/tabular.generate.ts | 93 +++ .../src/lib/tabular/tabular.generateStream.ts | 275 +++++++ backend/src/lib/tabular/tabular.prompt.ts | 31 + backend/src/lib/tabular/tabular.rows.ts | 142 ++++ backend/src/lib/tabular/tabular.shared.ts | 108 +++ backend/src/routes/documents.ts | 23 +- backend/src/routes/tabular.ts | 779 ++++-------------- .../__tests__/conversionWorker.test.ts | 146 ++++ .../__tests__/extractionWorker.test.ts | 298 +++++++ backend/src/workers/conversionWorker.ts | 153 ++++ backend/src/workers/extractionWorker.ts | 223 +++++ backend/src/workers/index.ts | 28 + backend/src/workers/registry.ts | 51 ++ 30 files changed, 3132 insertions(+), 635 deletions(-) create mode 100644 backend/src/lib/pdfjs.ts create mode 100644 backend/src/lib/queue/__tests__/conversionQueue.test.ts create mode 100644 backend/src/lib/queue/__tests__/extractionQueue.test.ts create mode 100644 backend/src/lib/queue/connection.ts create mode 100644 backend/src/lib/queue/conversionQueue.ts create mode 100644 backend/src/lib/queue/extractionQueue.ts create mode 100644 backend/src/lib/queue/runProgress.ts create mode 100644 backend/src/lib/sseHeartbeat.ts create mode 100644 backend/src/lib/tabular/__tests__/tabular.extractRow.test.ts create mode 100644 backend/src/lib/tabular/__tests__/tabular.generateStream.test.ts create mode 100644 backend/src/lib/tabular/tabular.extract.ts create mode 100644 backend/src/lib/tabular/tabular.extractRow.ts create mode 100644 backend/src/lib/tabular/tabular.generate.ts create mode 100644 backend/src/lib/tabular/tabular.generateStream.ts create mode 100644 backend/src/lib/tabular/tabular.prompt.ts create mode 100644 backend/src/lib/tabular/tabular.rows.ts create mode 100644 backend/src/lib/tabular/tabular.shared.ts create mode 100644 backend/src/workers/__tests__/conversionWorker.test.ts create mode 100644 backend/src/workers/__tests__/extractionWorker.test.ts create mode 100644 backend/src/workers/conversionWorker.ts create mode 100644 backend/src/workers/extractionWorker.ts create mode 100644 backend/src/workers/index.ts create mode 100644 backend/src/workers/registry.ts diff --git a/backend/.env.example b/backend/.env.example index 9ccb8488f..e0b9e3c3d 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -30,3 +30,19 @@ COURTLISTENER_API_TOKEN=your-courtlistener-token # GET /manifest-signing-key. Rotating the key does not invalidate past exports, # but whoever checks one needs the key that was current when it was made. MANIFEST_SIGNING_KEY= + +# Optional durable job queues (BullMQ). Only needed when an ASYNC_* flag below +# is "true"; the default (synchronous) deployment needs no Redis. +REDIS_URL=redis://localhost:6379 +# When "true", DOCX→PDF conversion is enqueued to the BullMQ document-conversion +# queue (uploads return status "processing"; an in-process worker converts and +# flips to "ready"). Requires REDIS_URL + the frontend to poll document status. +# Default "false" runs conversion inline on the request thread. +ASYNC_DOCUMENT_CONVERSION=false +# When "true", tabular-review cell extraction runs on the BullMQ +# tabular-extraction queue (one job per document) instead of inline in the +# POST /tabular-review/:id/generate request. Extraction then survives client +# disconnects + server restarts and retries failed documents; the request tails +# progress over Redis pub/sub and can be resumed via GET .../generate/stream. +# Requires REDIS_URL. Default "false" runs extraction inline. +ASYNC_TABULAR_EXTRACTION=false diff --git a/backend/bun.lock b/backend/bun.lock index c6bd03654..1bf2c981c 100644 --- a/backend/bun.lock +++ b/backend/bun.lock @@ -11,6 +11,7 @@ "@google/genai": "^1.50.1", "@modelcontextprotocol/sdk": "^1.29.0", "@supabase/supabase-js": "^2.49.4", + "bullmq": "^5.34.0", "cors": "^2.8.5", "docx": "^9.5.0", "dotenv": "^17.4.1", @@ -19,12 +20,14 @@ "fast-diff": "^1.3.0", "fast-xml-parser": "^5.7.1", "helmet": "^8.1.0", + "ioredis": "^5.11.1", "jszip": "^3.10.1", "libreoffice-convert": "^1.6.0", "mammoth": "^1.9.0", "multer": "^1.4.5-lts.2", "pdfjs-dist": "^4.10.38", "resend": "^4.5.1", + "undici": "^6.27.0", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", "zod": "^3.25.76", }, @@ -204,6 +207,8 @@ "@hono/node-server": ["@hono/node-server@1.19.14", "", { "peerDependencies": { "hono": "^4" } }, "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw=="], + "@ioredis/commands": ["@ioredis/commands@1.10.0", "", {}, "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q=="], + "@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="], "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="], @@ -212,6 +217,18 @@ "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.29.0", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ=="], + "@msgpackr-extract/msgpackr-extract-darwin-arm64": ["@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ=="], + + "@msgpackr-extract/msgpackr-extract-darwin-x64": ["@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w=="], + + "@msgpackr-extract/msgpackr-extract-linux-arm": ["@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4", "", { "os": "linux", "cpu": "arm" }, "sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw=="], + + "@msgpackr-extract/msgpackr-extract-linux-arm64": ["@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw=="], + + "@msgpackr-extract/msgpackr-extract-linux-x64": ["@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4", "", { "os": "linux", "cpu": "x64" }, "sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ=="], + + "@msgpackr-extract/msgpackr-extract-win32-x64": ["@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4", "", { "os": "win32", "cpu": "x64" }, "sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ=="], + "@napi-rs/canvas": ["@napi-rs/canvas@0.1.97", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "0.1.97", "@napi-rs/canvas-darwin-arm64": "0.1.97", "@napi-rs/canvas-darwin-x64": "0.1.97", "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.97", "@napi-rs/canvas-linux-arm64-gnu": "0.1.97", "@napi-rs/canvas-linux-arm64-musl": "0.1.97", "@napi-rs/canvas-linux-riscv64-gnu": "0.1.97", "@napi-rs/canvas-linux-x64-gnu": "0.1.97", "@napi-rs/canvas-linux-x64-musl": "0.1.97", "@napi-rs/canvas-win32-arm64-msvc": "0.1.97", "@napi-rs/canvas-win32-x64-msvc": "0.1.97" } }, "sha512-8cFniXvrIEnVwuNSRCW9wirRZbHvrD3JVujdS2P5n5xiJZNZMOZcfOvJ1pb66c7jXMKHHglJEDVJGbm8XWFcXQ=="], "@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@0.1.97", "", { "os": "android", "cpu": "arm64" }, "sha512-V1c/WVw+NzH8vk7ZK/O8/nyBSCQimU8sfMsB/9qeSvdkGKNU7+mxy/bIF0gTgeBFmHpj30S4E9WHMSrxXGQuVQ=="], @@ -520,6 +537,8 @@ "buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="], + "bullmq": ["bullmq@5.81.3", "", { "dependencies": { "cron-parser": "4.9.0", "ioredis": "5.11.1", "msgpackr": "2.0.5", "node-abort-controller": "3.1.1", "semver": "7.8.5", "tslib": "2.8.1" }, "peerDependencies": { "redis": ">=5.0.0" }, "optionalPeers": ["redis"] }, "sha512-Q7uEH2G92rVjX3Yl2qw3+6VEO15uxehC6DfeTHQBQTehddcRDSDuquD9zuvDjxiCffXsPUaBwrRlvlbVcgKs7g=="], + "busboy": ["busboy@1.6.0", "", { "dependencies": { "streamsearch": "^1.1.0" } }, "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA=="], "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="], @@ -530,6 +549,8 @@ "chai": ["chai@6.2.2", "", {}, "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg=="], + "cluster-key-slot": ["cluster-key-slot@1.1.1", "", {}, "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw=="], + "combined-stream": ["combined-stream@1.0.8", "", { "dependencies": { "delayed-stream": "~1.0.0" } }, "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg=="], "component-emitter": ["component-emitter@1.3.1", "", {}, "sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ=="], @@ -552,6 +573,8 @@ "cors": ["cors@2.8.6", "", { "dependencies": { "object-assign": "^4", "vary": "^1" } }, "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw=="], + "cron-parser": ["cron-parser@4.9.0", "", { "dependencies": { "luxon": "^3.2.1" } }, "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q=="], + "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], "data-uri-to-buffer": ["data-uri-to-buffer@4.0.1", "", {}, "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="], @@ -562,6 +585,8 @@ "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="], + "denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="], + "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="], "destroy": ["destroy@1.2.0", "", {}, "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg=="], @@ -706,6 +731,8 @@ "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], + "ioredis": ["ioredis@5.11.1", "", { "dependencies": { "@ioredis/commands": "1.10.0", "cluster-key-slot": "1.1.1", "debug": "4.4.3", "denque": "2.1.0", "redis-errors": "1.2.0", "redis-parser": "3.0.0", "standard-as-callback": "2.1.0" } }, "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A=="], + "ip-address": ["ip-address@10.2.0", "", {}, "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA=="], "ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], @@ -774,6 +801,8 @@ "lop": ["lop@0.4.2", "", { "dependencies": { "duck": "^0.1.12", "option": "~0.2.1", "underscore": "^1.13.1" } }, "sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw=="], + "luxon": ["luxon@3.7.2", "", {}, "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew=="], + "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], "magicast": ["magicast@0.5.3", "", { "dependencies": { "@babel/parser": "^7.29.3", "@babel/types": "^7.29.0", "source-map-js": "^1.2.1" } }, "sha512-pVKE4UdSQ7DvHzivsCIFx2BJn1mHG6KsyrFcaxFx6tONdneEuThrDx0Cj3AMg58KyN4pzYT+LHOotxDQDjNvkw=="], @@ -804,16 +833,24 @@ "ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], + "msgpackr": ["msgpackr@2.0.5", "", { "optionalDependencies": { "msgpackr-extract": "^3.0.4" } }, "sha512-cef05H/dSYpLpqp3sj/qyZh5vhUYCalnaLO7j1yOmpsR0y/XwLVtK7r5gn+U/F7CTEfMowcGhlUQJDLcLf7jcA=="], + + "msgpackr-extract": ["msgpackr-extract@3.0.4", "", { "dependencies": { "node-gyp-build-optional-packages": "5.2.2" }, "optionalDependencies": { "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.4", "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.4", "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.4", "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.4", "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.4", "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.4" }, "bin": { "download-msgpackr-prebuilds": "bin/download-prebuilds.js" } }, "sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw=="], + "multer": ["multer@1.4.5-lts.2", "", { "dependencies": { "append-field": "^1.0.0", "busboy": "^1.0.0", "concat-stream": "^1.5.2", "mkdirp": "^0.5.4", "object-assign": "^4.1.1", "type-is": "^1.6.4", "xtend": "^4.0.0" } }, "sha512-VzGiVigcG9zUAoCNU+xShztrlr1auZOlurXynNvO9GiWD1/mTBbUljOKY+qMeazBqXgRnjzeEgJI/wyjJUHg9A=="], "nanoid": ["nanoid@5.1.7", "", { "bin": "bin/nanoid.js" }, "sha512-ua3NDgISf6jdwezAheMOk4mbE1LXjm1DfMUDMuJf4AqxLFK3ccGpgWizwa5YV7Yz9EpXwEaWoRXSb/BnV0t5dQ=="], "negotiator": ["negotiator@0.6.3", "", {}, "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="], + "node-abort-controller": ["node-abort-controller@3.1.1", "", {}, "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ=="], + "node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="], "node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="], + "node-gyp-build-optional-packages": ["node-gyp-build-optional-packages@5.2.2", "", { "dependencies": { "detect-libc": "^2.0.1" }, "bin": { "node-gyp-build-optional-packages": "bin.js", "node-gyp-build-optional-packages-optional": "optional.js", "node-gyp-build-optional-packages-test": "build-test.js" } }, "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw=="], + "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="], @@ -878,6 +915,10 @@ "readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], + "redis-errors": ["redis-errors@1.2.0", "", {}, "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w=="], + + "redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="], + "require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="], "resend": ["resend@4.8.0", "", { "dependencies": { "@react-email/render": "1.1.2" } }, "sha512-R8eBOFQDO6dzRTDmaMEdpqrkmgSjPpVXt4nGfWsZdYOet0kqra0xgbvTES6HmCriZEXbmGk3e0DiGIaLFTFSHA=="], @@ -930,6 +971,8 @@ "stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="], + "standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="], + "statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="], "std-env": ["std-env@4.2.0", "", {}, "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw=="], @@ -972,6 +1015,8 @@ "underscore": ["underscore@1.13.8", "", {}, "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ=="], + "undici": ["undici@6.28.0", "", {}, "sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA=="], + "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], "unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="], @@ -1040,6 +1085,8 @@ "https-proxy-agent/debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + "ioredis/debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + "postcss/nanoid": ["nanoid@3.3.16", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q=="], "protobufjs/@types/node": ["@types/node@25.5.2", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-tO4ZIRKNC+MDWV4qKVZe3Ql/woTnmHDr5JD8UI5hn2pwBrHEwOEMZK7WlNb5RKB6EoJ02gwmQS9OrjuFnZYdpg=="], @@ -1104,6 +1151,8 @@ "https-proxy-agent/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "ioredis/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "protobufjs/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], "router/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], diff --git a/backend/package-lock.json b/backend/package-lock.json index e997f52d0..acd123ac5 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -15,6 +15,7 @@ "@google/genai": "^1.50.1", "@modelcontextprotocol/sdk": "^1.29.0", "@supabase/supabase-js": "^2.49.4", + "bullmq": "^5.34.0", "cors": "^2.8.5", "docx": "^9.5.0", "dotenv": "^17.4.1", @@ -23,6 +24,7 @@ "fast-diff": "^1.3.0", "fast-xml-parser": "^5.7.1", "helmet": "^8.1.0", + "ioredis": "^5.11.1", "jszip": "^3.10.1", "libreoffice-convert": "^1.6.0", "mammoth": "^1.9.0", @@ -1552,6 +1554,12 @@ "hono": "^4" } }, + "node_modules/@ioredis/commands": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", + "integrity": "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==", + "license": "MIT" + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", @@ -1937,6 +1945,84 @@ "url": "https://opencollective.com/express" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.4.tgz", + "integrity": "sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.4.tgz", + "integrity": "sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.4.tgz", + "integrity": "sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.4.tgz", + "integrity": "sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.4.tgz", + "integrity": "sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.4.tgz", + "integrity": "sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@napi-rs/canvas": { "version": "0.1.97", "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.97.tgz", @@ -4002,6 +4088,31 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "license": "MIT" }, + "node_modules/bullmq": { + "version": "5.81.3", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.81.3.tgz", + "integrity": "sha512-Q7uEH2G92rVjX3Yl2qw3+6VEO15uxehC6DfeTHQBQTehddcRDSDuquD9zuvDjxiCffXsPUaBwrRlvlbVcgKs7g==", + "license": "MIT", + "dependencies": { + "cron-parser": "4.9.0", + "ioredis": "5.11.1", + "msgpackr": "2.0.5", + "node-abort-controller": "3.1.1", + "semver": "7.8.5", + "tslib": "2.8.1" + }, + "engines": { + "node": ">=12.22.0" + }, + "peerDependencies": { + "redis": ">=5.0.0" + }, + "peerDependenciesMeta": { + "redis": { + "optional": true + } + } + }, "node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -4061,6 +4172,15 @@ "node": ">=18" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz", + "integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -4172,6 +4292,19 @@ "url": "https://opencollective.com/express" } }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "deprecated": "v4 is no longer maintained, upgrade to v5", + "license": "MIT", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -4223,6 +4356,15 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -4246,7 +4388,7 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "engines": { "node": ">=8" @@ -5192,6 +5334,51 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, + "node_modules/ioredis": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.11.1.tgz", + "integrity": "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.10.0", + "cluster-key-slot": "1.1.1", + "debug": "4.4.3", + "denque": "2.1.0", + "redis-errors": "1.2.0", + "redis-parser": "3.0.0", + "standard-as-callback": "2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ioredis/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/ioredis/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, "node_modules/ip-address": { "version": "10.2.0", "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", @@ -5659,6 +5846,15 @@ "underscore": "^1.13.1" } }, + "node_modules/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -5823,6 +6019,37 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "license": "MIT" }, + "node_modules/msgpackr": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-2.0.5.tgz", + "integrity": "sha512-cef05H/dSYpLpqp3sj/qyZh5vhUYCalnaLO7j1yOmpsR0y/XwLVtK7r5gn+U/F7CTEfMowcGhlUQJDLcLf7jcA==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.4" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.4.tgz", + "integrity": "sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.4" + } + }, "node_modules/multer": { "version": "1.4.5-lts.2", "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.5-lts.2.tgz", @@ -5869,6 +6096,12 @@ "node": ">= 0.6" } }, + "node_modules/node-abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", + "license": "MIT" + }, "node_modules/node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", @@ -5907,6 +6140,21 @@ "url": "https://opencollective.com/node-fetch" } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -6304,6 +6552,27 @@ "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "license": "MIT" }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", @@ -6485,7 +6754,6 @@ "version": "7.8.5", "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -6674,6 +6942,12 @@ "dev": true, "license": "MIT" }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", diff --git a/backend/package.json b/backend/package.json index 95cd7773c..70d8e15e9 100644 --- a/backend/package.json +++ b/backend/package.json @@ -17,6 +17,7 @@ "@google/genai": "^1.50.1", "@modelcontextprotocol/sdk": "^1.29.0", "@supabase/supabase-js": "^2.49.4", + "bullmq": "^5.34.0", "cors": "^2.8.5", "docx": "^9.5.0", "dotenv": "^17.4.1", @@ -25,6 +26,7 @@ "fast-diff": "^1.3.0", "fast-xml-parser": "^5.7.1", "helmet": "^8.1.0", + "ioredis": "^5.11.1", "jszip": "^3.10.1", "libreoffice-convert": "^1.6.0", "mammoth": "^1.9.0", diff --git a/backend/src/index.ts b/backend/src/index.ts index 1b9baf421..52630913d 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -1,5 +1,6 @@ import { app } from "./app"; import { manifestPublicKey } from "./lib/manifestSigning"; +import { anyWorkerEnabled, startWorkers, stopWorkers } from "./workers"; const PORT = process.env.PORT ?? 3001; @@ -17,6 +18,42 @@ try { process.exit(1); } -app.listen(PORT, () => { +const server = app.listen(PORT, () => { console.log(`Mike backend running on port ${PORT}`); + // Start in-process job-queue workers only when at least one async queue is + // enabled, so the default (synchronous) deployment needs no Redis. + if (anyWorkerEnabled()) { + startWorkers(); + } }); + +// Graceful shutdown: on SIGTERM/SIGINT (orchestrator rollout, Ctrl-C), stop +// accepting new connections, let in-flight requests/streams drain, close the +// job-queue workers + Redis, then exit 0. Without this the orchestrator's +// grace period elapses and SIGKILL drops in-flight streams and leaves queue +// state dirty. A hard timeout guards against a connection that never drains. +let shuttingDown = false; +async function shutdown(signal: string) { + if (shuttingDown) return; + shuttingDown = true; + console.log(`Shutting down gracefully (${signal})`); + const forceExit = setTimeout(() => { + console.error("Graceful shutdown timed out — forcing exit"); + process.exit(1); + }, 15_000); + forceExit.unref(); + try { + await new Promise((resolve, reject) => + server.close((err) => (err ? reject(err) : resolve())), + ); + await stopWorkers(); + console.log("Shutdown complete"); + process.exit(0); + } catch (err) { + console.error("Error during graceful shutdown", err); + process.exit(1); + } +} + +process.on("SIGTERM", () => void shutdown("SIGTERM")); +process.on("SIGINT", () => void shutdown("SIGINT")); diff --git a/backend/src/lib/pdfjs.ts b/backend/src/lib/pdfjs.ts new file mode 100644 index 000000000..30e98c371 --- /dev/null +++ b/backend/src/lib/pdfjs.ts @@ -0,0 +1,48 @@ +// Minimal typed facade over the slice of `pdfjs-dist` we actually use. +// +// We import the library's legacy ESM build via a dynamic `import()` whose +// specifier is cast to `string` so it resolves at runtime (the legacy build +// ships no usable type declarations). Rather than repeat an +// `as unknown as { getDocument: ... }` shape at every call site, we declare +// the surface once here and load through `loadPdfjs()`. + +export interface PdfTextItem { + str?: string; + hasEOL?: boolean; +} + +export interface PdfTextContent { + items: PdfTextItem[]; +} + +export interface PdfPage { + getTextContent(): Promise; +} + +export interface PdfDocument { + numPages: number; + getPage(n: number): Promise; +} + +export interface PdfDocumentTask { + promise: Promise; +} + +export interface PdfjsLib { + getDocument(opts: { + data: Uint8Array; + standardFontDataUrl?: string; + }): PdfDocumentTask; +} + +/** + * Load the pdfjs legacy build, typed as the {@link PdfjsLib} facade. + * + * The specifier is cast to `string` so TypeScript treats it as a dynamic + * runtime import (the legacy `.mjs` build has no bundled types); the awaited + * module is therefore `any`, which we narrow to the facade here in one place. + */ +export async function loadPdfjs(): Promise { + const mod = await import("pdfjs-dist/legacy/build/pdf.mjs" as string); + return mod as PdfjsLib; +} diff --git a/backend/src/lib/queue/__tests__/conversionQueue.test.ts b/backend/src/lib/queue/__tests__/conversionQueue.test.ts new file mode 100644 index 000000000..a2177acdc --- /dev/null +++ b/backend/src/lib/queue/__tests__/conversionQueue.test.ts @@ -0,0 +1,58 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +vi.mock("../connection", () => ({ + getRedisConnection: () => ({}), +})); + +const add = vi.fn(); +vi.mock("bullmq", () => ({ + Queue: class { + add = add; + }, +})); + +import { + conversionJobId, + enqueueConversion, + type ConversionJobData, +} from "../conversionQueue"; + +const DATA: ConversionJobData = { + documentId: "doc-1", + versionId: "ver-1", + userId: "user-1", + storagePath: "uploads/user-1/doc-1.docx", + fileType: "docx", +}; + +beforeEach(() => { + add.mockReset(); +}); + +describe("conversionJobId", () => { + it("is deterministic on the versionId", () => { + expect(conversionJobId("ver-1")).toBe("convert:ver-1"); + }); +}); + +describe("enqueueConversion", () => { + it("dedupes with a deterministic jobId of convert:", () => { + enqueueConversion(DATA); + + expect(add).toHaveBeenCalledTimes(1); + const [name, data, opts] = add.mock.calls[0]; + expect(name).toBe("convert"); + expect(data).toEqual(DATA); + expect(opts.jobId).toBe("convert:ver-1"); + }); + + it("keeps the existing retry/backoff/history options", () => { + enqueueConversion(DATA); + + const opts = add.mock.calls[0][2]; + expect(opts.attempts).toBe(3); + expect(opts.backoff).toEqual({ type: "exponential", delay: 2000 }); + expect(opts.removeOnComplete).toBe(100); + expect(opts.removeOnFail).toBe(500); + }); +}); diff --git a/backend/src/lib/queue/__tests__/extractionQueue.test.ts b/backend/src/lib/queue/__tests__/extractionQueue.test.ts new file mode 100644 index 000000000..ba1d38a02 --- /dev/null +++ b/backend/src/lib/queue/__tests__/extractionQueue.test.ts @@ -0,0 +1,59 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +vi.mock("../connection", () => ({ + getRedisConnection: () => ({}), +})); + +const add = vi.fn(); +vi.mock("bullmq", () => ({ + Queue: class { + add = add; + }, +})); + +import { + extractionJobId, + enqueueExtraction, + type ExtractionJobData, +} from "../extractionQueue"; + +const DATA: ExtractionJobData = { + reviewId: "rev-1", + userId: "user-1", + rowId: "row-1", +}; + +beforeEach(() => { + add.mockReset(); +}); + +describe("extractionJobId", () => { + it("is deterministic on (reviewId, rowId)", () => { + expect(extractionJobId("rev-1", "row-1")).toBe("extract:rev-1:row-1"); + }); +}); + +describe("enqueueExtraction", () => { + it("dedupes with a deterministic jobId of extract::", () => { + enqueueExtraction(DATA); + + expect(add).toHaveBeenCalledTimes(1); + const [name, data, opts] = add.mock.calls[0]; + expect(name).toBe("extract"); + expect(data).toEqual(DATA); + expect(opts.jobId).toBe("extract:rev-1:row-1"); + }); + + it("retries with backoff and removes terminal jobs so re-runs can re-enqueue", () => { + enqueueExtraction(DATA); + + const opts = add.mock.calls[0][2]; + expect(opts.attempts).toBe(3); + expect(opts.backoff).toEqual({ type: "exponential", delay: 2000 }); + // removeOnComplete/Fail === true (not a keep-N count) is deliberate: + // durable state lives in tabular_cells, and immediate removal lets a + // later regenerate enqueue the same deterministic jobId again. + expect(opts.removeOnComplete).toBe(true); + expect(opts.removeOnFail).toBe(true); + }); +}); diff --git a/backend/src/lib/queue/connection.ts b/backend/src/lib/queue/connection.ts new file mode 100644 index 000000000..73a64135d --- /dev/null +++ b/backend/src/lib/queue/connection.ts @@ -0,0 +1,32 @@ +import IORedis from "ioredis"; + +/** REDIS_URL points at the Redis instance backing BullMQ; defaults to + * localhost for bare-metal dev. Only ever dialled when an ASYNC_* queue + * flag is turned on — the default (synchronous) deployment needs no Redis. */ +export const REDIS_URL = process.env.REDIS_URL || "redis://localhost:6379"; + +/** + * Shared Redis connection for BullMQ (queues + workers). Lazily created and + * reused so producers and in-process workers share one client. + * + * `maxRetriesPerRequest: null` is required by BullMQ: its blocking commands + * (BRPOPLPUSH etc.) must not be aborted by ioredis's per-request retry cap. + */ +let connection: IORedis | null = null; + +export function getRedisConnection(): IORedis { + if (!connection) { + connection = new IORedis(REDIS_URL, { + maxRetriesPerRequest: null, + enableReadyCheck: false, + }); + } + return connection; +} + +export async function closeRedisConnection(): Promise { + if (connection) { + await connection.quit(); + connection = null; + } +} diff --git a/backend/src/lib/queue/conversionQueue.ts b/backend/src/lib/queue/conversionQueue.ts new file mode 100644 index 000000000..31d0cedd1 --- /dev/null +++ b/backend/src/lib/queue/conversionQueue.ts @@ -0,0 +1,58 @@ +import { Queue } from "bullmq"; +import { getRedisConnection } from "./connection"; + +/** BullMQ queue that runs DOCX/DOC → PDF conversion off the request thread. */ +export const CONVERSION_QUEUE = "document-conversion"; + +export interface ConversionJobData { + /** documents.id — the row whose status flips processing → ready. */ + documentId: string; + /** document_versions.id — the row whose pdf_storage_path the worker fills. */ + versionId: string; + /** Owner — used to derive the converted-PDF storage key. */ + userId: string; + /** Storage key of the uploaded original (the DOCX/DOC). */ + storagePath: string; + /** "docx" | "doc". */ + fileType: string; +} + +let queue: Queue | null = null; + +export function getConversionQueue(): Queue { + if (!queue) { + queue = new Queue(CONVERSION_QUEUE, { + connection: getRedisConnection(), + }); + } + return queue; +} + +/** Deterministic BullMQ jobId for a conversion. */ +export function conversionJobId(versionId: string): string { + return `convert:${versionId}`; +} + +/** + * Enqueue a conversion. Retries transient failures (storage/LibreOffice + * hiccups) with exponential backoff; keeps a bounded history for inspection. + * + * The jobId is derived from the (unique-per-upload) versionId so a double + * submit is deduped by BullMQ instead of racing two conversions. + */ +export function enqueueConversion(data: ConversionJobData) { + return getConversionQueue().add("convert", data, { + jobId: conversionJobId(data.versionId), + attempts: 3, + backoff: { type: "exponential", delay: 2000 }, + removeOnComplete: 100, + removeOnFail: 500, + }); +} + +export async function closeConversionQueue(): Promise { + if (queue) { + await queue.close(); + queue = null; + } +} diff --git a/backend/src/lib/queue/extractionQueue.ts b/backend/src/lib/queue/extractionQueue.ts new file mode 100644 index 000000000..8dae3e35d --- /dev/null +++ b/backend/src/lib/queue/extractionQueue.ts @@ -0,0 +1,68 @@ +import { Queue } from "bullmq"; +import { getRedisConnection } from "./connection"; + +/** + * BullMQ queue that runs tabular-review cell extraction off the request thread. + * + * One job == one (review, row) pair — a row is one document or a folder of + * source documents extracted together. The job re-derives everything it needs + * from the database at run time (review columns, current cell state, the row's + * source documents, the owner's model + API keys), so the job payload stays tiny + * and — importantly — carries NO secrets into Redis. This also makes the job + * idempotent and retry-safe: on a retry it re-reads cell state and only + * processes columns that are not already `done`. + */ +export const EXTRACTION_QUEUE = "tabular-extraction"; + +export interface ExtractionJobData { + /** tabular_reviews.id the cells belong to. */ + reviewId: string; + /** Owner — used to resolve the model + API keys the extraction runs under. */ + userId: string; + /** tabular_review_rows.id whose columns this job fills. */ + rowId: string; +} + +let queue: Queue | null = null; + +export function getExtractionQueue(): Queue { + if (!queue) { + queue = new Queue(EXTRACTION_QUEUE, { + connection: getRedisConnection(), + }); + } + return queue; +} + +/** Deterministic BullMQ jobId for one (review, row) extraction. */ +export function extractionJobId(reviewId: string, rowId: string): string { + return `extract:${reviewId}:${rowId}`; +} + +/** + * Enqueue extraction for one row of a review. Retries transient failures + * (LLM/network/storage hiccups) with exponential backoff. + * + * The jobId is deterministic on (reviewId, rowId) so a double submit — e.g. + * a client reconnecting and re-POSTing /generate — is deduped by BullMQ into the + * in-flight job instead of racing a second extraction over the same row. + * We `removeOnComplete`/`removeOnFail` immediately (not keep-N) precisely so a + * later re-run (regenerate) can enqueue the same jobId again; durable state + * lives in the `tabular_cells` table, not in the job record. + */ +export function enqueueExtraction(data: ExtractionJobData) { + return getExtractionQueue().add("extract", data, { + jobId: extractionJobId(data.reviewId, data.rowId), + attempts: 3, + backoff: { type: "exponential", delay: 2000 }, + removeOnComplete: true, + removeOnFail: true, + }); +} + +export async function closeExtractionQueue(): Promise { + if (queue) { + await queue.close(); + queue = null; + } +} diff --git a/backend/src/lib/queue/runProgress.ts b/backend/src/lib/queue/runProgress.ts new file mode 100644 index 000000000..a3c376370 --- /dev/null +++ b/backend/src/lib/queue/runProgress.ts @@ -0,0 +1,46 @@ +import { getRedisConnection } from "./connection"; + +/** + * Redis pub/sub bridge between the extraction worker and the SSE request that a + * client is tailing. The worker publishes per-cell progress; the /generate + * stream subscribes and forwards those frames to the browser. + * + * The DB (`tabular_cells`) is the source of truth — pub/sub is only the + * low-latency delivery path. The stream handler additionally reconciles against + * the DB on an interval, so a dropped message never leaves a stream hung. + */ + +/** Channel a given review's extraction progress is published on. */ +export function runProgressChannel(reviewId: string): string { + return `tabular-run:${reviewId}`; +} + +/** One progress frame — the same shape the SSE `cell_update` event carries. */ +export interface CellUpdate { + type: "cell_update"; + row_id: string; + column_index: number; + content: unknown; + status: "generating" | "done" | "error"; +} + +/** + * Publish one cell update for a review. Best-effort: a publish failure must not + * fail the extraction (the DB write is what matters), so errors are swallowed. + * PUBLISH is an ordinary Redis command, so it safely shares the BullMQ + * connection (which is never put into subscriber mode). + */ +export async function publishCellUpdate( + reviewId: string, + update: CellUpdate, +): Promise { + try { + await getRedisConnection().publish( + runProgressChannel(reviewId), + JSON.stringify(update), + ); + } catch { + // Non-fatal: the worker has already persisted the cell; the tailing + // stream's DB-poll backstop will pick the state change up. + } +} diff --git a/backend/src/lib/sseHeartbeat.ts b/backend/src/lib/sseHeartbeat.ts new file mode 100644 index 000000000..42c18cc54 --- /dev/null +++ b/backend/src/lib/sseHeartbeat.ts @@ -0,0 +1,29 @@ +import type { Response } from "express"; + +/** Default heartbeat cadence: comfortably under the ~30–60s idle window that + * most proxies/load-balancers enforce before dropping a quiet connection. */ +export const SSE_HEARTBEAT_MS = 15_000; + +/** + * Keep an SSE connection warm during long silences. + * + * A long-running tool call can produce no SSE output for many seconds, and + * proxies/load-balancers frequently close a connection that's been idle (no + * bytes) for ~30–60s — killing the stream mid-tool-call. This writes an SSE + * comment line (`:\n\n`), which EventSource clients ignore, at a fixed interval + * so the pipe keeps seeing traffic. (Distinct from the 180s stream watchdog, + * which bounds total duration; this bounds *idle* duration.) + * + * Returns a stop() that clears the timer; safe to call more than once. The timer + * is unref'd so it never keeps the process alive on its own. + */ +export function startSseHeartbeat( + res: Pick, + intervalMs: number = SSE_HEARTBEAT_MS, +): () => void { + const timer = setInterval(() => { + if (!res.writableEnded) res.write(": keepalive\n\n"); + }, intervalMs); + if (typeof timer.unref === "function") timer.unref(); + return () => clearInterval(timer); +} diff --git a/backend/src/lib/tabular/__tests__/tabular.extractRow.test.ts b/backend/src/lib/tabular/__tests__/tabular.extractRow.test.ts new file mode 100644 index 000000000..012d99500 --- /dev/null +++ b/backend/src/lib/tabular/__tests__/tabular.extractRow.test.ts @@ -0,0 +1,166 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +const queryTabularAllColumns = vi.fn(); +vi.mock("../tabular.extract", () => ({ + queryTabularAllColumns: (...a: unknown[]) => queryTabularAllColumns(...a), +})); + +const loadRowDocumentText = vi.fn(); +vi.mock("../tabular.rows", () => ({ + loadRowDocumentText: (...a: unknown[]) => loadRowDocumentText(...a), +})); + +import { extractRowColumns } from "../tabular.extractRow"; +import type { ReviewRow } from "../tabular.rows"; + +type Call = { table: string; op: string; payload?: Record }; +function makeDb() { + const calls: Call[] = []; + function from(table: string) { + const state: Call = { table, op: "select" }; + const b: Record = { + update(payload: Record) { + state.op = "update"; + state.payload = payload; + return b; + }, + insert(payload: Record) { + calls.push({ table, op: "insert", payload }); + return Promise.resolve({ data: null, error: null }); + }, + eq() { + return b; + }, + then(onF: (v: unknown) => unknown) { + calls.push({ ...state }); + return Promise.resolve({ data: null, error: null }).then(onF); + }, + }; + return b; + } + return { calls, from }; +} + +const COLUMNS = [ + { index: 0, name: "A", prompt: "a" }, + { index: 1, name: "B", prompt: "b" }, +]; +const ROW: ReviewRow = { + id: "row-1", + review_id: "rev-1", + label: "Contract.pdf", + row_type: "document", + folder_id: null, + library_folder_id: null, + document_id: "doc-1", + sort_index: 0, + source_document_ids: ["doc-1"], +}; +const RESULT = (i: number) => ({ summary: `c${i}`, flag: "green" as const, reasoning: "" }); + +function sinkSpy() { + return { + generating: vi.fn(), + done: vi.fn(), + }; +} + +beforeEach(() => { + loadRowDocumentText.mockReset(); + loadRowDocumentText.mockResolvedValue("## Source document: Contract.pdf\ntext"); + queryTabularAllColumns.mockReset(); +}); + +describe("extractRowColumns", () => { + it("processes all columns, persists done, and reports none missing", async () => { + queryTabularAllColumns.mockImplementation( + async (_m, _f, _t, cols, onResult) => { + for (const c of cols) await onResult(c.index, RESULT(c.index)); + }, + ); + const db = makeDb(); + const sink = sinkSpy(); + + const out = await extractRowColumns({ + db: db as never, + reviewId: "rev-1", + row: ROW, + columns: COLUMNS, + existingByColumn: new Map(), // no cells yet + model: "m", + apiKeys: {}, + sink, + }); + + expect(out.processed).toHaveLength(2); + expect([...out.received].sort()).toEqual([0, 1]); + expect(out.missing).toEqual([]); + // new cells are inserted with the row identity attached + const inserts = db.calls.filter((c) => c.op === "insert"); + expect(inserts).toHaveLength(2); + expect(inserts[0].payload).toMatchObject({ + review_id: "rev-1", + row_id: "row-1", + document_id: "doc-1", + }); + expect(sink.generating).toHaveBeenCalledTimes(2); + expect(sink.generating).toHaveBeenCalledWith("row-1", 0); + expect(sink.done).toHaveBeenCalledTimes(2); + // the LLM is prompted with the row's label and combined source text + expect(queryTabularAllColumns.mock.calls[0][1]).toBe("Contract.pdf"); + expect(loadRowDocumentText).toHaveBeenCalledTimes(1); + }); + + it("skips columns already done with content (no LLM call, no text load)", async () => { + const db = makeDb(); + const sink = sinkSpy(); + + const out = await extractRowColumns({ + db: db as never, + reviewId: "rev-1", + row: ROW, + columns: COLUMNS, + existingByColumn: new Map([ + [0, { id: "c0", status: "done", content: "{}" }], + [1, { id: "c1", status: "done", content: "{}" }], + ]), + model: "m", + apiKeys: {}, + sink, + }); + + expect(out.processed).toHaveLength(0); + expect(queryTabularAllColumns).not.toHaveBeenCalled(); + expect(loadRowDocumentText).not.toHaveBeenCalled(); + expect(sink.generating).not.toHaveBeenCalled(); + }); + + it("reports columns the model omitted as missing without throwing", async () => { + queryTabularAllColumns.mockImplementation( + async (_m, _f, _t, _cols, onResult) => { + await onResult(0, RESULT(0)); // only column 0 returns + }, + ); + const db = makeDb(); + const sink = sinkSpy(); + + const out = await extractRowColumns({ + db: db as never, + reviewId: "rev-1", + row: ROW, + columns: COLUMNS, + existingByColumn: new Map([ + [0, { id: "c0", status: "pending", content: null }], + [1, { id: "c1", status: "pending", content: null }], + ]), + model: "m", + apiKeys: {}, + sink, + }); + + expect(out.missing).toEqual([1]); + expect(sink.done).toHaveBeenCalledTimes(1); + // pre-existing cells → update (not insert) to mark generating + expect(db.calls.filter((c) => c.op === "insert")).toHaveLength(0); + }); +}); diff --git a/backend/src/lib/tabular/__tests__/tabular.generateStream.test.ts b/backend/src/lib/tabular/__tests__/tabular.generateStream.test.ts new file mode 100644 index 000000000..78aea1b2f --- /dev/null +++ b/backend/src/lib/tabular/__tests__/tabular.generateStream.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect } from "vitest"; + +import { targetPendingCells } from "../tabular.generateStream"; + +const COLUMNS = [ + { index: 0, name: "A", prompt: "a" }, + { index: 1, name: "B", prompt: "b" }, +]; +const ROWS = [{ id: "row-1" }, { id: "row-2" }]; + +function cellMapOf(entries: [string, Record][]) { + return new Map(entries); +} + +describe("targetPendingCells", () => { + it("treats every cell as pending when there are no cells yet", () => { + const { rowIds, pending } = targetPendingCells( + COLUMNS, + ROWS, + cellMapOf([]), + ); + expect(rowIds).toEqual(["row-1", "row-2"]); + expect([...pending].sort()).toEqual([ + "row-1:0", + "row-1:1", + "row-2:0", + "row-2:1", + ]); + }); + + it("excludes cells that are done with content, and drops fully-done rows", () => { + const { rowIds, pending } = targetPendingCells(COLUMNS, ROWS, cellMapOf([ + ["row-1:0", { status: "done", content: "{}" }], + ["row-1:1", { status: "done", content: "{}" }], + ["row-2:0", { status: "done", content: "{}" }], + // row-2:1 missing → still pending + ])); + // row-1 is fully done → not enqueued; row-2 has one outstanding column. + expect(rowIds).toEqual(["row-2"]); + expect([...pending]).toEqual(["row-2:1"]); + }); + + it("keeps a done-but-empty cell pending (content required, not just status)", () => { + const { pending } = targetPendingCells(COLUMNS, [{ id: "row-1" }], cellMapOf([ + ["row-1:0", { status: "done", content: null }], + ["row-1:1", { status: "error", content: null }], + ])); + expect([...pending].sort()).toEqual(["row-1:0", "row-1:1"]); + }); +}); diff --git a/backend/src/lib/tabular/tabular.extract.ts b/backend/src/lib/tabular/tabular.extract.ts new file mode 100644 index 000000000..a36a40292 --- /dev/null +++ b/backend/src/lib/tabular/tabular.extract.ts @@ -0,0 +1,297 @@ +// Extraction for the tabular-review module: the LLM cell-extraction helpers +// and document (PDF/DOCX/Office) text extraction. + +import { docxToPdf, normalizeDocxZipPaths } from "../convert"; +import { + isPresentationDocumentType, + isSpreadsheetDocumentType, + isWordDocumentType, +} from "../documentTypes"; +import { extractPresentationText } from "../officeText"; +import { spreadsheetToLLMText } from "../spreadsheet"; +import { + completeText, + streamChatWithTools, + type UserApiKeys, +} from "../llm"; +import { safeErrorLog } from "../safeError"; +import { loadPdfjs } from "../pdfjs"; +import { formatPromptSuffix } from "./tabular.prompt"; +import { type CellResult, type Column } from "./tabular.shared"; + +// --------------------------------------------------------------------------- +// LLM extraction helpers +// --------------------------------------------------------------------------- + +export async function queryTabularCell( + model: string, + filename: string, + documentText: string, + columnPrompt: string, + format?: string, + tags?: string[], + apiKeys?: UserApiKeys, +): Promise { + const suffix = formatPromptSuffix(format as never, tags); + const fullPrompt = `${columnPrompt}${suffix} If not found, state "Not Found". Leave all reasoning and explanation in the "reasoning" field only.`; + + const EXTRACTION_SYSTEM = `You are a legal document analyst. Return ONLY valid JSON: +{"summary": string, "flag": "green"|"grey"|"yellow"|"red", "reasoning": string} + +The "summary" and "reasoning" field values may use markdown formatting (bullets, bold, italics, etc.) — the values are still plain JSON strings (escape newlines as \\n), but the text inside will be rendered as markdown in the UI. + +The "summary" field must contain only the extracted value with inline citations — no explanation or reasoning. Every factual claim in "summary" must be followed immediately by a citation in the format [[document:SOURCE_DOCUMENT_ID||page:N||quote:exact quoted text]], using the exact source document ID shown before the supporting document. For spreadsheets, use [[document:SOURCE_DOCUMENT_ID||sheet:SHEET_NAME||cell:A1||quote:exact cell text]]. The quote must be a short verbatim excerpt (≤ 25 words) narrowly scoped to the specific claim. Do not have multiple claims share the same long quote; if two different statements need different evidence, give each its own short, precise quote. All reasoning and explanation belongs in "reasoning" only, which may also contain citations.`; + + let raw: string; + try { + raw = await completeText({ + model, + systemPrompt: EXTRACTION_SYSTEM, + user: `Document: ${filename}\n\n${documentText}\n\n---\nInstruction: ${fullPrompt}`, + maxTokens: 2048, + apiKeys, + }); + } catch (err) { + console.error("[queryTabularCell] completion failed", safeErrorLog(err)); + return null; + } + try { + const parsed = JSON.parse( + raw + .replace(/^```(?:json)?\n?/i, "") + .replace(/\n?```$/, "") + .trim(), + ) as { + summary?: unknown; + value?: unknown; + flag?: unknown; + reasoning?: unknown; + }; + return { + summary: + String(parsed.summary ?? parsed.value ?? "").trim() || + "Not addressed", + flag: (["green", "grey", "yellow", "red"] as const).includes( + parsed.flag as "green", + ) + ? (parsed.flag as "green") + : "grey", + reasoning: String(parsed.reasoning ?? ""), + }; + } catch { + return raw.trim() + ? { + summary: raw.trim().slice(0, 500), + flag: "grey" as const, + reasoning: "", + } + : null; + } +} + +export async function generateChatTitle( + model: string, + firstUserMessage: string, + context?: { reviewTitle?: string | null; projectName?: string | null }, + apiKeys?: UserApiKeys, +): Promise { + try { + const contextLines: string[] = []; + if (context?.projectName) + contextLines.push(`Project: ${context.projectName}`); + if (context?.reviewTitle) + contextLines.push(`Tabular review: ${context.reviewTitle}`); + const contextBlock = contextLines.length + ? `This chat is in the context of a tabular review.\n${contextLines.join("\n")}\n\n` + : ""; + + const raw = await completeText({ + model, + user: `${contextBlock}Generate a short title (4-6 words) for a chat that starts with the message below. The title should reflect the user's specific question, not the review or project name. Return only the title, no punctuation, no quotes:\n\n${firstUserMessage}`, + maxTokens: 64, + apiKeys, + }); + return raw.trim().slice(0, 80) || null; + } catch { + return null; + } +} + +export async function queryTabularAllColumns( + model: string, + filename: string, + documentText: string, + columns: Column[], + onResult: (columnIndex: number, result: CellResult) => Promise, + apiKeys?: UserApiKeys, +): Promise { + const columnsDesc = columns + .map((col) => { + const suffix = formatPromptSuffix(col.format as never, col.tags); + const fullPrompt = `${col.prompt}${suffix} If not found, state "Not Found".`; + return `Column ${col.index} — "${col.name}": ${fullPrompt}`; + }) + .join("\n"); + + const SYSTEM = `You are a legal document analyst. Extract information for each column listed below. + +For each column, output exactly one minified JSON object on its own line (no line breaks inside the JSON), then a newline. Process columns in order and output each result as soon as you finish it. + +Line format: +{"column_index": , "summary": , "flag": <"green"|"grey"|"yellow"|"red">, "reasoning": } + +Rules: +- "summary": the extracted value with inline citations [[document:SOURCE_DOCUMENT_ID||page:N||quote:verbatim excerpt ≤25 words]] after every factual claim, using the exact source document ID shown before the supporting document. For spreadsheets, use [[document:SOURCE_DOCUMENT_ID||sheet:SHEET_NAME||cell:A1||quote:exact cell text]]. No explanation or reasoning here. Quotes must be narrowly scoped to the specific claim — extract only the exact supporting words, not the full surrounding sentence. Do not reuse one long quote across multiple statements; give each claim its own short, precise quote. +- "flag": green = standard/favorable, yellow = needs attention, red = problematic/unfavorable, grey = neutral/not found +- "reasoning": brief explanation of the extraction +- The "summary" and "reasoning" string VALUES may use markdown (bullets, bold, italics, etc.) — escape newlines as \\n inside the JSON string. This markdown is rendered in the UI. +- Output ONLY the JSON lines themselves. Do NOT wrap the response in markdown code fences (e.g. \`\`\`json), and do not add any preamble or summary.`; + + const USER = `Document: ${filename}\n\n${documentText}\n\n---\nColumns to extract:\n${columnsDesc}`; + + let contentBuffer = ""; + const pending: Promise[] = []; + + const processLine = async (line: string) => { + const trimmed = line.trim(); + if (!trimmed) return; + try { + const parsed = JSON.parse(trimmed) as { + column_index?: unknown; + summary?: unknown; + flag?: unknown; + reasoning?: unknown; + }; + if (typeof parsed.column_index !== "number") return; + const col = columns.find((c) => c.index === parsed.column_index); + if (!col) return; + await onResult(parsed.column_index, { + summary: String(parsed.summary ?? "").trim() || "Not addressed", + flag: (["green", "grey", "yellow", "red"] as const).includes( + parsed.flag as "green", + ) + ? (parsed.flag as CellResult["flag"]) + : "grey", + reasoning: String(parsed.reasoning ?? ""), + }); + } catch { + // malformed line — skip + } + }; + + try { + await streamChatWithTools({ + model, + systemPrompt: SYSTEM, + messages: [{ role: "user", content: USER }], + tools: [], + apiKeys, + callbacks: { + onContentDelta: (delta) => { + contentBuffer += delta; + let newlineIdx: number; + while ((newlineIdx = contentBuffer.indexOf("\n")) !== -1) { + const completedLine = contentBuffer.slice( + 0, + newlineIdx, + ); + contentBuffer = contentBuffer.slice(newlineIdx + 1); + pending.push(processLine(completedLine)); + } + }, + }, + }); + } catch (err) { + console.error("[queryTabularAllColumns] stream failed", safeErrorLog(err)); + } + + if (contentBuffer.trim()) pending.push(processLine(contentBuffer)); + await Promise.all(pending); +} + +// --------------------------------------------------------------------------- +// Document text extraction +// --------------------------------------------------------------------------- + +/** + * Route a document buffer to the right text extractor for its file type: + * PDFs and DOCX extract directly; spreadsheets go through SheetJS; PPTX has a + * native XML extractor; remaining Office types take the LibreOffice → PDF + * detour. + */ +export async function extractDocumentMarkdown( + buf: ArrayBuffer, + fileType: string | null | undefined, +): Promise { + const normalizedType = (fileType ?? "").toLowerCase(); + if (normalizedType === "pdf") return extractPdfMarkdown(buf); + if (normalizedType === "docx") return extractDocxMarkdown(buf); + if (isSpreadsheetDocumentType(normalizedType)) { + // SheetJS handles .xlsx/.xlsm/.xls directly, no PDF detour. + return spreadsheetToLLMText(Buffer.from(buf)); + } + if (normalizedType === "pptx") { + return extractPresentationText(Buffer.from(buf)); + } + if ( + isPresentationDocumentType(normalizedType) || + isWordDocumentType(normalizedType) + ) { + const pdfBuf = await docxToPdf(Buffer.from(buf)); + const pdfArrayBuffer = pdfBuf.buffer.slice( + pdfBuf.byteOffset, + pdfBuf.byteOffset + pdfBuf.byteLength, + ) as ArrayBuffer; + return extractPdfMarkdown(pdfArrayBuffer); + } + return extractDocxMarkdown(buf); +} + +export async function extractPdfMarkdown(buf: ArrayBuffer): Promise { + try { + const pdfjsLib = await loadPdfjs(); + const pdf = await pdfjsLib.getDocument({ data: new Uint8Array(buf) }) + .promise; + const pages: string[] = []; + for (let i = 1; i <= pdf.numPages; i++) { + const page = await pdf.getPage(i); + const tc = await page.getTextContent(); + const text = tc.items + .filter((it): it is { str: string } => "str" in it) + .map((it) => it.str) + .join(" ") + .trim(); + if (text) pages.push(`## Page ${i}\n\n${text}`); + } + return pages.join("\n\n"); + } catch { + return ""; + } +} + +export async function extractDocxMarkdown(buf: ArrayBuffer): Promise { + try { + const mammoth = await import("mammoth"); + const normalized = await normalizeDocxZipPaths(Buffer.from(buf)); + const { value: html } = await mammoth.convertToHtml({ + buffer: normalized, + }); + return html + .replace( + /]*>(.*?)<\/h\1>/gi, + (_, l, t) => "#".repeat(Number(l)) + " " + t + "\n\n", + ) + .replace(/]*>(.*?)<\/strong>/gi, "**$1**") + .replace(/]*>(.*?)<\/li>/gi, "- $1\n") + .replace(/]*>(.*?)<\/p>/gi, "$1\n\n") + .replace(/<[^>]+>/g, "") + .replace(/ /g, " ") + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/\n{3,}/g, "\n\n") + .trim(); + } catch { + return ""; + } +} diff --git a/backend/src/lib/tabular/tabular.extractRow.ts b/backend/src/lib/tabular/tabular.extractRow.ts new file mode 100644 index 000000000..3a8488b86 --- /dev/null +++ b/backend/src/lib/tabular/tabular.extractRow.ts @@ -0,0 +1,122 @@ +// The single source of truth for extracting one row's cells. +// +// A row is the review grid's unit of work: one document, or a folder of source +// documents whose combined text is extracted together. Both entry points +// delegate here so the extraction loop lives in exactly one place: +// - the synchronous SSE route (POST /:reviewId/generate) — sink writes SSE +// frames; the caller marks any `missing` columns "error" inline. +// - the async worker (workers/extractionWorker.ts) — sink publishes over +// Redis; the caller throws on `missing` so BullMQ retries. +// +// This function owns the DB writes (mark generating, persist done) and the +// row-text loading + single multi-column LLM call. It does NOT decide the +// terminal policy for columns the model failed to return — it reports them via +// `missing` and lets each caller apply its own policy. + +import { type UserApiKeys } from "../llm"; +import { queryTabularAllColumns } from "./tabular.extract"; +import { loadRowDocumentText, type ReviewRow } from "./tabular.rows"; +import { type CellResult, type Column, type Db } from "./tabular.shared"; + +/** + * Where per-cell transitions are announced. Sync uses this to write SSE frames; + * async uses it to publish over Redis. Both `generating` and `done` mirror the + * DB writes this module has already performed. + */ +export interface CellSink { + generating(rowId: string, columnIndex: number): void | Promise; + done( + rowId: string, + columnIndex: number, + result: CellResult, + ): void | Promise; +} + +export interface ExtractRowResult { + /** Columns that were not already done and so were (re)processed. */ + processed: Column[]; + /** Columns the model returned a result for. */ + received: Set; + /** Processed columns the model did NOT return — caller decides the policy. */ + missing: number[]; +} + +/** + * Extract every not-yet-`done` column for one row. + * + * Idempotent: columns already `done` with content are skipped, so a re-run only + * touches outstanding columns. `queryTabularAllColumns` swallows its own LLM/ + * stream errors (surfacing them as unreturned columns), so this function does + * not throw on model failure — it reports `missing` instead. + */ +export async function extractRowColumns(args: { + db: Db; + reviewId: string; + row: ReviewRow; + columns: Column[]; + /** Current cell records for THIS row, keyed by column index. */ + existingByColumn: Map>; + model: string; + apiKeys: UserApiKeys; + sink: CellSink; +}): Promise { + const { db, reviewId, row, columns, existingByColumn, model, apiKeys, sink } = + args; + + const processed = columns.filter((col) => { + const cell = existingByColumn.get(col.index); + return !(cell?.status === "done" && cell?.content); + }); + if (processed.length === 0) + return { processed, received: new Set(), missing: [] }; + + // Mark each outstanding column "generating" (insert the cell if it's new) + // and announce it, so the grid shows spinners immediately. + for (const col of processed) { + const existing = existingByColumn.get(col.index); + if (existing?.id) { + await db + .from("tabular_cells") + .update({ status: "generating", content: null }) + .eq("id", existing.id); + } else { + await db.from("tabular_cells").insert({ + review_id: reviewId, + row_id: row.id, + document_id: row.document_id, + column_index: col.index, + status: "generating", + }); + } + await sink.generating(row.id, col.index); + } + + // Load the row's combined source-document text once (each section is + // prefixed with its source document id so citations can name it). + const markdown = await loadRowDocumentText(db, row); + + // One LLM call for all outstanding columns; persist + announce each result. + const received = new Set(); + await queryTabularAllColumns( + model, + row.label, + markdown, + processed, + async (columnIndex, result) => { + received.add(columnIndex); + await db + .from("tabular_cells") + .update({ content: JSON.stringify(result), status: "done" }) + .eq("review_id", reviewId) + .eq("row_id", row.id) + .eq("column_index", columnIndex); + await sink.done(row.id, columnIndex, result); + }, + apiKeys, + ); + + const missing = processed + .filter((c) => !received.has(c.index)) + .map((c) => c.index); + return { processed, received, missing }; +} diff --git a/backend/src/lib/tabular/tabular.generate.ts b/backend/src/lib/tabular/tabular.generate.ts new file mode 100644 index 000000000..ed1a11ab7 --- /dev/null +++ b/backend/src/lib/tabular/tabular.generate.ts @@ -0,0 +1,93 @@ +// Streaming prepare guard for the tabular-review generate stream. +// +// STREAMING: the SSE endpoint (POST /:reviewId/generate) keeps its streaming +// loop, abort handling, and per-cell persistence in the route. Only the +// NON-streaming work lives here — the pre-stream "prepare" guard (access +// checks, row loading, missing-API-key checks) that returns the data the +// route then streams over. + +import { type UserApiKeys } from "../llm"; +import { getUserModelSettings } from "../userSettings"; +import { + ensureReviewAccess, + filterAccessibleDocumentIds, +} from "../access"; +import { loadReviewRows, type ReviewRow } from "./tabular.rows"; +import { + missingModelApiKey, + type Column, + type Db, + type MissingApiKey, +} from "./tabular.shared"; + +// --------------------------------------------------------------------------- +// Streaming prepare guards (non-streaming work before the SSE loop) +// --------------------------------------------------------------------------- + +export type PreparedGenerate = { + columns: Column[]; + /** Existing cells keyed `${row_id}:${column_index}`. */ + cellMap: Map>; + /** The review's rows, restricted to rows whose sources are all accessible. */ + rows: ReviewRow[]; + tabular_model: string; + api_keys: UserApiKeys; +}; + +export async function prepareTabularGenerate( + db: Db, + args: { reviewId: string; userId: string; userEmail: string | undefined }, +): Promise< + | { ok: true; data: PreparedGenerate } + | { ok: false; kind: "not_found" } + | { ok: false; kind: "no_columns" } + | { ok: false; kind: "cells_error"; message: string } + | { ok: false; kind: "missing_api_key"; missingKey: MissingApiKey } +> { + const { reviewId, userId, userEmail } = args; + + const { data: review, error: reviewError } = await db + .from("tabular_reviews") + .select("*") + .eq("id", reviewId) + .single(); + if (reviewError || !review) return { ok: false, kind: "not_found" }; + const access = await ensureReviewAccess(review, userId, userEmail, db); + if (!access.ok) return { ok: false, kind: "not_found" }; + + const columns: Column[] = review.columns_config ?? []; + if (columns.length === 0) return { ok: false, kind: "no_columns" }; + + let rows = await loadReviewRows(db, reviewId); + + const { data: cells, error: cellsError } = await db + .from("tabular_cells") + .select("*") + .eq("review_id", reviewId); + if (cellsError) + return { ok: false, kind: "cells_error", message: cellsError.message }; + const cellMap = new Map>(); + for (const cell of cells ?? []) + cellMap.set(`${cell.row_id}:${cell.column_index}`, cell); + + // A row is only extractable if the requester can access every source + // document feeding it; drop rows containing anything they cannot see. + const sourceIds = [ + ...new Set(rows.flatMap((row) => row.source_document_ids ?? [])), + ]; + const allowedSourceIds = new Set( + await filterAccessibleDocumentIds(sourceIds, userId, userEmail, db), + ); + rows = rows.filter((row) => + (row.source_document_ids ?? []).every((id) => allowedSourceIds.has(id)), + ); + + const { tabular_model, api_keys } = await getUserModelSettings(userId, db); + const missingKey = missingModelApiKey(tabular_model, api_keys); + if (missingKey) return { ok: false, kind: "missing_api_key", missingKey }; + + return { + ok: true, + data: { columns, cellMap, rows, tabular_model, api_keys }, + }; +} diff --git a/backend/src/lib/tabular/tabular.generateStream.ts b/backend/src/lib/tabular/tabular.generateStream.ts new file mode 100644 index 000000000..9f90d6441 --- /dev/null +++ b/backend/src/lib/tabular/tabular.generateStream.ts @@ -0,0 +1,275 @@ +// Async + reconnectable variants of the tabular generate stream. +// +// Extraction is handed to durable BullMQ jobs (one per row) that retry and +// survive a client disconnect or server restart. The HTTP request becomes a +// *view* over that work: it subscribes to the review's Redis progress channel +// and forwards each cell update as the same `cell_update` SSE frame the +// synchronous path emits, with a DB-poll backstop so a dropped pub/sub message +// can never leave the stream hung. +// +// Two entry points share the `tailTabularRun` core: +// - streamTabularGenerateAsync — POST /:reviewId/generate: enqueues the work, +// then tails it. +// - streamTabularRunView — GET /:reviewId/generate/stream: tails an already- +// running (or already-finished) run without enqueuing, so a client that +// dropped can reconnect and catch up. + +import IORedis from "ioredis"; +import type { Response } from "express"; +import { REDIS_URL } from "../queue/connection"; +import { startSseHeartbeat } from "../sseHeartbeat"; +import { enqueueExtraction } from "../queue/extractionQueue"; +import { + runProgressChannel, + type CellUpdate, +} from "../queue/runProgress"; +import { safeErrorLog } from "../safeError"; +import { parseCellContent, type Column, type Db, type Log } from "./tabular.shared"; +import type { PreparedGenerate } from "./tabular.generate"; + +/** How often the DB-poll backstop reconciles cell state (ms). */ +const RECONCILE_INTERVAL_MS = 3_000; +/** Hard ceiling on a single stream so a vanished job can't hold it open forever. */ +const STREAM_MAX_MS = 15 * 60 * 1000; + +const cellKey = (rowId: string, columnIndex: number) => + `${rowId}:${columnIndex}`; + +/** + * Given the review's columns, its rows, and current cell state, compute the + * set of cells that still need extracting and the rows that own at least one + * of them. Pure and side-effect free so it can be unit-tested. + */ +export function targetPendingCells( + columns: Column[], + rows: { id: string }[], + cellMap: Map>, +): { rowIds: string[]; pending: Set } { + const pending = new Set(); + const rowIds: string[] = []; + for (const row of rows) { + const rowId = row.id; + let hasPending = false; + for (const col of columns) { + const cell = cellMap.get(`${rowId}:${col.index}`); + if (!(cell?.status === "done" && cell?.content)) { + pending.add(cellKey(rowId, col.index)); + hasPending = true; + } + } + if (hasPending) rowIds.push(rowId); + } + return { rowIds, pending }; +} + +/** + * The shared streaming core: open the SSE response, subscribe to the review's + * progress channel, run `afterSubscribe` (POST enqueues here; GET does not), + * then forward cell updates — resolving each pending cell on a terminal status — + * until every targeted cell is terminal, the client disconnects, or the cap + * elapses. A DB-poll backstop reconciles missed messages. + */ +async function tailTabularRun(args: { + res: Response; + db: Db; + reviewId: string; + log: Log; + pending: Set; + afterSubscribe?: () => Promise; +}): Promise { + const { res, db, reviewId, log, pending, afterSubscribe } = args; + + res.setHeader("Content-Type", "text/event-stream"); + res.setHeader("Cache-Control", "no-cache"); + res.setHeader("Connection", "keep-alive"); + res.setHeader("X-Accel-Buffering", "no"); + res.flushHeaders(); + + const stopHeartbeat = startSseHeartbeat(res); + const write = (payload: unknown) => { + try { + if (!res.writableEnded) + res.write(`data: ${JSON.stringify(payload)}\n\n`); + } catch { + // Client gone; the "close" handler will tear the stream down. + } + }; + + let sub: IORedis | null = null; + let poll: ReturnType | null = null; + let cap: ReturnType | null = null; + let finished = false; + + const cleanup = () => { + stopHeartbeat(); + if (poll) clearInterval(poll); + if (cap) clearTimeout(cap); + if (sub) void sub.quit().catch(() => {}); + sub = null; + }; + // End the SSE response (client saw [DONE]). Any enqueued jobs keep running + // regardless — this only closes the *view*. + const finish = () => { + if (finished) return; + finished = true; + try { + if (!res.writableEnded) res.write("data: [DONE]\n\n"); + } catch { + /* client already gone */ + } + cleanup(); + if (!res.writableEnded) res.end(); + }; + // Client disconnected first: stop tailing but do NOT end (already closed), + // and leave any workers running so the extraction still completes. + const abandon = () => { + if (finished) return; + finished = true; + cleanup(); + }; + + // Terminal update for a pending cell: forward it and drop it from the set. + const resolve = (key: string, update: CellUpdate) => { + if (!pending.delete(key)) return; + write(update); + if (pending.size === 0) finish(); + }; + const onUpdate = (update: CellUpdate) => { + const key = cellKey(update.row_id, update.column_index); + if (update.status === "generating") { + if (pending.has(key)) write(update); // spinner feedback; still pending + return; + } + resolve(key, update); // "done" | "error" + }; + + res.on("close", abandon); + + // Nothing to do — every targeted cell is already done. + if (pending.size === 0) return void finish(); + + // Subscribe BEFORE enqueuing so a fast worker can't publish into the void. + try { + sub = new IORedis(REDIS_URL, { maxRetriesPerRequest: null }); + await sub.subscribe(runProgressChannel(reviewId)); + sub.on("message", (_channel, message) => { + try { + onUpdate(JSON.parse(message) as CellUpdate); + } catch { + /* ignore malformed frame */ + } + }); + } catch (err) { + log.error( + "[tabular/generate-async] subscribe failed", + { err: safeErrorLog(err), reviewId }, + ); + } + + if (afterSubscribe) await afterSubscribe(); + + // Backstop: reconcile against the DB in case a pub/sub frame was missed (or, + // for a reconnecting view, to replay progress that happened while away). + poll = setInterval(() => { + if (finished) return; + void (async () => { + const { data: cells } = await db + .from("tabular_cells") + .select("row_id, column_index, status, content") + .eq("review_id", reviewId); + for (const c of (cells ?? []) as { + row_id: string; + column_index: number; + status: string; + content: unknown; + }[]) { + const key = cellKey(c.row_id, c.column_index); + if (!pending.has(key)) continue; + if (c.status === "done" && c.content) { + resolve(key, { + type: "cell_update", + row_id: c.row_id, + column_index: c.column_index, + content: parseCellContent(c.content), + status: "done", + }); + } else if (c.status === "error") { + resolve(key, { + type: "cell_update", + row_id: c.row_id, + column_index: c.column_index, + content: null, + status: "error", + }); + } + } + })().catch((err) => + log.error( + "[tabular/generate-async] reconcile poll failed", + { err: safeErrorLog(err), reviewId }, + ), + ); + }, RECONCILE_INTERVAL_MS); + if (typeof poll.unref === "function") poll.unref(); + + cap = setTimeout(finish, STREAM_MAX_MS); + if (typeof cap.unref === "function") cap.unref(); +} + +/** POST /:reviewId/generate — enqueue the outstanding work, then tail it. */ +export async function streamTabularGenerateAsync(args: { + res: Response; + db: Db; + reviewId: string; + userId: string; + prepared: PreparedGenerate; + log: Log; +}): Promise { + const { res, db, reviewId, userId, prepared, log } = args; + const { rowIds, pending } = targetPendingCells( + prepared.columns, + prepared.rows, + prepared.cellMap, + ); + + await tailTabularRun({ + res, + db, + reviewId, + log, + pending, + afterSubscribe: async () => { + for (const rowId of rowIds) { + try { + await enqueueExtraction({ reviewId, userId, rowId }); + } catch (err) { + log.error( + "[tabular/generate-async] enqueue failed", + { err: safeErrorLog(err), reviewId, rowId }, + ); + } + } + }, + }); +} + +/** + * GET /:reviewId/generate/stream — reconnect to an in-flight (or finished) run + * without re-triggering work. Pure observer: it tails progress and catches up + * from the DB, so a client that dropped mid-run can resume. + */ +export async function streamTabularRunView(args: { + res: Response; + db: Db; + reviewId: string; + prepared: PreparedGenerate; + log: Log; +}): Promise { + const { res, db, reviewId, prepared, log } = args; + const { pending } = targetPendingCells( + prepared.columns, + prepared.rows, + prepared.cellMap, + ); + await tailTabularRun({ res, db, reviewId, log, pending }); +} diff --git a/backend/src/lib/tabular/tabular.prompt.ts b/backend/src/lib/tabular/tabular.prompt.ts new file mode 100644 index 000000000..65f459e5b --- /dev/null +++ b/backend/src/lib/tabular/tabular.prompt.ts @@ -0,0 +1,31 @@ +// Prompt construction for the tabular-review extraction: per-format prompt +// suffixes appended to each column's instruction. + +// --------------------------------------------------------------------------- +// Prompt formatting +// --------------------------------------------------------------------------- + +export function formatPromptSuffix(format?: string, tags?: string[]): string { + switch (format) { + case "bulleted_list": + return ' The "summary" field in your JSON response must be a markdown bulleted list only — no prose. Format: each item on its own line, prefixed with "* " (asterisk + single space), e.g.\n* First item\n* Second item\n* Third item'; + case "number": + return ' The "summary" field in your JSON response must be a single number only. No units or explanation.'; + case "percentage": + return ' The "summary" field in your JSON response must be a single percentage value only (e.g. 42%). No explanation.'; + case "monetary_amount": + return ' The "summary" field in your JSON response must be the monetary value only, including currency symbol (e.g. $1,234.56). No explanation.'; + case "currency": + return ' The "summary" field in your JSON response must contain only the currency code(s). Wrap each code in double square brackets, e.g. [[USD]] or [[EUR]]. No other text.'; + case "yes_no": + return ' The "summary" field in your JSON response must be [[Yes]] or [[No]] only. The "reasoning" field MUST include an inline citation [[document:SOURCE_DOCUMENT_ID||page:N||quote:verbatim excerpt ≤25 words]] pointing to the exact language in the document that supports the Yes/No answer.'; + case "date": + return ' The "summary" field in your JSON response must be the date only in DD Month YYYY format (e.g. 1 January 2024). If a range, give both dates separated by an em dash. The "reasoning" field MUST include an inline citation [[document:SOURCE_DOCUMENT_ID||page:N||quote:verbatim excerpt ≤25 words]] pointing to the exact place in the document where the date is found.'; + case "tag": + return tags?.length + ? ` The \"summary\" field in your JSON response must contain exactly one tag wrapped in double square brackets. Available tags: ${tags.map((t) => `[[${t}]]`).join(", ")}. No other text. The \"reasoning\" field MUST include an inline citation [[document:SOURCE_DOCUMENT_ID||page:N||quote:verbatim excerpt ≤25 words]] pointing to the exact language in the document that supports the chosen tag.` + : ""; + default: + return ""; + } +} diff --git a/backend/src/lib/tabular/tabular.rows.ts b/backend/src/lib/tabular/tabular.rows.ts new file mode 100644 index 000000000..608a69c20 --- /dev/null +++ b/backend/src/lib/tabular/tabular.rows.ts @@ -0,0 +1,142 @@ +// Row loading for the tabular-review module. +// +// A review's grid is made of ROWS (tabular_review_rows): a row is either one +// document or a folder grouping several source documents. These helpers load +// the rows with their source-document ids resolved, and build the combined +// text a row's extraction runs over. Moved out of routes/tabular.ts so the +// synchronous SSE route and the async extraction worker share one copy. + +import { downloadFile } from "../storage"; +import { attachActiveVersionPaths } from "../documentVersions"; +import { safeErrorLog } from "../safeError"; +import { extractDocumentMarkdown } from "./tabular.extract"; +import { type Db } from "./tabular.shared"; + +export type ReviewRow = { + id: string; + review_id: string; + label: string; + row_type: "document" | "folder"; + folder_id: string | null; + library_folder_id: string | null; + document_id: string | null; + sort_index: number; + source_document_ids?: string[]; +}; + +export type SourceDocument = { + id: string; + filename: string; + file_type: string | null; + current_version_id?: string | null; + project_id?: string | null; + folder_id?: string | null; + library_folder_id?: string | null; +}; + +export async function fetchSourceDocuments( + db: Db, + documentIds: string[], +): Promise { + if (documentIds.length === 0) return []; + const { data, error } = await db + .from("documents") + .select( + "id, current_version_id, project_id, folder_id, library_folder_id", + ) + .in("id", documentIds); + if (error) throw new Error(error.message); + const docs = (data ?? []) as (Omit< + SourceDocument, + "filename" | "file_type" + > & { + filename?: string | null; + file_type?: string | null; + })[]; + await attachActiveVersionPaths(db, docs); + const position = new Map(documentIds.map((id, index) => [id, index])); + return docs + .map((doc) => ({ + ...doc, + filename: doc.filename?.trim() || "Untitled document", + file_type: doc.file_type ?? null, + })) + .sort((a, b) => (position.get(a.id) ?? 0) - (position.get(b.id) ?? 0)); +} + +export async function loadReviewRows( + db: Db, + reviewId: string, +): Promise { + const { data, error } = await db + .from("tabular_review_rows") + .select("*") + .eq("review_id", reviewId) + .order("sort_index", { ascending: true }); + if (error) throw new Error(error.message); + const rows = (data ?? []) as ReviewRow[]; + if (!rows.length) return rows; + const { data: sources, error: sourceError } = await db + .from("tabular_review_row_sources") + .select("row_id, document_id") + .in("row_id", rows.map((row) => row.id)) + .order("sort_index", { ascending: true }); + if (sourceError) throw new Error(sourceError.message); + const byRow = new Map(); + for (const source of sources ?? []) { + byRow.set(source.row_id, [ + ...(byRow.get(source.row_id) ?? []), + source.document_id, + ]); + } + return rows.map((row) => ({ + ...row, + source_document_ids: + byRow.get(row.id) ?? (row.document_id ? [row.document_id] : []), + })); +} + +/** Load one row of a review (with its source ids resolved), or null. */ +export async function loadReviewRow( + db: Db, + reviewId: string, + rowId: string, +): Promise { + const rows = await loadReviewRows(db, reviewId); + return rows.find((row) => row.id === rowId) ?? null; +} + +export async function loadRowDocumentText( + db: Db, + row: ReviewRow, +): Promise { + const sourceIds = + row.source_document_ids ?? (row.document_id ? [row.document_id] : []); + const docs = await fetchSourceDocuments(db, sourceIds); + const sections: string[] = []; + for (const doc of docs) { + const storagePath = (doc as SourceDocument & { storage_path?: string }) + .storage_path; + let markdown = ""; + if (storagePath) { + const buf = await downloadFile(storagePath); + if (buf) { + try { + markdown = await extractDocumentMarkdown( + buf, + doc.file_type, + ); + } catch (error) { + console.error( + `[tabular] extraction error doc=${doc.id}`, + safeErrorLog(error), + ); + } + } + } + sections.push( + `## Source document: ${doc.filename}\nSource document ID: ${doc.id}\n\n${markdown}`, + ); + } + return sections.join("\n\n---\n\n"); +} diff --git a/backend/src/lib/tabular/tabular.shared.ts b/backend/src/lib/tabular/tabular.shared.ts new file mode 100644 index 000000000..0be370754 --- /dev/null +++ b/backend/src/lib/tabular/tabular.shared.ts @@ -0,0 +1,108 @@ +// Shared types + helpers used across the tabular extraction files. +// +// These are module-internal: they are exported here so sibling files +// (tabular.prompt.ts, tabular.extract.ts, …) and routes/tabular.ts can +// import them. + +import { createServerSupabase } from "../supabase"; +import { providerForModel, type Provider, type UserApiKeys } from "../llm"; + +export type Db = ReturnType; + +// Structural logging slice — service functions only ever .error(). +export type Log = Pick; + +// --------------------------------------------------------------------------- +// Model helpers +// --------------------------------------------------------------------------- + +function providerLabel(provider: Provider): string { + if (provider === "claude") return "Anthropic"; + if (provider === "openai") return "OpenAI"; + return "Gemini"; +} + +export type MissingApiKey = { + provider: Provider; + model: string; + detail: string; +}; + +export function missingModelApiKey( + model: string, + apiKeys: UserApiKeys, +): MissingApiKey | null { + const provider = providerForModel(model); + if (provider === "ollama") return null; // local, no key + if (apiKeys[provider]?.trim()) return null; + return { + provider, + model, + detail: `${providerLabel(provider)} API key is required to use ${model}. Add an API key or select a different tabular review model.`, + }; +} + +// --------------------------------------------------------------------------- +// Cell content parsing +// --------------------------------------------------------------------------- + +export function parseCellContent( + raw: unknown, +): { summary: string; flag?: string; reasoning?: string } | null { + if (!raw) return null; + if (typeof raw === "object" && raw !== null && "summary" in raw) { + const c = raw as { + summary?: unknown; + flag?: unknown; + reasoning?: unknown; + }; + return { + summary: String(c.summary ?? ""), + flag: (["green", "grey", "yellow", "red"] as const).includes( + c.flag as "green", + ) + ? (c.flag as string) + : undefined, + reasoning: typeof c.reasoning === "string" ? c.reasoning : "", + }; + } + if (typeof raw === "string") { + try { + const p = JSON.parse(raw) as { + summary?: unknown; + value?: unknown; + flag?: unknown; + reasoning?: unknown; + }; + return { + summary: String(p.summary ?? p.value ?? "").trim(), + flag: (["green", "grey", "yellow", "red"] as const).includes( + p.flag as "green", + ) + ? (p.flag as string) + : undefined, + reasoning: typeof p.reasoning === "string" ? p.reasoning : "", + }; + } catch { + return { summary: raw, flag: "grey", reasoning: "" }; + } + } + return null; +} + +// --------------------------------------------------------------------------- +// Extraction result / column shapes +// --------------------------------------------------------------------------- + +export type CellResult = { + summary: string; + flag: "green" | "grey" | "yellow" | "red"; + reasoning: string; +}; +export type Column = { + index: number; + name: string; + prompt: string; + format?: string; + tags?: string[]; +}; diff --git a/backend/src/routes/documents.ts b/backend/src/routes/documents.ts index e890d193a..44fe203b4 100644 --- a/backend/src/routes/documents.ts +++ b/backend/src/routes/documents.ts @@ -11,6 +11,7 @@ import { versionStorageKey, } from "../lib/storage"; import { docxToPdf, convertedPdfKey } from "../lib/convert"; +import { enqueueConversion } from "../lib/queue/conversionQueue"; import { extractTrackedChangeIds, resolveTrackedChange, @@ -1380,9 +1381,15 @@ export async function handleDocumentUpload( ) as ArrayBuffer; const pageCount = suffix === "pdf" ? await countPdfPages(rawBuf) : null; + // When the job queue is enabled, defer Office → PDF conversion to the + // BullMQ worker instead of blocking the upload request on LibreOffice. + const deferConversion = + shouldConvertToPdf(suffix) && + process.env.ASYNC_DOCUMENT_CONVERSION === "true"; + // Convert Office files → PDF for display. PDFs are their own rendition. let pdfStoragePath: string | null = null; - if (shouldConvertToPdf(suffix)) { + if (!deferConversion && shouldConvertToPdf(suffix)) { try { const pdfBuf = await docxToPdf(content); const pdfKey = convertedPdfKey(userId, docId); @@ -1434,11 +1441,23 @@ export async function handleDocumentUpload( .from("documents") .update({ current_version_id: versionRow.id, - status: "ready", + // Deferred conversion leaves the doc "processing" until the worker + // produces the PDF and flips it to "ready". + status: deferConversion ? "processing" : "ready", updated_at: new Date().toISOString(), }) .eq("id", docId); + if (deferConversion) { + await enqueueConversion({ + documentId: docId, + versionId: versionRow.id, + userId, + storagePath: key, + fileType: suffix, + }); + } + const { data: updated } = await db .from("documents") .select("*") diff --git a/backend/src/routes/tabular.ts b/backend/src/routes/tabular.ts index 642edbbd9..db2a8db9a 100644 --- a/backend/src/routes/tabular.ts +++ b/backend/src/routes/tabular.ts @@ -1,16 +1,7 @@ import { Router } from "express"; import { requireAuth } from "../middleware/auth"; import { createServerSupabase } from "../lib/supabase"; -import { downloadFile } from "../lib/storage"; import { attachActiveVersionPaths } from "../lib/documentVersions"; -import { docxToPdf, normalizeDocxZipPaths } from "../lib/convert"; -import { - isPresentationDocumentType, - isSpreadsheetDocumentType, - isWordDocumentType, -} from "../lib/documentTypes"; -import { extractPresentationText } from "../lib/officeText"; -import { spreadsheetToLLMText } from "../lib/spreadsheet"; import { AssistantStreamError, buildCancelledAssistantMessage, @@ -21,13 +12,30 @@ import { type ChatMessage, type TabularCellStore, } from "../lib/chat"; +import { completeText } from "../lib/llm"; +import { + extractDocumentMarkdown, + generateChatTitle, + queryTabularCell, +} from "../lib/tabular/tabular.extract"; +import { + missingModelApiKey, + parseCellContent, + type Column, +} from "../lib/tabular/tabular.shared"; +import { extractRowColumns } from "../lib/tabular/tabular.extractRow"; +import { prepareTabularGenerate } from "../lib/tabular/tabular.generate"; +import { + streamTabularGenerateAsync, + streamTabularRunView, +} from "../lib/tabular/tabular.generateStream"; import { - completeText, - providerForModel, - streamChatWithTools, - type Provider, - type UserApiKeys, -} from "../lib/llm"; + fetchSourceDocuments, + loadReviewRows, + loadRowDocumentText, + type ReviewRow, + type SourceDocument, +} from "../lib/tabular/tabular.rows"; import { getUserModelSettings } from "../lib/userSettings"; import { checkProjectAccess, @@ -48,90 +56,15 @@ import { parseTabularReviewScope, } from "../lib/tabularReviewsOverview"; -function formatPromptSuffix(format?: string, tags?: string[]): string { - switch (format) { - case "bulleted_list": - return ' The "summary" field in your JSON response must be a markdown bulleted list only — no prose. Format: each item on its own line, prefixed with "* " (asterisk + single space), e.g.\n* First item\n* Second item\n* Third item'; - case "number": - return ' The "summary" field in your JSON response must be a single number only. No units or explanation.'; - case "percentage": - return ' The "summary" field in your JSON response must be a single percentage value only (e.g. 42%). No explanation.'; - case "monetary_amount": - return ' The "summary" field in your JSON response must be the monetary value only, including currency symbol (e.g. $1,234.56). No explanation.'; - case "currency": - return ' The "summary" field in your JSON response must contain only the currency code(s). Wrap each code in double square brackets, e.g. [[USD]] or [[EUR]]. No other text.'; - case "yes_no": - return ' The "summary" field in your JSON response must be [[Yes]] or [[No]] only. The "reasoning" field MUST include an inline citation [[document:SOURCE_DOCUMENT_ID||page:N||quote:verbatim excerpt ≤25 words]] pointing to the exact language in the document that supports the Yes/No answer.'; - case "date": - return ' The "summary" field in your JSON response must be the date only in DD Month YYYY format (e.g. 1 January 2024). If a range, give both dates separated by an em dash. The "reasoning" field MUST include an inline citation [[document:SOURCE_DOCUMENT_ID||page:N||quote:verbatim excerpt ≤25 words]] pointing to the exact place in the document where the date is found.'; - case "tag": - return tags?.length - ? ` The \"summary\" field in your JSON response must contain exactly one tag wrapped in double square brackets. Available tags: ${tags.map((t) => `[[${t}]]`).join(", ")}. No other text. The \"reasoning\" field MUST include an inline citation [[document:SOURCE_DOCUMENT_ID||page:N||quote:verbatim excerpt ≤25 words]] pointing to the exact language in the document that supports the chosen tag.` - : ""; - default: - return ""; - } -} - export const tabularRouter = Router(); type DocumentGrouping = "document" | "folder"; -type ReviewRow = { - id: string; - review_id: string; - label: string; - row_type: "document" | "folder"; - folder_id: string | null; - library_folder_id: string | null; - document_id: string | null; - sort_index: number; - source_document_ids?: string[]; -}; -type SourceDocument = { - id: string; - filename: string; - file_type: string | null; - current_version_id?: string | null; - project_id?: string | null; - folder_id?: string | null; - library_folder_id?: string | null; -}; type SupabaseDb = ReturnType; function normalizeGrouping(value: unknown): DocumentGrouping { return value === "folder" ? "folder" : "document"; } -async function fetchSourceDocuments( - db: SupabaseDb, - documentIds: string[], -): Promise { - if (documentIds.length === 0) return []; - const { data, error } = await db - .from("documents") - .select( - "id, current_version_id, project_id, folder_id, library_folder_id", - ) - .in("id", documentIds); - if (error) throw new Error(error.message); - const docs = (data ?? []) as (Omit< - SourceDocument, - "filename" | "file_type" - > & { - filename?: string | null; - file_type?: string | null; - })[]; - await attachActiveVersionPaths(db, docs); - const position = new Map(documentIds.map((id, index) => [id, index])); - return docs - .map((doc) => ({ - ...doc, - filename: doc.filename?.trim() || "Untitled document", - file_type: doc.file_type ?? null, - })) - .sort((a, b) => (position.get(a.id) ?? 0) - (position.get(b.id) ?? 0)); -} - function buildFolderPathMap( folders: { id: string; @@ -399,90 +332,6 @@ async function syncCellsForReviewRows( } } -async function loadReviewRows( - db: SupabaseDb, - reviewId: string, -): Promise { - const { data, error } = await db - .from("tabular_review_rows") - .select("*") - .eq("review_id", reviewId) - .order("sort_index", { ascending: true }); - if (error) throw new Error(error.message); - const rows = (data ?? []) as ReviewRow[]; - if (!rows.length) return rows; - const { data: sources, error: sourceError } = await db - .from("tabular_review_row_sources") - .select("row_id, document_id") - .in("row_id", rows.map((row) => row.id)) - .order("sort_index", { ascending: true }); - if (sourceError) throw new Error(sourceError.message); - const byRow = new Map(); - for (const source of sources ?? []) { - byRow.set(source.row_id, [ - ...(byRow.get(source.row_id) ?? []), - source.document_id, - ]); - } - return rows.map((row) => ({ - ...row, - source_document_ids: - byRow.get(row.id) ?? (row.document_id ? [row.document_id] : []), - })); -} - -async function loadRowDocumentText( - db: SupabaseDb, - row: ReviewRow, -): Promise { - const sourceIds = - row.source_document_ids ?? (row.document_id ? [row.document_id] : []); - const docs = await fetchSourceDocuments(db, sourceIds); - const sections: string[] = []; - for (const doc of docs) { - const storagePath = (doc as SourceDocument & { storage_path?: string }) - .storage_path; - let markdown = ""; - if (storagePath) { - const buf = await downloadFile(storagePath); - if (buf) { - try { - markdown = await extractDocumentMarkdown( - buf, - doc.file_type, - ); - } catch (error) { - console.error( - `[tabular] extraction error doc=${doc.id}`, - safeErrorLog(error), - ); - } - } - } - sections.push( - `## Source document: ${doc.filename}\nSource document ID: ${doc.id}\n\n${markdown}`, - ); - } - return sections.join("\n\n---\n\n"); -} - -function providerLabel(provider: Provider): string { - if (provider === "claude") return "Anthropic"; - if (provider === "openai") return "OpenAI"; - return "Gemini"; -} - -function missingModelApiKey(model: string, apiKeys: UserApiKeys) { - const provider = providerForModel(model); - if (provider === "ollama") return null; // local, no key - if (apiKeys[provider]?.trim()) return null; - return { - provider, - model, - detail: `${providerLabel(provider)} API key is required to use ${model}. Add an API key or select a different tabular review model.`, - }; -} - // GET /tabular-review tabularRouter.get("/", requireAuth, async (req, res) => { const userId = res.locals.userId as string; @@ -1151,58 +1000,44 @@ tabularRouter.post("/:reviewId/generate", requireAuth, async (req, res) => { const { reviewId } = req.params; const db = createServerSupabase(); - const { data: review, error: reviewError } = await db - .from("tabular_reviews") - .select("*") - .eq("id", reviewId) - .single(); - if (reviewError || !review) - return void res.status(404).json({ detail: "Review not found" }); - const access = await ensureReviewAccess(review, userId, userEmail, db); - if (!access.ok) - return void res.status(404).json({ detail: "Review not found" }); - - const columns: { - index: number; - name: string; - prompt: string; - format?: string; - tags?: string[]; - }[] = review.columns_config ?? []; - if (columns.length === 0) - return void res.status(400).json({ detail: "No columns configured" }); - - let rows = await loadReviewRows(db, reviewId); - - const { data: cells, error: cellsError } = await db - .from("tabular_cells") - .select("*") - .eq("review_id", reviewId); - if (cellsError) - return void res.status(500).json({ detail: cellsError.message }); - const cellMap = new Map>(); - for (const cell of cells ?? []) - cellMap.set(`${cell.row_id}:${cell.column_index}`, cell); - - const sourceIds = [ - ...new Set(rows.flatMap((row) => row.source_document_ids ?? [])), - ]; - const allowedSourceIds = new Set( - await filterAccessibleDocumentIds(sourceIds, userId, userEmail, db), - ); - rows = rows.filter((row) => - (row.source_document_ids ?? []).every((id) => allowedSourceIds.has(id)), - ); - - const { tabular_model, api_keys } = await getUserModelSettings(userId, db); - const missingKey = missingModelApiKey(tabular_model, api_keys); - if (missingKey) { + const prepared = await prepareTabularGenerate(db, { + reviewId, + userId, + userEmail, + }); + if (!prepared.ok) { + if (prepared.kind === "not_found") + return void res.status(404).json({ detail: "Review not found" }); + if (prepared.kind === "no_columns") + return void res + .status(400) + .json({ detail: "No columns configured" }); + if (prepared.kind === "cells_error") + return void res.status(500).json({ detail: prepared.message }); return void res.status(422).json({ code: "missing_api_key", - ...missingKey, + ...prepared.missingKey, + }); + } + + // Async path: hand extraction to the durable BullMQ queue and turn this + // request into a reconnectable view that tails progress. The work survives + // a disconnect and retries on failure. Falls through to the historical + // inline path when the flag is off (no Redis required). + if (process.env.ASYNC_TABULAR_EXTRACTION === "true") { + await streamTabularGenerateAsync({ + res, + db, + reviewId, + userId, + prepared: prepared.data, + log: console, }); + return; } + const { columns, cellMap, rows, tabular_model, api_keys } = prepared.data; + res.setHeader("Content-Type", "text/event-stream"); res.setHeader("Cache-Control", "no-cache"); res.setHeader("Connection", "keep-alive"); @@ -1211,88 +1046,57 @@ tabularRouter.post("/:reviewId/generate", requireAuth, async (req, res) => { const write = (line: string) => res.write(line); + const cellFrame = ( + rowId: string, + columnIndex: number, + content: unknown, + status: "generating" | "done" | "error", + ): void => { + write( + `data: ${JSON.stringify({ type: "cell_update", row_id: rowId, column_index: columnIndex, content, status })}\n\n`, + ); + }; + try { await Promise.all( rows.map(async (row) => { - const markdown = await loadRowDocumentText( - db, - row, - ); - - // Filter to only columns that need processing - const columnsToProcess = columns.filter((col) => { + const existingByColumn = new Map< + number, + Record + >(); + for (const col of columns) { const cell = cellMap.get(`${row.id}:${col.index}`); - return !(cell?.status === "done" && cell?.content); - }); - if (columnsToProcess.length === 0) return; - - // Mark all as generating upfront - for (const col of columnsToProcess) { - write( - `data: ${JSON.stringify({ type: "cell_update", row_id: row.id, column_index: col.index, content: null, status: "generating" })}\n\n`, - ); - const existingCell = cellMap.get(`${row.id}:${col.index}`); - if (existingCell) { - await db - .from("tabular_cells") - .update({ status: "generating", content: null }) - .eq("id", existingCell.id); - } else { - await db.from("tabular_cells").insert({ - review_id: reviewId, - row_id: row.id, - document_id: row.document_id, - column_index: col.index, - status: "generating", - }); - } + if (cell) existingByColumn.set(col.index, cell); } - // Single LLM call for all columns, streaming one JSON line per column - const receivedColumns = new Set(); - try { - await queryTabularAllColumns( - tabular_model, - row.label, - markdown, - columnsToProcess, - async (columnIndex, result) => { - receivedColumns.add(columnIndex); - await db - .from("tabular_cells") - .update({ - content: JSON.stringify(result), - status: "done", - }) - .eq("review_id", reviewId) - .eq("row_id", row.id) - .eq("column_index", columnIndex); - write( - `data: ${JSON.stringify({ type: "cell_update", row_id: row.id, column_index: columnIndex, content: result, status: "done" })}\n\n`, - ); - }, - api_keys, - ); - } catch (err) { - console.error( - `[tabular/generate] queryTabularAllColumns error row=${row.id}`, - safeErrorLog(err), - ); - } + // Shared extraction core (identical to the async worker); the + // sink writes SSE frames. Columns the model omits come back in + // `missing` — the synchronous path marks them "error" inline + // (the async path retries them instead). + const { missing } = await extractRowColumns({ + db, + reviewId, + row, + columns, + existingByColumn, + model: tabular_model, + apiKeys: api_keys, + sink: { + generating: (rowId, ci) => + cellFrame(rowId, ci, null, "generating"), + done: (rowId, ci, result) => + cellFrame(rowId, ci, result, "done"), + }, + }); - // Mark any columns the LLM didn't return as error - for (const col of columnsToProcess) { - if (!receivedColumns.has(col.index)) { - await db - .from("tabular_cells") - .update({ status: "error" }) - .eq("review_id", reviewId) - .eq("row_id", row.id) - .eq("column_index", col.index); - write( - `data: ${JSON.stringify({ type: "cell_update", row_id: row.id, column_index: col.index, content: null, status: "error" })}\n\n`, - ); - } + for (const columnIndex of missing) { + await db + .from("tabular_cells") + .update({ status: "error" }) + .eq("review_id", reviewId) + .eq("row_id", row.id) + .eq("column_index", columnIndex); + cellFrame(row.id, columnIndex, null, "error"); } }), ); @@ -1305,13 +1109,61 @@ tabularRouter.post("/:reviewId/generate", requireAuth, async (req, res) => { `data: ${JSON.stringify({ type: "error", message: safeErrorMessage(err, "Stream error") })}\n\ndata: [DONE]\n\n`, ); } catch { - /* ignore */ + // Best-effort error notification: if the client has already + // disconnected the SSE write throws. We are in the error path with + // nothing left to do, so swallow and let `finally` end the stream. } } finally { res.end(); } }); +// GET /tabular-review/:reviewId/generate/stream — reconnect to an in-flight (or +// just-finished) generate run without re-triggering work. A client whose POST +// /generate stream dropped can resume here and catch up on the remaining cells. +// Pure observer: it never enqueues. (Registered before the /:reviewId/chats +// group; no path collision since the segments differ.) +tabularRouter.get( + "/:reviewId/generate/stream", + requireAuth, + async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { reviewId } = req.params; + const db = createServerSupabase(); + + const prepared = await prepareTabularGenerate(db, { + reviewId, + userId, + userEmail, + }); + if (!prepared.ok) { + if (prepared.kind === "not_found") + return void res + .status(404) + .json({ detail: "Review not found" }); + if (prepared.kind === "no_columns") + return void res + .status(400) + .json({ detail: "No columns configured" }); + if (prepared.kind === "cells_error") + return void res.status(500).json({ detail: prepared.message }); + return void res.status(422).json({ + code: "missing_api_key", + ...prepared.missingKey, + }); + } + + await streamTabularRunView({ + res, + db, + reviewId, + prepared: prepared.data, + log: console, + }); + }, +); + // GET /tabular-review/:reviewId/chats — list chats (metadata only, no messages) tabularRouter.get("/:reviewId/chats", requireAuth, async (req, res) => { const userId = res.locals.userId as string; @@ -1795,336 +1647,3 @@ tabularRouter.post("/:reviewId/chat", requireAuth, async (req, res) => { } }); -function parseCellContent( - raw: unknown, -): { summary: string; flag?: string; reasoning?: string } | null { - if (!raw) return null; - if (typeof raw === "object" && raw !== null && "summary" in raw) { - const c = raw as { - summary?: unknown; - flag?: unknown; - reasoning?: unknown; - }; - return { - summary: String(c.summary ?? ""), - flag: (["green", "grey", "yellow", "red"] as const).includes( - c.flag as "green", - ) - ? (c.flag as string) - : undefined, - reasoning: typeof c.reasoning === "string" ? c.reasoning : "", - }; - } - if (typeof raw === "string") { - try { - const p = JSON.parse(raw) as { - summary?: unknown; - value?: unknown; - flag?: unknown; - reasoning?: unknown; - }; - return { - summary: String(p.summary ?? p.value ?? "").trim(), - flag: (["green", "grey", "yellow", "red"] as const).includes( - p.flag as "green", - ) - ? (p.flag as string) - : undefined, - reasoning: typeof p.reasoning === "string" ? p.reasoning : "", - }; - } catch { - return { summary: raw, flag: "grey", reasoning: "" }; - } - } - return null; -} - -async function queryTabularCell( - model: string, - filename: string, - documentText: string, - columnPrompt: string, - format?: string, - tags?: string[], - apiKeys?: import("../lib/llm").UserApiKeys, -) { - const suffix = formatPromptSuffix(format as never, tags); - const fullPrompt = `${columnPrompt}${suffix} If not found, state "Not Found". Leave all reasoning and explanation in the "reasoning" field only.`; - - const EXTRACTION_SYSTEM = `You are a legal document analyst. Return ONLY valid JSON: -{"summary": string, "flag": "green"|"grey"|"yellow"|"red", "reasoning": string} - -The "summary" and "reasoning" field values may use markdown formatting (bullets, bold, italics, etc.) — the values are still plain JSON strings (escape newlines as \\n), but the text inside will be rendered as markdown in the UI. - -The "summary" field must contain only the extracted value with inline citations — no explanation or reasoning. Every factual claim in "summary" must be followed immediately by a citation in the format [[document:SOURCE_DOCUMENT_ID||page:N||quote:exact quoted text]], using the exact source document ID shown before the supporting document. For spreadsheets, use [[document:SOURCE_DOCUMENT_ID||sheet:SHEET_NAME||cell:A1||quote:exact cell text]]. The quote must be a short verbatim excerpt (≤ 25 words) narrowly scoped to the specific claim. Do not have multiple claims share the same long quote; if two different statements need different evidence, give each its own short, precise quote. All reasoning and explanation belongs in "reasoning" only, which may also contain citations.`; - - let raw: string; - try { - raw = await completeText({ - model, - systemPrompt: EXTRACTION_SYSTEM, - user: `Document: ${filename}\n\n${documentText}\n\n---\nInstruction: ${fullPrompt}`, - maxTokens: 2048, - apiKeys, - }); - } catch (err) { - console.error("[queryTabularCell] completion failed", safeErrorLog(err)); - return null; - } - try { - const parsed = JSON.parse( - raw - .replace(/^```(?:json)?\n?/i, "") - .replace(/\n?```$/, "") - .trim(), - ) as { - summary?: unknown; - value?: unknown; - flag?: unknown; - reasoning?: unknown; - }; - return { - summary: - String(parsed.summary ?? parsed.value ?? "").trim() || - "Not addressed", - flag: (["green", "grey", "yellow", "red"] as const).includes( - parsed.flag as "green", - ) - ? (parsed.flag as "green") - : "grey", - reasoning: String(parsed.reasoning ?? ""), - }; - } catch { - return raw.trim() - ? { - summary: raw.trim().slice(0, 500), - flag: "grey" as const, - reasoning: "", - } - : null; - } -} - -async function generateChatTitle( - model: string, - firstUserMessage: string, - context?: { reviewTitle?: string | null; projectName?: string | null }, - apiKeys?: import("../lib/llm").UserApiKeys, -): Promise { - try { - const contextLines: string[] = []; - if (context?.projectName) - contextLines.push(`Project: ${context.projectName}`); - if (context?.reviewTitle) - contextLines.push(`Tabular review: ${context.reviewTitle}`); - const contextBlock = contextLines.length - ? `This chat is in the context of a tabular review.\n${contextLines.join("\n")}\n\n` - : ""; - - const raw = await completeText({ - model, - user: `${contextBlock}Generate a short title (4-6 words) for a chat that starts with the message below. The title should reflect the user's specific question, not the review or project name. Return only the title, no punctuation, no quotes:\n\n${firstUserMessage}`, - maxTokens: 64, - apiKeys, - }); - return raw.trim().slice(0, 80) || null; - } catch { - return null; - } -} - -type CellResult = { - summary: string; - flag: "green" | "grey" | "yellow" | "red"; - reasoning: string; -}; -type Column = { - index: number; - name: string; - prompt: string; - format?: string; - tags?: string[]; -}; - -async function queryTabularAllColumns( - model: string, - filename: string, - documentText: string, - columns: Column[], - onResult: (columnIndex: number, result: CellResult) => Promise, - apiKeys?: import("../lib/llm").UserApiKeys, -): Promise { - const columnsDesc = columns - .map((col) => { - const suffix = formatPromptSuffix(col.format as never, col.tags); - const fullPrompt = `${col.prompt}${suffix} If not found, state "Not Found".`; - return `Column ${col.index} — "${col.name}": ${fullPrompt}`; - }) - .join("\n"); - - const SYSTEM = `You are a legal document analyst. Extract information for each column listed below. - -For each column, output exactly one minified JSON object on its own line (no line breaks inside the JSON), then a newline. Process columns in order and output each result as soon as you finish it. - -Line format: -{"column_index": , "summary": , "flag": <"green"|"grey"|"yellow"|"red">, "reasoning": } - -Rules: -- "summary": the extracted value with inline citations [[document:SOURCE_DOCUMENT_ID||page:N||quote:verbatim excerpt ≤25 words]] after every factual claim, using the exact source document ID shown before the supporting document. For spreadsheets, use [[document:SOURCE_DOCUMENT_ID||sheet:SHEET_NAME||cell:A1||quote:exact cell text]]. No explanation or reasoning here. Quotes must be narrowly scoped to the specific claim — extract only the exact supporting words, not the full surrounding sentence. Do not reuse one long quote across multiple statements; give each claim its own short, precise quote. -- "flag": green = standard/favorable, yellow = needs attention, red = problematic/unfavorable, grey = neutral/not found -- "reasoning": brief explanation of the extraction -- The "summary" and "reasoning" string VALUES may use markdown (bullets, bold, italics, etc.) — escape newlines as \\n inside the JSON string. This markdown is rendered in the UI. -- Output ONLY the JSON lines themselves. Do NOT wrap the response in markdown code fences (e.g. \`\`\`json), and do not add any preamble or summary.`; - - const USER = `Document: ${filename}\n\n${documentText}\n\n---\nColumns to extract:\n${columnsDesc}`; - - let contentBuffer = ""; - const pending: Promise[] = []; - - const processLine = async (line: string) => { - const trimmed = line.trim(); - if (!trimmed) return; - try { - const parsed = JSON.parse(trimmed) as { - column_index?: unknown; - summary?: unknown; - flag?: unknown; - reasoning?: unknown; - }; - if (typeof parsed.column_index !== "number") return; - const col = columns.find((c) => c.index === parsed.column_index); - if (!col) return; - await onResult(parsed.column_index, { - summary: String(parsed.summary ?? "").trim() || "Not addressed", - flag: (["green", "grey", "yellow", "red"] as const).includes( - parsed.flag as "green", - ) - ? (parsed.flag as CellResult["flag"]) - : "grey", - reasoning: String(parsed.reasoning ?? ""), - }); - } catch { - // malformed line — skip - } - }; - - try { - await streamChatWithTools({ - model, - systemPrompt: SYSTEM, - messages: [{ role: "user", content: USER }], - tools: [], - apiKeys, - callbacks: { - onContentDelta: (delta) => { - contentBuffer += delta; - let newlineIdx: number; - while ((newlineIdx = contentBuffer.indexOf("\n")) !== -1) { - const completedLine = contentBuffer.slice( - 0, - newlineIdx, - ); - contentBuffer = contentBuffer.slice(newlineIdx + 1); - pending.push(processLine(completedLine)); - } - }, - }, - }); - } catch (err) { - console.error("[queryTabularAllColumns] stream failed", safeErrorLog(err)); - } - - if (contentBuffer.trim()) pending.push(processLine(contentBuffer)); - await Promise.all(pending); -} - -async function extractDocumentMarkdown( - buf: ArrayBuffer, - fileType: string | null | undefined, -): Promise { - const normalizedType = (fileType ?? "").toLowerCase(); - if (normalizedType === "pdf") return extractPdfMarkdown(buf); - if (normalizedType === "docx") return extractDocxMarkdown(buf); - if (isSpreadsheetDocumentType(normalizedType)) { - // SheetJS handles .xlsx/.xlsm/.xls directly, no PDF detour. - return spreadsheetToLLMText(Buffer.from(buf)); - } - if (normalizedType === "pptx") { - return extractPresentationText(Buffer.from(buf)); - } - if ( - isPresentationDocumentType(normalizedType) || - isWordDocumentType(normalizedType) - ) { - const pdfBuf = await docxToPdf(Buffer.from(buf)); - const pdfArrayBuffer = pdfBuf.buffer.slice( - pdfBuf.byteOffset, - pdfBuf.byteOffset + pdfBuf.byteLength, - ) as ArrayBuffer; - return extractPdfMarkdown(pdfArrayBuffer); - } - return extractDocxMarkdown(buf); -} - -async function extractPdfMarkdown(buf: ArrayBuffer): Promise { - try { - const pdfjsLib = await import( - "pdfjs-dist/legacy/build/pdf.mjs" as string - ); - const pdf = await ( - pdfjsLib as unknown as { - getDocument: (opts: unknown) => { - promise: Promise<{ - numPages: number; - getPage: (n: number) => Promise<{ - getTextContent: () => Promise<{ - items: { str?: string; hasEOL?: boolean }[]; - }>; - }>; - }>; - }; - } - ).getDocument({ data: new Uint8Array(buf) }).promise; - const pages: string[] = []; - for (let i = 1; i <= pdf.numPages; i++) { - const page = await pdf.getPage(i); - const tc = await page.getTextContent(); - const text = tc.items - .filter((it): it is { str: string } => "str" in it) - .map((it) => it.str) - .join(" ") - .trim(); - if (text) pages.push(`## Page ${i}\n\n${text}`); - } - return pages.join("\n\n"); - } catch { - return ""; - } -} - -async function extractDocxMarkdown(buf: ArrayBuffer): Promise { - try { - const mammoth = await import("mammoth"); - const normalized = await normalizeDocxZipPaths(Buffer.from(buf)); - const { value: html } = await mammoth.convertToHtml({ - buffer: normalized, - }); - return html - .replace( - /]*>(.*?)<\/h\1>/gi, - (_, l, t) => "#".repeat(Number(l)) + " " + t + "\n\n", - ) - .replace(/]*>(.*?)<\/strong>/gi, "**$1**") - .replace(/]*>(.*?)<\/li>/gi, "- $1\n") - .replace(/]*>(.*?)<\/p>/gi, "$1\n\n") - .replace(/<[^>]+>/g, "") - .replace(/ /g, " ") - .replace(/&/g, "&") - .replace(/</g, "<") - .replace(/>/g, ">") - .replace(/\n{3,}/g, "\n\n") - .trim(); - } catch { - return ""; - } -} diff --git a/backend/src/workers/__tests__/conversionWorker.test.ts b/backend/src/workers/__tests__/conversionWorker.test.ts new file mode 100644 index 000000000..a75ca9345 --- /dev/null +++ b/backend/src/workers/__tests__/conversionWorker.test.ts @@ -0,0 +1,146 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +// Never construct a real Supabase client during the unit test. +vi.mock("../../lib/supabase", () => ({ + createServerSupabase: vi.fn(), +})); + +const downloadFile = vi.fn(); +const uploadFile = vi.fn(); +vi.mock("../../lib/storage", () => ({ + downloadFile: (...a: unknown[]) => downloadFile(...a), + uploadFile: (...a: unknown[]) => uploadFile(...a), +})); + +const docxToPdf = vi.fn(); +vi.mock("../../lib/convert", () => ({ + docxToPdf: (...a: unknown[]) => docxToPdf(...a), + convertedPdfKey: (userId: string, docId: string) => + `converted-pdfs/${userId}/${docId}.pdf`, +})); + +import { + runConversionJob, + setDocumentTerminalStatus, + isPermanentFailure, +} from "../conversionWorker"; +import type { Job } from "bullmq"; +import type { ConversionJobData } from "../../lib/queue/conversionQueue"; + +type Call = { table: string; update: Record }; + +function makeDb() { + const calls: Call[] = []; + return { + calls, + from(table: string) { + return { + update(update: Record) { + return { + eq: async () => { + calls.push({ table, update }); + return {}; + }, + }; + }, + }; + }, + }; +} + +const JOB = { + documentId: "doc-1", + versionId: "ver-1", + userId: "user-1", + storagePath: "uploads/user-1/doc-1.docx", + fileType: "docx", +}; + +beforeEach(() => { + downloadFile.mockReset(); + uploadFile.mockReset(); + docxToPdf.mockReset(); +}); + +describe("runConversionJob", () => { + it("converts, stores the PDF, and flips the document to ready", async () => { + downloadFile.mockResolvedValue(new ArrayBuffer(8)); + docxToPdf.mockResolvedValue(Buffer.from("%PDF-1.7 fake")); + uploadFile.mockResolvedValue(undefined); + const db = makeDb(); + + await runConversionJob(JOB, db as never); + + expect(uploadFile).toHaveBeenCalledWith( + "converted-pdfs/user-1/doc-1.pdf", + expect.anything(), + "application/pdf", + ); + expect(db.calls).toContainEqual({ + table: "document_versions", + update: { pdf_storage_path: "converted-pdfs/user-1/doc-1.pdf" }, + }); + const docUpdate = db.calls.find((c) => c.table === "documents"); + expect(docUpdate?.update.status).toBe("ready"); + }); + + it("treats a conversion failure as non-fatal: still marks ready, no PDF stored", async () => { + downloadFile.mockResolvedValue(new ArrayBuffer(8)); + docxToPdf.mockRejectedValue(new Error("soffice exploded")); + const db = makeDb(); + + await runConversionJob(JOB, db as never); + + expect(uploadFile).not.toHaveBeenCalled(); + expect(db.calls.some((c) => c.table === "document_versions")).toBe(false); + const docUpdate = db.calls.find((c) => c.table === "documents"); + expect(docUpdate?.update.status).toBe("ready"); + }); + + it("throws when the original is missing so BullMQ retries", async () => { + downloadFile.mockResolvedValue(null); + const db = makeDb(); + + await expect(runConversionJob(JOB, db as never)).rejects.toThrow( + /original not found/, + ); + expect(docxToPdf).not.toHaveBeenCalled(); + expect(db.calls).toHaveLength(0); + }); +}); + +describe("setDocumentTerminalStatus", () => { + it("updates the document to the given terminal status", async () => { + const db = makeDb(); + + await setDocumentTerminalStatus(db as never, "doc-1", "error"); + + expect(db.calls).toHaveLength(1); + expect(db.calls[0].table).toBe("documents"); + expect(db.calls[0].update.status).toBe("error"); + expect(db.calls[0].update).toHaveProperty("updated_at"); + }); +}); + +describe("isPermanentFailure", () => { + const job = (attemptsMade: number, attempts?: number) => + ({ + attemptsMade, + opts: { attempts }, + }) as unknown as Job; + + it("is false while retries remain", () => { + expect(isPermanentFailure(job(1, 3))).toBe(false); + expect(isPermanentFailure(job(2, 3))).toBe(false); + }); + + it("is true once retries are exhausted", () => { + expect(isPermanentFailure(job(3, 3))).toBe(true); + expect(isPermanentFailure(job(4, 3))).toBe(true); + }); + + it("defaults to a single attempt when opts.attempts is unset", () => { + expect(isPermanentFailure(job(1))).toBe(true); + expect(isPermanentFailure(job(0))).toBe(false); + }); +}); diff --git a/backend/src/workers/__tests__/extractionWorker.test.ts b/backend/src/workers/__tests__/extractionWorker.test.ts new file mode 100644 index 000000000..5c52c5ebe --- /dev/null +++ b/backend/src/workers/__tests__/extractionWorker.test.ts @@ -0,0 +1,298 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +vi.mock("../../lib/supabase", () => ({ + createServerSupabase: vi.fn(), +})); + +const loadReviewRow = vi.fn(); +const loadRowDocumentText = vi.fn(); +vi.mock("../../lib/tabular/tabular.rows", () => ({ + loadReviewRow: (...a: unknown[]) => loadReviewRow(...a), + loadRowDocumentText: (...a: unknown[]) => loadRowDocumentText(...a), +})); + +vi.mock("../../lib/userSettings", () => ({ + getUserModelSettings: async () => ({ + tabular_model: "claude-test", + api_keys: {}, + }), +})); + +const queryTabularAllColumns = vi.fn(); +vi.mock("../../lib/tabular/tabular.extract", () => ({ + queryTabularAllColumns: (...a: unknown[]) => queryTabularAllColumns(...a), +})); + +import { + runExtractionJob, + markExtractionFailed, + isPermanentFailure, +} from "../extractionWorker"; +import type { Job } from "bullmq"; +import type { ExtractionJobData } from "../../lib/queue/extractionQueue"; + +type Call = { + table: string; + op: "select" | "update" | "insert"; + payload?: Record; + filters: Record; +}; + +// Minimal chainable Supabase test double. `responses[table].select` feeds +// select/single reads; update/insert resolve empty and are recorded in `calls`. +function makeDb(responses: Record) { + const calls: Call[] = []; + function from(table: string) { + const state: Call = { table, op: "select", filters: {} }; + const resolveRead = () => + (responses[table]?.select as { data: unknown }) ?? { data: null }; + const b: Record = { + select() { + state.op = "select"; + return b; + }, + update(payload: Record) { + state.op = "update"; + state.payload = payload; + return b; + }, + insert(payload: Record) { + state.op = "insert"; + state.payload = payload; + calls.push({ ...state, filters: { ...state.filters } }); + return Promise.resolve({ data: null, error: null }); + }, + eq(col: string, val: unknown) { + state.filters[col] = val; + return b; + }, + single() { + calls.push({ ...state, filters: { ...state.filters } }); + return Promise.resolve(resolveRead()); + }, + then(onF: (v: unknown) => unknown, onR?: (e: unknown) => unknown) { + calls.push({ ...state, filters: { ...state.filters } }); + const value = + state.op === "select" + ? resolveRead() + : { data: null, error: null }; + return Promise.resolve(value).then(onF, onR); + }, + }; + return b; + } + return { calls, from }; +} + +const DATA: ExtractionJobData = { + reviewId: "rev-1", + userId: "user-1", + rowId: "row-1", +}; + +const ROW = { + id: "row-1", + review_id: "rev-1", + label: "Contract.pdf", + row_type: "document", + folder_id: null, + library_folder_id: null, + document_id: "doc-1", + sort_index: 0, + source_document_ids: ["doc-1"], +}; + +const COLUMNS = [ + { index: 0, name: "Parties", prompt: "Who are the parties?" }, + { index: 1, name: "Term", prompt: "What is the term?" }, +]; + +const CELL = (index: number, result: Record) => ({ + summary: `col ${index}`, + flag: "green", + reasoning: "", + ...result, +}); + +beforeEach(() => { + loadReviewRow.mockReset(); + loadReviewRow.mockResolvedValue(ROW); + loadRowDocumentText.mockReset(); + loadRowDocumentText.mockResolvedValue("extracted text"); + queryTabularAllColumns.mockReset(); +}); + +describe("runExtractionJob", () => { + it("marks every column generating then done and publishes each", async () => { + const publish = vi.fn(async () => {}); + const db = makeDb({ + tabular_reviews: { select: { data: { columns_config: COLUMNS } } }, + tabular_cells: { select: { data: [] } }, // no cells yet + }); + queryTabularAllColumns.mockImplementation( + async (_m, _f, _t, cols, onResult) => { + for (const c of cols) await onResult(c.index, CELL(c.index, {})); + }, + ); + + await runExtractionJob(DATA, { db: db as never, publish }); + + // Two "generating" inserts (no pre-existing cells) + two "done" updates. + const inserts = db.calls.filter((c) => c.op === "insert"); + expect(inserts).toHaveLength(2); + expect(inserts[0].payload).toMatchObject({ + review_id: "rev-1", + row_id: "row-1", + document_id: "doc-1", + }); + const doneUpdates = db.calls.filter( + (c) => c.op === "update" && c.payload?.status === "done", + ); + expect(doneUpdates).toHaveLength(2); + + const frames = publish.mock.calls.map( + (c) => c[1] as { status: string; row_id: string }, + ); + expect(frames.every((f) => f.row_id === "row-1")).toBe(true); + const statuses = frames.map((f) => f.status); + expect(statuses.filter((s) => s === "generating")).toHaveLength(2); + expect(statuses.filter((s) => s === "done")).toHaveLength(2); + }); + + it("reuses existing cell records (update, not insert) when they already exist", async () => { + const publish = vi.fn(async () => {}); + const db = makeDb({ + tabular_reviews: { select: { data: { columns_config: COLUMNS } } }, + tabular_cells: { + select: { + data: [ + { id: "c0", column_index: 0, status: "error", content: null }, + { id: "c1", column_index: 1, status: "pending", content: null }, + ], + }, + }, + }); + queryTabularAllColumns.mockImplementation( + async (_m, _f, _t, cols, onResult) => { + for (const c of cols) await onResult(c.index, CELL(c.index, {})); + }, + ); + + await runExtractionJob(DATA, { db: db as never, publish }); + + expect(db.calls.filter((c) => c.op === "insert")).toHaveLength(0); + const generatingUpdates = db.calls.filter( + (c) => c.op === "update" && c.payload?.status === "generating", + ); + expect(generatingUpdates).toHaveLength(2); + }); + + it("skips columns already done with content — no LLM call", async () => { + const publish = vi.fn(async () => {}); + const db = makeDb({ + tabular_reviews: { select: { data: { columns_config: COLUMNS } } }, + tabular_cells: { + select: { + data: [ + { id: "c0", column_index: 0, status: "done", content: "{}" }, + { id: "c1", column_index: 1, status: "done", content: "{}" }, + ], + }, + }, + }); + + await runExtractionJob(DATA, { db: db as never, publish }); + + expect(queryTabularAllColumns).not.toHaveBeenCalled(); + expect(publish).not.toHaveBeenCalled(); + }); + + it("throws when the model omits a column so BullMQ retries", async () => { + const publish = vi.fn(async () => {}); + const db = makeDb({ + tabular_reviews: { select: { data: { columns_config: COLUMNS } } }, + tabular_cells: { select: { data: [] } }, + }); + // Only column 0 comes back. + queryTabularAllColumns.mockImplementation( + async (_m, _f, _t, _cols, onResult) => { + await onResult(0, CELL(0, {})); + }, + ); + + await expect( + runExtractionJob(DATA, { db: db as never, publish }), + ).rejects.toThrow(/incomplete extraction/); + }); + + it("returns early when the review has no columns", async () => { + const publish = vi.fn(async () => {}); + const db = makeDb({ + tabular_reviews: { select: { data: { columns_config: [] } } }, + }); + + await runExtractionJob(DATA, { db: db as never, publish }); + + expect(loadReviewRow).not.toHaveBeenCalled(); + expect(queryTabularAllColumns).not.toHaveBeenCalled(); + expect(db.calls.some((c) => c.table === "tabular_cells")).toBe(false); + }); + + it("returns early when the row no longer exists (deleted between enqueue and run)", async () => { + const publish = vi.fn(async () => {}); + loadReviewRow.mockResolvedValue(null); + const db = makeDb({ + tabular_reviews: { select: { data: { columns_config: COLUMNS } } }, + }); + + await runExtractionJob(DATA, { db: db as never, publish }); + + expect(queryTabularAllColumns).not.toHaveBeenCalled(); + expect(db.calls.some((c) => c.table === "tabular_cells")).toBe(false); + }); +}); + +describe("markExtractionFailed", () => { + it("marks only unfinished cells error and publishes them", async () => { + const publish = vi.fn(async () => {}); + const db = makeDb({ + tabular_cells: { + select: { + data: [ + { id: "c0", column_index: 0, status: "generating", content: null }, + { id: "c1", column_index: 1, status: "done", content: "{}" }, + ], + }, + }, + }); + + await markExtractionFailed(DATA, { db: db as never, publish }); + + const errorUpdates = db.calls.filter( + (c) => c.op === "update" && c.payload?.status === "error", + ); + expect(errorUpdates).toHaveLength(1); + expect(errorUpdates[0].filters.id).toBe("c0"); + expect(publish).toHaveBeenCalledTimes(1); + const frame = publish.mock.calls[0][1] as { + row_id: string; + column_index: number; + }; + expect(frame.row_id).toBe("row-1"); + expect(frame.column_index).toBe(0); + }); +}); + +describe("isPermanentFailure", () => { + const job = (attemptsMade: number, attempts?: number) => + ({ attemptsMade, opts: { attempts } }) as unknown as Job; + + it("is false while retries remain", () => { + expect(isPermanentFailure(job(1, 3))).toBe(false); + expect(isPermanentFailure(job(2, 3))).toBe(false); + }); + + it("is true once retries are exhausted", () => { + expect(isPermanentFailure(job(3, 3))).toBe(true); + }); +}); diff --git a/backend/src/workers/conversionWorker.ts b/backend/src/workers/conversionWorker.ts new file mode 100644 index 000000000..0f9dfefac --- /dev/null +++ b/backend/src/workers/conversionWorker.ts @@ -0,0 +1,153 @@ +import { Worker, type Job } from "bullmq"; +import { getRedisConnection } from "../lib/queue/connection"; +import { + CONVERSION_QUEUE, + type ConversionJobData, +} from "../lib/queue/conversionQueue"; +import { downloadFile, uploadFile } from "../lib/storage"; +import { docxToPdf, convertedPdfKey } from "../lib/convert"; +import { createServerSupabase } from "../lib/supabase"; + +type Db = ReturnType; + +/** + * Convert one uploaded DOCX/DOC to PDF and finalize the document. + * + * Extracted from the worker callback so it can be unit-tested with injected + * deps. Mirrors the synchronous upload path's semantics: a *conversion* + * failure is non-fatal — the document is still usable (just without a PDF + * rendition), so we still flip it to "ready". Only failure to fetch the + * original is thrown, so BullMQ retries it. + */ +export async function runConversionJob( + data: ConversionJobData, + db: Db = createServerSupabase(), +): Promise { + const { documentId, versionId, userId, storagePath } = data; + + const original = await downloadFile(storagePath); + if (!original) { + // Transient (eventual-consistency) or a real miss — let BullMQ retry. + throw new Error( + `[conversion-worker] original not found at ${storagePath}`, + ); + } + + try { + const pdfBuf = await docxToPdf(Buffer.from(original)); + const pdfKey = convertedPdfKey(userId, documentId); + await uploadFile( + pdfKey, + pdfBuf.buffer.slice( + pdfBuf.byteOffset, + pdfBuf.byteOffset + pdfBuf.byteLength, + ) as ArrayBuffer, + "application/pdf", + ); + await db + .from("document_versions") + .update({ pdf_storage_path: pdfKey }) + .eq("id", versionId); + await db + .from("documents") + .update({ status: "ready", updated_at: new Date().toISOString() }) + .eq("id", documentId); + console.log("[conversion-worker] converted", { documentId, versionId }); + } catch (err) { + console.error( + "[conversion-worker] DOCX→PDF failed; finalizing without a PDF rendition", + { err, documentId, versionId }, + ); + await db + .from("documents") + .update({ status: "ready", updated_at: new Date().toISOString() }) + .eq("id", documentId); + } +} + +/** + * Move a document to a terminal status (e.g. "error"). Extracted so the + * permanent-failure path is unit-testable without a live queue/Redis. + */ +export async function setDocumentTerminalStatus( + db: Db, + documentId: string, + status: string, +): Promise { + await db + .from("documents") + .update({ status, updated_at: new Date().toISOString() }) + .eq("id", documentId); +} + +/** True once a job has exhausted its retries (BullMQ 'failed', no attempts left). */ +export function isPermanentFailure(job: Job): boolean { + const maxAttempts = job.opts.attempts ?? 1; + return job.attemptsMade >= maxAttempts; +} + +let worker: Worker | null = null; + +export function createConversionWorker(): Worker { + if (worker) return worker; + worker = new Worker( + CONVERSION_QUEUE, + async (job: Job) => { + await runConversionJob(job.data); + }, + { + connection: getRedisConnection(), + concurrency: 2, + // Recover jobs orphaned by a worker crash mid-run: re-queue a job + // whose lock hasn't been renewed within stalledInterval, up to + // maxStalledCount times before it's failed for good. + stalledInterval: 30_000, + maxStalledCount: 2, + }, + ); + worker.on("stalled", (jobId) => { + console.warn( + "[conversion-worker] job stalled; will be re-queued", + { jobId }, + ); + }); + worker.on("failed", async (job, err) => { + if (!job) { + console.error("[conversion-worker] job failed (no job)", { err }); + return; + } + if (!isPermanentFailure(job)) { + console.error( + "[conversion-worker] job failed (will retry, attempts remain)", + { jobId: job.id, err }, + ); + return; + } + // Retries exhausted: the document is stuck "processing" with no PDF and + // no path forward — surface it to the user as a terminal "error". + console.error( + "[conversion-worker] job permanently failed; marking document error", + { jobId: job.id, documentId: job.data.documentId, err }, + ); + try { + await setDocumentTerminalStatus( + createServerSupabase(), + job.data.documentId, + "error", + ); + } catch (updateErr) { + console.error( + "[conversion-worker] failed to mark document error", + { jobId: job.id, documentId: job.data.documentId, updateErr }, + ); + } + }); + return worker; +} + +export async function stopConversionWorker(): Promise { + if (worker) { + await worker.close(); + worker = null; + } +} diff --git a/backend/src/workers/extractionWorker.ts b/backend/src/workers/extractionWorker.ts new file mode 100644 index 000000000..bb091414a --- /dev/null +++ b/backend/src/workers/extractionWorker.ts @@ -0,0 +1,223 @@ +import { Worker, type Job } from "bullmq"; +import { getRedisConnection } from "../lib/queue/connection"; +import { + EXTRACTION_QUEUE, + type ExtractionJobData, +} from "../lib/queue/extractionQueue"; +import { + publishCellUpdate as defaultPublish, + type CellUpdate, +} from "../lib/queue/runProgress"; +import { getUserModelSettings } from "../lib/userSettings"; +import { extractRowColumns } from "../lib/tabular/tabular.extractRow"; +import { loadReviewRow } from "../lib/tabular/tabular.rows"; +import type { Column } from "../lib/tabular/tabular.shared"; +import { createServerSupabase } from "../lib/supabase"; + +type Db = ReturnType; + +export interface ExtractionDeps { + db: Db; + /** Publish a progress frame (injectable so the job is unit-testable). */ + publish: (reviewId: string, update: CellUpdate) => Promise; +} + +function defaultDeps(): ExtractionDeps { + return { db: createServerSupabase(), publish: defaultPublish }; +} + +/** + * Extract every not-yet-`done` column for one (review, row) pair. + * + * This is the async counterpart of the inline loop that used to live in the + * POST /:reviewId/generate handler — pulled into a standalone, dependency- + * injected function so it can run on a worker and be unit-tested without a live + * queue/Redis. + * + * Idempotent + retry-safe: it re-reads current cell state and only processes + * columns that are not already `done` with content. A retry therefore narrows + * to the columns still outstanding. If any targeted column fails to come back + * from the model, the function THROWS so BullMQ retries the job; the permanent- + * failure handler (below) is what finally marks stragglers `error`. + */ +export async function runExtractionJob( + data: ExtractionJobData, + deps: ExtractionDeps = defaultDeps(), +): Promise { + const { reviewId, userId, rowId } = data; + const { db, publish } = deps; + + // 1. Columns configured on the review. + const { data: review } = await db + .from("tabular_reviews") + .select("columns_config") + .eq("id", reviewId) + .single(); + const columns: Column[] = (review?.columns_config as Column[]) ?? []; + if (columns.length === 0) return; + + // 2. The row this job fills (with its source-document ids resolved). A row + // deleted between enqueue and run is not an error — nothing to do. + const row = await loadReviewRow(db, reviewId, rowId); + if (!row) return; + + // 3. Current cell state for this row, keyed by column. + const { data: cells } = await db + .from("tabular_cells") + .select("*") + .eq("review_id", reviewId) + .eq("row_id", rowId); + const existingByColumn = new Map>(); + for (const cell of (cells ?? []) as Record[]) + existingByColumn.set(cell.column_index as number, cell); + + // 4. Model + keys for the owner (never serialized into the job payload). + const { tabular_model, api_keys } = await getUserModelSettings(userId, db); + + // 5. Run the shared extraction core; publish transitions over Redis so a + // tailing /generate request sees them live. + const { processed, missing } = await extractRowColumns({ + db, + reviewId, + row, + columns, + existingByColumn, + model: tabular_model, + apiKeys: api_keys, + sink: { + generating: (id, columnIndex) => + publish(reviewId, { + type: "cell_update", + row_id: id, + column_index: columnIndex, + content: null, + status: "generating", + }), + done: (id, columnIndex, result) => + publish(reviewId, { + type: "cell_update", + row_id: id, + column_index: columnIndex, + content: result, + status: "done", + }), + }, + }); + if (processed.length === 0) return; + + // 6. If the model didn't return every column, throw so BullMQ retries the + // still-outstanding ones. Cells are left "generating" — the permanent- + // failure handler flips the survivors to "error" once retries run out. + if (missing.length > 0) { + throw new Error( + `[extraction-worker] incomplete extraction for row ${rowId}: ` + + `missing columns ${missing.join(", ")}`, + ); + } +} + +/** True once a job has exhausted its retries (BullMQ 'failed', no attempts left). */ +export function isPermanentFailure(job: Job): boolean { + const maxAttempts = job.opts.attempts ?? 1; + return job.attemptsMade >= maxAttempts; +} + +/** + * Terminal cleanup for a permanently failed job: flip every still-unfinished + * cell for this row to "error" and announce it, so the grid shows a clear + * terminal state instead of a spinner that never resolves. Extracted so it is + * unit-testable without a live queue. + */ +export async function markExtractionFailed( + data: ExtractionJobData, + deps: ExtractionDeps = defaultDeps(), +): Promise { + const { reviewId, rowId } = data; + const { db, publish } = deps; + + const { data: cells } = await db + .from("tabular_cells") + .select("id, column_index, status, content") + .eq("review_id", reviewId) + .eq("row_id", rowId); + + for (const cell of (cells ?? []) as Record[]) { + if (cell.status === "done" && cell.content) continue; + await db + .from("tabular_cells") + .update({ status: "error" }) + .eq("id", cell.id); + await publish(reviewId, { + type: "cell_update", + row_id: rowId, + column_index: cell.column_index as number, + content: null, + status: "error", + }); + } +} + +let worker: Worker | null = null; + +export function createExtractionWorker(): Worker { + if (worker) return worker; + worker = new Worker( + EXTRACTION_QUEUE, + async (job: Job) => { + await runExtractionJob(job.data); + }, + { + connection: getRedisConnection(), + concurrency: 3, + // Recover jobs orphaned by a worker crash mid-run: re-queue a job + // whose lock hasn't been renewed within stalledInterval, up to + // maxStalledCount times before it's failed for good. + stalledInterval: 30_000, + maxStalledCount: 2, + }, + ); + worker.on("stalled", (jobId) => { + console.warn( + "[extraction-worker] job stalled; will be re-queued", + { jobId }, + ); + }); + worker.on("failed", async (job, err) => { + if (!job) { + console.error("[extraction-worker] job failed (no job)", { err }); + return; + } + if (!isPermanentFailure(job)) { + console.error( + "[extraction-worker] job failed (will retry, attempts remain)", + { jobId: job.id, err }, + ); + return; + } + console.error( + "[extraction-worker] job permanently failed; marking cells error", + { + jobId: job.id, + reviewId: job.data.reviewId, + rowId: job.data.rowId, + err, + }, + ); + try { + await markExtractionFailed(job.data); + } catch (updateErr) { + console.error( + "[extraction-worker] failed to mark cells error", + { jobId: job.id, updateErr }, + ); + } + }); + return worker; +} + +export async function stopExtractionWorker(): Promise { + if (worker) { + await worker.close(); + worker = null; + } +} diff --git a/backend/src/workers/index.ts b/backend/src/workers/index.ts new file mode 100644 index 000000000..506d90c41 --- /dev/null +++ b/backend/src/workers/index.ts @@ -0,0 +1,28 @@ +import { WORKER_REGISTRY } from "./registry"; +import { closeRedisConnection } from "../lib/queue/connection"; + +/** True when at least one background worker is enabled by the current config. */ +export function anyWorkerEnabled(): boolean { + return WORKER_REGISTRY.some((w) => w.enabled()); +} + +/** + * Start the in-process BullMQ workers whose feature flag is on. Called from the + * server entrypoint only when `anyWorkerEnabled()`, so the default (synchronous) + * deployment needs no Redis. Running workers in the API process keeps the + * dev/single-node story simple; split them into a dedicated process by calling + * this from a separate entrypoint when you need to scale them apart. + */ +export function startWorkers(): void { + for (const w of WORKER_REGISTRY) { + if (!w.enabled()) continue; + w.create(); + console.log(`[workers] ${w.name} worker started`); + } +} + +export async function stopWorkers(): Promise { + for (const w of WORKER_REGISTRY) await w.stop(); + for (const w of WORKER_REGISTRY) await w.closeQueue(); + await closeRedisConnection(); +} diff --git a/backend/src/workers/registry.ts b/backend/src/workers/registry.ts new file mode 100644 index 000000000..ecde7a645 --- /dev/null +++ b/backend/src/workers/registry.ts @@ -0,0 +1,51 @@ +import { + createConversionWorker, + stopConversionWorker, +} from "./conversionWorker"; +import { + createExtractionWorker, + stopExtractionWorker, +} from "./extractionWorker"; +import { closeConversionQueue } from "../lib/queue/conversionQueue"; +import { closeExtractionQueue } from "../lib/queue/extractionQueue"; + +/** + * One background queue's lifecycle, described declaratively. `startWorkers()` / + * `stopWorkers()` iterate this list, so the server entrypoint and shutdown path + * never need to know which queues exist. + */ +export interface WorkerDescriptor { + /** Log/identify label. */ + name: string; + /** Whether this worker should run in the current configuration. */ + enabled: () => boolean; + /** Start the in-process BullMQ worker (idempotent). */ + create: () => void; + /** Gracefully stop the worker. */ + stop: () => Promise; + /** Close the worker's producer-side queue. */ + closeQueue: () => Promise; +} + +/** + * To add a background queue: implement its queue (`lib/queue/Queue.ts`) + * and worker (`workers/Worker.ts`), then append one descriptor here. + * `startWorkers()`, `stopWorkers()`, `anyWorkerEnabled()`, and the server + * entrypoint all pick it up with no further change. + */ +export const WORKER_REGISTRY: WorkerDescriptor[] = [ + { + name: "document-conversion", + enabled: () => process.env.ASYNC_DOCUMENT_CONVERSION === "true", + create: createConversionWorker, + stop: stopConversionWorker, + closeQueue: closeConversionQueue, + }, + { + name: "tabular-extraction", + enabled: () => process.env.ASYNC_TABULAR_EXTRACTION === "true", + create: createExtractionWorker, + stop: stopExtractionWorker, + closeQueue: closeExtractionQueue, + }, +]; From 3a3ecbfb3a63e0044af3d7fbebb60998cd4fb652 Mon Sep 17 00:00:00 2001 From: Amal Date: Thu, 6 Aug 2026 10:23:20 -0700 Subject: [PATCH 2/3] =?UTF-8?q?feat:=20complete=20the=20async=20story=20?= =?UTF-8?q?=E2=80=94=20full=20conversion=20coverage,=20queued=20regenerate?= =?UTF-8?q?-cell,=20stale-work=20reaper,=20and=20a=20frontend=20that=20can?= =?UTF-8?q?=20consume=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WHY THIS MATTERS The first commit made two workloads durable, but a feature flag is only real if flipping it on produces a working product. Three gaps stood in the way. First, ASYNC_DOCUMENT_CONVERSION covered one of the five places that spawn LibreOffice — project uploads, added versions, replaced versions and document-to-version copies still blocked their requests for seconds to minutes. Second, regenerate-cell ran a full LLM extraction inline even with the extraction queue enabled, and a crash mid-call stranded the cell in "generating" forever. Third, the frontend had no way to see async results: nothing polled a "processing" document, and the reconnectable generate stream had zero callers — enabling the flags produced spinners that never resolved. WHAT IS AN ORPHANED TRANSIENT STATE Transient statuses ("processing", "generating") encode a promise: some running code will eventually write a terminal state. A crash in the window between the transient write and the terminal write breaks the promise, and because nothing else owns the row, the lie persists forever. The fix has two halves: narrow the windows (hand the work to a queue that retries and survives restarts) and add an owner of last resort (a reaper that flips provably-orphaned rows to "error"). HOW IT WORKS - Conversion queue covers all five LibreOffice call sites. Version flows pass a per-version pdfKey (renditions of different versions must not collide on the document-level key) and finalizeDocumentStatus: false — their document is already "ready", so a rendition failure must not flip a healthy document to "error"; only the initial-upload flow parks the document "processing" and lets the worker finalize it. Terminal conversion jobs are now removed immediately (same rationale as extraction): replace-file reuses the versionId, and a lingering completed job record would silently dedupe the re-conversion. - Regenerate-cell becomes a single-cell job: payload gains columnIndex, jobId gains a column suffix (extract:::) so it never dedupes against a full-row job, and the worker narrows to that one column. The route keeps its synchronous JSON contract by waiting on the cell's terminal state (pub/sub + DB-poll backstop); if the wait budget elapses it answers 202 {status:"generating"} — the job keeps running and the client catches up through the resume stream. The disconnect-divergence bug (client marks error, backend later writes done) is gone: the DB is the only authority. - Stale-work reaper (lib/maintenance/staleWork.ts, swept at boot + every 10 min): documents "processing" past a 30-minute age gate with no live conversion job flip to "error"; "generating" cells with no live job flip to "error" (async mode only — cells have no timestamp column, so in sync mode a live inline run is indistinguishable from an orphan). Job existence is the liveness signal, which immediate job removal makes trustworthy. - Frontend catch-up: GET /single-documents/:documentId exists so the client can poll one document instead of refetching the collection; DocTable polls pending/processing rows every 3s and merges status changes through the existing update path. The tabular view now aborts its generate stream on unmount, reconnects once through GET /generate/stream on a dropped stream, resumes an in-flight run found at mount (cells still "generating"), and treats regenerate's 202 as "keep the skeleton, tail the stream" instead of an error. - The GET stream view no longer dials Redis in synchronous deployments (the subscribe is flag-gated; the DB-poll backstop does the resolving there), so the no-Redis-by-default invariant holds for every new path. Tests: backend 523 passing (+13: payload passthrough, per-version pdf keys, finalize semantics, single-cell narrowing in worker + failure handler, reaper liveness/age-gate/no-op cases); frontend 174 passing, tsc and production build clean. Co-Authored-By: Claude Fable 5 --- backend/.env.example | 5 + backend/src/index.ts | 21 ++ .../maintenance/__tests__/staleWork.test.ts | 183 ++++++++++++++++++ backend/src/lib/maintenance/staleWork.ts | 166 ++++++++++++++++ .../queue/__tests__/conversionQueue.test.ts | 21 +- .../queue/__tests__/extractionQueue.test.ts | 19 ++ backend/src/lib/queue/conversionQueue.ts | 30 ++- backend/src/lib/queue/extractionQueue.ts | 20 +- .../src/lib/tabular/tabular.generateStream.ts | 146 ++++++++++++-- backend/src/routes/documents.ts | 129 ++++++++++-- backend/src/routes/projects.ts | 24 ++- backend/src/routes/tabular.ts | 54 ++++++ .../__tests__/conversionWorker.test.ts | 57 ++++++ .../__tests__/extractionWorker.test.ts | 63 ++++++ backend/src/workers/conversionWorker.ts | 45 +++-- backend/src/workers/extractionWorker.ts | 14 +- .../src/app/components/documents/DocTable.tsx | 32 +++ .../components/tabular/TabularReviewView.tsx | 141 ++++++++++---- frontend/src/app/lib/mikeApi.test.ts | 2 +- frontend/src/app/lib/mikeApi.ts | 31 ++- 20 files changed, 1100 insertions(+), 103 deletions(-) create mode 100644 backend/src/lib/maintenance/__tests__/staleWork.test.ts create mode 100644 backend/src/lib/maintenance/staleWork.ts diff --git a/backend/.env.example b/backend/.env.example index e0b9e3c3d..562c5358a 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -46,3 +46,8 @@ ASYNC_DOCUMENT_CONVERSION=false # progress over Redis pub/sub and can be resumed via GET .../generate/stream. # Requires REDIS_URL. Default "false" runs extraction inline. ASYNC_TABULAR_EXTRACTION=false +# Stale-work reaper: documents stuck "processing" longer than this (with no +# live conversion job) are flipped to "error" so the UI never spins forever. +# The sweep runs every STALE_SWEEP_INTERVAL_MS. Defaults: 30 min / 10 min. +#STALE_DOC_PROCESSING_MS=1800000 +#STALE_SWEEP_INTERVAL_MS=600000 diff --git a/backend/src/index.ts b/backend/src/index.ts index 52630913d..8a8d8c054 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -1,5 +1,6 @@ import { app } from "./app"; import { manifestPublicKey } from "./lib/manifestSigning"; +import { runStaleWorkSweep } from "./lib/maintenance/staleWork"; import { anyWorkerEnabled, startWorkers, stopWorkers } from "./workers"; const PORT = process.env.PORT ?? 3001; @@ -27,6 +28,26 @@ const server = app.listen(PORT, () => { } }); +// Stale-work reaper: a crash between "status = processing/generating" and the +// finalizing write strands rows in a transient state forever — nothing else +// owns them. Sweep shortly after boot (crash recovery) and on an interval. +// The sweep itself only dials Redis when an ASYNC_* flag is on. +const SWEEP_INTERVAL_MS = (() => { + const raw = Number(process.env.STALE_SWEEP_INTERVAL_MS); + return Number.isFinite(raw) && raw > 0 ? raw : 10 * 60 * 1000; +})(); +const runSweep = () => + void runStaleWorkSweep() + .then(({ documents, cells }) => { + if (documents || cells) + console.warn("[stale-sweep] flipped", { documents, cells }); + }) + .catch((err) => console.error("[stale-sweep] failed", err)); +const initialSweep = setTimeout(runSweep, 30_000); +initialSweep.unref(); +const sweepTimer = setInterval(runSweep, SWEEP_INTERVAL_MS); +sweepTimer.unref(); + // Graceful shutdown: on SIGTERM/SIGINT (orchestrator rollout, Ctrl-C), stop // accepting new connections, let in-flight requests/streams drain, close the // job-queue workers + Redis, then exit 0. Without this the orchestrator's diff --git a/backend/src/lib/maintenance/__tests__/staleWork.test.ts b/backend/src/lib/maintenance/__tests__/staleWork.test.ts new file mode 100644 index 000000000..c9b3d95bd --- /dev/null +++ b/backend/src/lib/maintenance/__tests__/staleWork.test.ts @@ -0,0 +1,183 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +vi.mock("../../supabase", () => ({ + createServerSupabase: vi.fn(), +})); + +const conversionGetJob = vi.fn(); +vi.mock("../../queue/conversionQueue", () => ({ + getConversionQueue: () => ({ getJob: conversionGetJob }), + conversionJobId: (versionId: string) => `convert:${versionId}`, +})); + +const extractionGetJob = vi.fn(); +vi.mock("../../queue/extractionQueue", () => ({ + getExtractionQueue: () => ({ getJob: extractionGetJob }), + extractionJobId: (reviewId: string, rowId: string, columnIndex?: number) => + columnIndex == null + ? `extract:${reviewId}:${rowId}` + : `extract:${reviewId}:${rowId}:${columnIndex}`, +})); + +import { + sweepStaleProcessingDocuments, + sweepStaleGeneratingCells, +} from "../staleWork"; + +type Call = { + table: string; + op: "select" | "update"; + payload?: Record; + filters: Record; +}; + +// Chainable Supabase double: select responses come from `responses[table]`; +// updates resolve empty and are recorded. +function makeDb(responses: Record) { + const calls: Call[] = []; + function from(table: string) { + const state: Call = { table, op: "select", filters: {} }; + const b: Record = { + select() { + state.op = "select"; + return b; + }, + update(payload: Record) { + state.op = "update"; + state.payload = payload; + return b; + }, + eq(col: string, val: unknown) { + state.filters[col] = val; + return b; + }, + lt(col: string, val: unknown) { + state.filters[`lt:${col}`] = val; + return b; + }, + then(onF: (v: unknown) => unknown, onR?: (e: unknown) => unknown) { + calls.push({ ...state, filters: { ...state.filters } }); + const value = + state.op === "select" + ? { data: responses[table] ?? [], error: null } + : { data: null, error: null }; + return Promise.resolve(value).then(onF, onR); + }, + }; + return b; + } + return { calls, from }; +} + +const ENV_KEYS = [ + "ASYNC_DOCUMENT_CONVERSION", + "ASYNC_TABULAR_EXTRACTION", + "STALE_DOC_PROCESSING_MS", +] as const; +const saved: Record = {}; + +beforeEach(() => { + for (const k of ENV_KEYS) { + saved[k] = process.env[k]; + delete process.env[k]; + } + conversionGetJob.mockReset(); + extractionGetJob.mockReset(); +}); + +afterEach(() => { + for (const k of ENV_KEYS) { + if (saved[k] === undefined) delete process.env[k]; + else process.env[k] = saved[k]; + } +}); + +describe("sweepStaleProcessingDocuments", () => { + it("flips stale processing documents to error (queue off: no job check)", async () => { + const db = makeDb({ + documents: [ + { id: "doc-1", current_version_id: "ver-1" }, + { id: "doc-2", current_version_id: null }, + ], + }); + + const flipped = await sweepStaleProcessingDocuments(db as never); + + expect(flipped).toBe(2); + expect(conversionGetJob).not.toHaveBeenCalled(); + const updates = db.calls.filter((c) => c.op === "update"); + expect(updates).toHaveLength(2); + // Guarded flip: only rows still "processing" are touched. + expect(updates[0].filters.status).toBe("processing"); + expect(updates[0].payload?.status).toBe("error"); + }); + + it("skips documents whose conversion job is still live (queue on)", async () => { + process.env.ASYNC_DOCUMENT_CONVERSION = "true"; + conversionGetJob.mockImplementation(async (jobId: string) => + jobId === "convert:ver-live" ? { id: jobId } : null, + ); + const db = makeDb({ + documents: [ + { id: "doc-live", current_version_id: "ver-live" }, + { id: "doc-dead", current_version_id: "ver-dead" }, + ], + }); + + const flipped = await sweepStaleProcessingDocuments(db as never); + + expect(flipped).toBe(1); + const updates = db.calls.filter((c) => c.op === "update"); + expect(updates).toHaveLength(1); + expect(updates[0].filters.id).toBe("doc-dead"); + }); +}); + +describe("sweepStaleGeneratingCells", () => { + it("is a no-op when the extraction queue is disabled", async () => { + const db = makeDb({ tabular_cells: [{ id: "c1" }] }); + + const flipped = await sweepStaleGeneratingCells(db as never); + + expect(flipped).toBe(0); + expect(db.calls).toHaveLength(0); + }); + + it("flips orphaned generating cells and spares those with a live job", async () => { + process.env.ASYNC_TABULAR_EXTRACTION = "true"; + extractionGetJob.mockImplementation(async (jobId: string) => + jobId === "extract:rev-1:row-live" ? { id: jobId } : null, + ); + const db = makeDb({ + tabular_cells: [ + { id: "c-live", review_id: "rev-1", row_id: "row-live", column_index: 0 }, + { id: "c-dead", review_id: "rev-1", row_id: "row-dead", column_index: 1 }, + ], + }); + + const flipped = await sweepStaleGeneratingCells(db as never); + + expect(flipped).toBe(1); + const updates = db.calls.filter((c) => c.op === "update"); + expect(updates).toHaveLength(1); + expect(updates[0].filters.id).toBe("c-dead"); + expect(updates[0].filters.status).toBe("generating"); + }); + + it("spares a cell whose single-cell (regenerate) job is live", async () => { + process.env.ASYNC_TABULAR_EXTRACTION = "true"; + extractionGetJob.mockImplementation(async (jobId: string) => + jobId === "extract:rev-1:row-1:2" ? { id: jobId } : null, + ); + const db = makeDb({ + tabular_cells: [ + { id: "c2", review_id: "rev-1", row_id: "row-1", column_index: 2 }, + ], + }); + + const flipped = await sweepStaleGeneratingCells(db as never); + + expect(flipped).toBe(0); + expect(db.calls.filter((c) => c.op === "update")).toHaveLength(0); + }); +}); diff --git a/backend/src/lib/maintenance/staleWork.ts b/backend/src/lib/maintenance/staleWork.ts new file mode 100644 index 000000000..1859c09ff --- /dev/null +++ b/backend/src/lib/maintenance/staleWork.ts @@ -0,0 +1,166 @@ +// Stale-work reaper: flips transient statuses that lost their owner to a +// terminal "error" so the UI never shows an eternal spinner. +// +// Transient statuses ("processing" documents, "generating" tabular cells) are +// normally resolved by the request that set them or by a queue worker. A crash +// in the wrong window strands them: the request died mid-pipeline, or a job +// was lost between the status write and the enqueue. Nothing else ever +// resolves them — this sweep is the missing owner of last resort. +// +// Safety model: +// - Documents are age-gated on updated_at (STALE_DOC_PROCESSING_MS, default +// 30 min) so an in-flight synchronous upload is never touched, and — when +// the conversion queue is enabled — a document whose conversion job still +// exists in the queue is skipped regardless of age. +// - Cells have no updated_at column, so their sweep runs ONLY when the +// extraction queue is enabled, where "generating with no live job" is +// sufficient evidence of orphanhood (sync-mode in-flight work cannot be +// distinguished from a stranded cell without an age signal, so sync +// deployments keep today's behavior: a stuck cell is fixed by re-clicking). + +import { createServerSupabase } from "../supabase"; +import { getConversionQueue, conversionJobId } from "../queue/conversionQueue"; +import { getExtractionQueue, extractionJobId } from "../queue/extractionQueue"; + +type Db = ReturnType; + +const DEFAULT_DOC_STALE_MS = 30 * 60 * 1000; + +function docStaleMs(): number { + const raw = Number(process.env.STALE_DOC_PROCESSING_MS); + return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_DOC_STALE_MS; +} + +/** + * Flip documents stuck in "processing" past the age threshold to "error", + * skipping any that still have a live conversion job. + */ +export async function sweepStaleProcessingDocuments( + db: Db = createServerSupabase(), +): Promise { + const cutoff = new Date(Date.now() - docStaleMs()).toISOString(); + const { data: docs, error } = await db + .from("documents") + .select("id, current_version_id") + .eq("status", "processing") + .lt("updated_at", cutoff); + if (error) { + console.error("[stale-sweep] documents query failed", error); + return 0; + } + + const queueOn = process.env.ASYNC_DOCUMENT_CONVERSION === "true"; + let flipped = 0; + for (const doc of (docs ?? []) as { + id: string; + current_version_id?: string | null; + }[]) { + if (queueOn && doc.current_version_id) { + // A job that still exists (waiting/active/delayed) owns this + // document; terminal jobs are removed immediately, so existence + // is the liveness signal. + const job = await getConversionQueue().getJob( + conversionJobId(doc.current_version_id), + ); + if (job) continue; + } + const { error: updateErr } = await db + .from("documents") + .update({ status: "error", updated_at: new Date().toISOString() }) + .eq("id", doc.id) + .eq("status", "processing"); + if (updateErr) { + console.error("[stale-sweep] document flip failed", { + documentId: doc.id, + error: updateErr, + }); + continue; + } + flipped += 1; + console.warn( + "[stale-sweep] stale processing document flipped to error", + { documentId: doc.id }, + ); + } + return flipped; +} + +/** + * Flip "generating" cells whose extraction job no longer exists to "error". + * Only meaningful (and only run) when the extraction queue is enabled — see + * the safety model above. + */ +export async function sweepStaleGeneratingCells( + db: Db = createServerSupabase(), +): Promise { + if (process.env.ASYNC_TABULAR_EXTRACTION !== "true") return 0; + + const { data: cells, error } = await db + .from("tabular_cells") + .select("id, review_id, row_id, column_index") + .eq("status", "generating"); + if (error) { + console.error("[stale-sweep] cells query failed", error); + return 0; + } + + const queue = getExtractionQueue(); + // One liveness lookup per (review, row) — full-row jobs cover every cell + // of their row; single-cell jobs are checked individually. + const rowJobLive = new Map(); + let flipped = 0; + for (const cell of (cells ?? []) as { + id: string; + review_id: string; + row_id: string; + column_index: number; + }[]) { + const rowKey = `${cell.review_id}:${cell.row_id}`; + if (!rowJobLive.has(rowKey)) { + const rowJob = await queue.getJob( + extractionJobId(cell.review_id, cell.row_id), + ); + rowJobLive.set(rowKey, !!rowJob); + } + if (rowJobLive.get(rowKey)) continue; + const cellJob = await queue.getJob( + extractionJobId(cell.review_id, cell.row_id, cell.column_index), + ); + if (cellJob) continue; + + const { error: updateErr } = await db + .from("tabular_cells") + .update({ status: "error" }) + .eq("id", cell.id) + .eq("status", "generating"); + if (updateErr) { + console.error("[stale-sweep] cell flip failed", { + cellId: cell.id, + error: updateErr, + }); + continue; + } + flipped += 1; + console.warn("[stale-sweep] orphaned generating cell flipped to error", { + reviewId: cell.review_id, + rowId: cell.row_id, + columnIndex: cell.column_index, + }); + } + return flipped; +} + +/** Run both sweeps; errors are contained per sweep. */ +export async function runStaleWorkSweep( + db: Db = createServerSupabase(), +): Promise<{ documents: number; cells: number }> { + const documents = await sweepStaleProcessingDocuments(db).catch((err) => { + console.error("[stale-sweep] document sweep crashed", err); + return 0; + }); + const cells = await sweepStaleGeneratingCells(db).catch((err) => { + console.error("[stale-sweep] cell sweep crashed", err); + return 0; + }); + return { documents, cells }; +} diff --git a/backend/src/lib/queue/__tests__/conversionQueue.test.ts b/backend/src/lib/queue/__tests__/conversionQueue.test.ts index a2177acdc..2562b6c3f 100644 --- a/backend/src/lib/queue/__tests__/conversionQueue.test.ts +++ b/backend/src/lib/queue/__tests__/conversionQueue.test.ts @@ -46,13 +46,28 @@ describe("enqueueConversion", () => { expect(opts.jobId).toBe("convert:ver-1"); }); - it("keeps the existing retry/backoff/history options", () => { + it("retries with backoff and removes terminal jobs so re-conversions can re-enqueue", () => { enqueueConversion(DATA); const opts = add.mock.calls[0][2]; expect(opts.attempts).toBe(3); expect(opts.backoff).toEqual({ type: "exponential", delay: 2000 }); - expect(opts.removeOnComplete).toBe(100); - expect(opts.removeOnFail).toBe(500); + // Immediate removal (not keep-N) is deliberate: replace-file reuses + // the versionId, and a lingering completed job record would silently + // dedupe the re-conversion into the old job. + expect(opts.removeOnComplete).toBe(true); + expect(opts.removeOnFail).toBe(true); + }); + + it("carries the version-flow fields (pdfKey, finalizeDocumentStatus) through", () => { + enqueueConversion({ + ...DATA, + pdfKey: "converted-pdfs/user-1/doc-1/slug.pdf", + finalizeDocumentStatus: false, + }); + + const data = add.mock.calls[0][1]; + expect(data.pdfKey).toBe("converted-pdfs/user-1/doc-1/slug.pdf"); + expect(data.finalizeDocumentStatus).toBe(false); }); }); diff --git a/backend/src/lib/queue/__tests__/extractionQueue.test.ts b/backend/src/lib/queue/__tests__/extractionQueue.test.ts index ba1d38a02..53a8bb25f 100644 --- a/backend/src/lib/queue/__tests__/extractionQueue.test.ts +++ b/backend/src/lib/queue/__tests__/extractionQueue.test.ts @@ -31,6 +31,25 @@ describe("extractionJobId", () => { it("is deterministic on (reviewId, rowId)", () => { expect(extractionJobId("rev-1", "row-1")).toBe("extract:rev-1:row-1"); }); + + it("suffixes single-cell jobs so they never dedupe against full-row jobs", () => { + expect(extractionJobId("rev-1", "row-1", 2)).toBe( + "extract:rev-1:row-1:2", + ); + expect(extractionJobId("rev-1", "row-1", 0)).toBe( + "extract:rev-1:row-1:0", + ); + }); +}); + +describe("enqueueExtraction (single-cell)", () => { + it("uses the column-suffixed jobId and carries columnIndex", () => { + enqueueExtraction({ ...DATA, columnIndex: 1 }); + + const [, data, opts] = add.mock.calls[0]; + expect(data.columnIndex).toBe(1); + expect(opts.jobId).toBe("extract:rev-1:row-1:1"); + }); }); describe("enqueueExtraction", () => { diff --git a/backend/src/lib/queue/conversionQueue.ts b/backend/src/lib/queue/conversionQueue.ts index 31d0cedd1..5a763c894 100644 --- a/backend/src/lib/queue/conversionQueue.ts +++ b/backend/src/lib/queue/conversionQueue.ts @@ -15,6 +15,21 @@ export interface ConversionJobData { storagePath: string; /** "docx" | "doc". */ fileType: string; + /** + * Storage key the rendition should be written to. Version flows use a + * per-version key (`converted-pdfs///.pdf`) so renditions + * of different versions never collide; when omitted the worker falls back + * to the document-level `convertedPdfKey`. + */ + pdfKey?: string; + /** + * When false, the worker only fills the version's pdf_storage_path and + * never touches documents.status. Version add/replace/copy flows use this: + * their document is already "ready" and a rendition failure must not + * flip a healthy document to "error". Defaults to true (the initial-upload + * flow, where the document is parked "processing" until conversion ends). + */ + finalizeDocumentStatus?: boolean; } let queue: Queue | null = null; @@ -35,18 +50,23 @@ export function conversionJobId(versionId: string): string { /** * Enqueue a conversion. Retries transient failures (storage/LibreOffice - * hiccups) with exponential backoff; keeps a bounded history for inspection. + * hiccups) with exponential backoff. * - * The jobId is derived from the (unique-per-upload) versionId so a double - * submit is deduped by BullMQ instead of racing two conversions. + * The jobId is derived from the versionId so a double submit is deduped by + * BullMQ instead of racing two conversions. Terminal jobs are removed + * immediately (same rationale as the extraction queue): a version can be + * re-converted later — replace-file reuses the versionId — and a completed + * job record left behind would silently swallow that re-enqueue as a + * duplicate. Durable state lives in document_versions/documents, not in the + * job record. */ export function enqueueConversion(data: ConversionJobData) { return getConversionQueue().add("convert", data, { jobId: conversionJobId(data.versionId), attempts: 3, backoff: { type: "exponential", delay: 2000 }, - removeOnComplete: 100, - removeOnFail: 500, + removeOnComplete: true, + removeOnFail: true, }); } diff --git a/backend/src/lib/queue/extractionQueue.ts b/backend/src/lib/queue/extractionQueue.ts index 8dae3e35d..ae22d0ed7 100644 --- a/backend/src/lib/queue/extractionQueue.ts +++ b/backend/src/lib/queue/extractionQueue.ts @@ -21,6 +21,12 @@ export interface ExtractionJobData { userId: string; /** tabular_review_rows.id whose columns this job fills. */ rowId: string; + /** + * When set, the job targets ONE cell (regenerate-cell) instead of every + * outstanding column of the row. Single-cell jobs get their own jobId + * suffix so they never dedupe against a full-row job for the same row. + */ + columnIndex?: number; } let queue: Queue | null = null; @@ -34,9 +40,15 @@ export function getExtractionQueue(): Queue { return queue; } -/** Deterministic BullMQ jobId for one (review, row) extraction. */ -export function extractionJobId(reviewId: string, rowId: string): string { - return `extract:${reviewId}:${rowId}`; +/** Deterministic BullMQ jobId for one (review, row[, column]) extraction. */ +export function extractionJobId( + reviewId: string, + rowId: string, + columnIndex?: number, +): string { + return columnIndex == null + ? `extract:${reviewId}:${rowId}` + : `extract:${reviewId}:${rowId}:${columnIndex}`; } /** @@ -52,7 +64,7 @@ export function extractionJobId(reviewId: string, rowId: string): string { */ export function enqueueExtraction(data: ExtractionJobData) { return getExtractionQueue().add("extract", data, { - jobId: extractionJobId(data.reviewId, data.rowId), + jobId: extractionJobId(data.reviewId, data.rowId, data.columnIndex), attempts: 3, backoff: { type: "exponential", delay: 2000 }, removeOnComplete: true, diff --git a/backend/src/lib/tabular/tabular.generateStream.ts b/backend/src/lib/tabular/tabular.generateStream.ts index 9f90d6441..e030e1d58 100644 --- a/backend/src/lib/tabular/tabular.generateStream.ts +++ b/backend/src/lib/tabular/tabular.generateStream.ts @@ -149,21 +149,26 @@ async function tailTabularRun(args: { if (pending.size === 0) return void finish(); // Subscribe BEFORE enqueuing so a fast worker can't publish into the void. - try { - sub = new IORedis(REDIS_URL, { maxRetriesPerRequest: null }); - await sub.subscribe(runProgressChannel(reviewId)); - sub.on("message", (_channel, message) => { - try { - onUpdate(JSON.parse(message) as CellUpdate); - } catch { - /* ignore malformed frame */ - } - }); - } catch (err) { - log.error( - "[tabular/generate-async] subscribe failed", - { err: safeErrorLog(err), reviewId }, - ); + // Only when the async flag is on: the GET view is also reachable in + // synchronous (no-Redis) deployments, where dialing Redis would hang the + // stream — there the DB-poll backstop below does all the resolving. + if (process.env.ASYNC_TABULAR_EXTRACTION === "true") { + try { + sub = new IORedis(REDIS_URL, { maxRetriesPerRequest: null }); + await sub.subscribe(runProgressChannel(reviewId)); + sub.on("message", (_channel, message) => { + try { + onUpdate(JSON.parse(message) as CellUpdate); + } catch { + /* ignore malformed frame */ + } + }); + } catch (err) { + log.error( + "[tabular/generate-async] subscribe failed", + { err: safeErrorLog(err), reviewId }, + ); + } } if (afterSubscribe) await afterSubscribe(); @@ -216,6 +221,117 @@ async function tailTabularRun(args: { if (typeof cap.unref === "function") cap.unref(); } +/** + * Wait for one cell to reach a terminal state — the "view" half of an + * async regenerate-cell. The job is already enqueued; this subscribes to the + * review's progress channel (flag on) and polls the DB as a backstop, then + * returns the cell's terminal content, or null if `timeoutMs` elapses first + * (the job keeps running — the caller reports "still generating"). + */ +export async function awaitCellTerminal(args: { + db: Db; + reviewId: string; + rowId: string; + columnIndex: number; + log: Log; + timeoutMs?: number; + pollMs?: number; +}): Promise< + | { status: "done"; content: ReturnType } + | { status: "error" } + | null +> { + const { db, reviewId, rowId, columnIndex, log } = args; + const timeoutMs = args.timeoutMs ?? 120_000; + const pollMs = args.pollMs ?? 1_000; + + let sub: IORedis | null = null; + let poll: ReturnType | null = null; + let timer: ReturnType | null = null; + + try { + return await new Promise((resolve) => { + let settled = false; + const settle = ( + value: + | { status: "done"; content: ReturnType } + | { status: "error" } + | null, + ) => { + if (settled) return; + settled = true; + resolve(value); + }; + + const checkDb = async () => { + const { data: cell } = await db + .from("tabular_cells") + .select("status, content") + .eq("review_id", reviewId) + .eq("row_id", rowId) + .eq("column_index", columnIndex) + .maybeSingle(); + if (!cell) return; + if (cell.status === "done" && cell.content) + settle({ + status: "done", + content: parseCellContent(cell.content), + }); + else if (cell.status === "error") settle({ status: "error" }); + }; + + if (process.env.ASYNC_TABULAR_EXTRACTION === "true") { + try { + sub = new IORedis(REDIS_URL, { maxRetriesPerRequest: null }); + void sub + .subscribe(runProgressChannel(reviewId)) + .catch(() => {}); + sub.on("message", (_channel, message) => { + try { + const update = JSON.parse(message) as CellUpdate; + if ( + update.row_id !== rowId || + update.column_index !== columnIndex + ) + return; + if (update.status === "done") + settle({ + status: "done", + content: update.content as ReturnType< + typeof parseCellContent + >, + }); + else if (update.status === "error") + settle({ status: "error" }); + } catch { + /* ignore malformed frame */ + } + }); + } catch (err) { + log.error("[tabular/regenerate-async] subscribe failed", { + err: safeErrorLog(err), + reviewId, + }); + } + } + + poll = setInterval(() => { + void checkDb().catch((err) => + log.error("[tabular/regenerate-async] poll failed", { + err: safeErrorLog(err), + reviewId, + }), + ); + }, pollMs); + timer = setTimeout(() => settle(null), timeoutMs); + }); + } finally { + if (poll) clearInterval(poll); + if (timer) clearTimeout(timer); + if (sub) void (sub as IORedis).quit().catch(() => {}); + } +} + /** POST /:reviewId/generate — enqueue the outstanding work, then tail it. */ export async function streamTabularGenerateAsync(args: { res: Response; diff --git a/backend/src/routes/documents.ts b/backend/src/routes/documents.ts index 44fe203b4..5ae607969 100644 --- a/backend/src/routes/documents.ts +++ b/backend/src/routes/documents.ts @@ -79,6 +79,35 @@ documentsRouter.get("/", requireAuth, async (req, res) => { res.json(docs); }); +// GET /single-documents/:documentId +// One document, same shape as a list entry. Exists so the client can poll a +// single document's status while a deferred conversion runs, instead of +// refetching the whole collection. +documentsRouter.get("/:documentId", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { documentId } = req.params; + const db = createServerSupabase(); + + const { data: doc } = await db + .from("documents") + .select("*") + .eq("id", documentId) + .single(); + if (!doc) return void res.status(404).json({ detail: "Document not found" }); + const access = await ensureDocAccess(doc, userId, userEmail, db); + if (!access.ok) + return void res.status(404).json({ detail: "Document not found" }); + + const docs = [doc] as unknown as { + id: string; + current_version_id?: string | null; + }[]; + await attachLatestVersionNumbers(db, docs); + await attachActiveVersionPaths(db, docs); + res.json(docs[0]); +}); + // POST /single-documents documentsRouter.post( "/", @@ -478,6 +507,7 @@ documentsRouter.post( } let pdfStoragePath: string | null = null; + let deferConversion = false; if (suffix === "pdf") { pdfStoragePath = key; } else if (active.pdf_storage_path) { @@ -492,23 +522,30 @@ documentsRouter.post( } } } else if (shouldConvertToPdf(suffix)) { - try { - const pdfBuf = await docxToPdf(Buffer.from(bytes)); - const pdfKey = `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`; - await uploadFile( - pdfKey, - pdfBuf.buffer.slice( - pdfBuf.byteOffset, - pdfBuf.byteOffset + pdfBuf.byteLength, - ) as ArrayBuffer, - "application/pdf", - ); - pdfStoragePath = pdfKey; - } catch (err) { - console.error( - `[versions/copy] Office→PDF conversion failed for ${filename}:`, - err, - ); + // Only reached when the source has no rendition to copy — this is the + // one branch of the copy flow that pays for LibreOffice, so it's the + // branch the conversion queue takes over when the flag is on. + if (process.env.ASYNC_DOCUMENT_CONVERSION === "true") { + deferConversion = true; + } else { + try { + const pdfBuf = await docxToPdf(Buffer.from(bytes)); + const pdfKey = `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`; + await uploadFile( + pdfKey, + pdfBuf.buffer.slice( + pdfBuf.byteOffset, + pdfBuf.byteOffset + pdfBuf.byteLength, + ) as ArrayBuffer, + "application/pdf", + ); + pdfStoragePath = pdfKey; + } catch (err) { + console.error( + `[versions/copy] Office→PDF conversion failed for ${filename}:`, + err, + ); + } } } @@ -559,6 +596,18 @@ documentsRouter.post( .json({ detail: "Failed to update document current version." }); } + if (deferConversion) { + await enqueueConversion({ + documentId, + versionId: versionRow.id as string, + userId, + storagePath: key, + fileType: suffix, + pdfKey: `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`, + finalizeDocumentStatus: false, + }); + } + if (willDeleteSource) { const { error: deleteErr } = await deleteDocumentAndVersionFiles( db, @@ -643,8 +692,14 @@ documentsRouter.post( // Render this version's bytes to PDF up front so /display can show // historical versions without on-demand conversion. Same logic as the // initial-upload pipeline; failures don't block the version row. + // With the job queue enabled the LibreOffice work is deferred to the + // conversion worker instead of blocking this request; the version row is + // created with pdf_storage_path null and the worker fills it in. + const deferConversion = + shouldConvertToPdf(suffix) && + process.env.ASYNC_DOCUMENT_CONVERSION === "true"; let pdfStoragePath: string | null = null; - if (shouldConvertToPdf(suffix)) { + if (!deferConversion && shouldConvertToPdf(suffix)) { try { const pdfBuf = await docxToPdf(file.buffer); const pdfKey = `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`; @@ -732,6 +787,20 @@ documentsRouter.post( .json({ detail: "Failed to update document current version." }); } + if (deferConversion) { + // The document itself stays "ready" — only this version's rendition is + // pending, so the worker must not touch documents.status. + await enqueueConversion({ + documentId, + versionId: versionRow.id as string, + userId, + storagePath: key, + fileType: suffix, + pdfKey: `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`, + finalizeDocumentStatus: false, + }); + } + res.status(201).json(versionRow); }, ); @@ -857,8 +926,15 @@ documentsRouter.put( .json({ detail: "Failed to upload replacement version." }); } + // Same queue deferral as version uploads: the replacement's rendition is + // produced by the conversion worker when the flag is on. The old rendition + // is deleted below either way, so /display briefly falls back until the + // worker writes the new one. + const deferConversion = + shouldConvertToPdf(suffix) && + process.env.ASYNC_DOCUMENT_CONVERSION === "true"; let pdfStoragePath: string | null = null; - if (shouldConvertToPdf(suffix)) { + if (!deferConversion && shouldConvertToPdf(suffix)) { try { const pdfBuf = await docxToPdf(file.buffer); const pdfKey = `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`; @@ -927,6 +1003,21 @@ documentsRouter.put( .map((path) => deleteFile(path).catch(() => {})), ); + if (deferConversion) { + // Replace reuses the versionId, which is exactly why terminal jobs are + // removed from the queue immediately — this enqueue must not be deduped + // against a completed job for the same version. + await enqueueConversion({ + documentId, + versionId, + userId, + storagePath: key, + fileType: suffix, + pdfKey: `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`, + finalizeDocumentStatus: false, + }); + } + res.json(updated); }, ); diff --git a/backend/src/routes/projects.ts b/backend/src/routes/projects.ts index ed5750b29..2fd1eeae1 100644 --- a/backend/src/routes/projects.ts +++ b/backend/src/routes/projects.ts @@ -1,6 +1,7 @@ import { Router } from "express"; import { requireAuth, requireMfaIfEnrolled } from "../middleware/auth"; import { createServerSupabase } from "../lib/supabase"; +import { enqueueConversion } from "../lib/queue/conversionQueue"; import { createClient } from "@supabase/supabase-js"; import { attachActiveVersionPaths, @@ -1038,9 +1039,16 @@ export async function handleDocumentUpload( ) as ArrayBuffer; const pageCount = suffix === "pdf" ? await countPdfPages(rawBuf) : null; + // When the job queue is enabled, defer Office → PDF conversion to the + // BullMQ worker instead of blocking the upload request on LibreOffice — + // the same deferral the single-document upload path makes. + const deferConversion = + shouldConvertToPdf(suffix) && + process.env.ASYNC_DOCUMENT_CONVERSION === "true"; + // Convert Office files → PDF for display. PDFs are their own rendition. let pdfStoragePath: string | null = null; - if (shouldConvertToPdf(suffix)) { + if (!deferConversion && shouldConvertToPdf(suffix)) { try { const pdfBuf = await docxToPdf(content); const pdfKey = convertedPdfKey(userId, docId); @@ -1091,11 +1099,23 @@ export async function handleDocumentUpload( .from("documents") .update({ current_version_id: versionRow.id, - status: "ready", + // Deferred conversion leaves the doc "processing" until the worker + // produces the PDF and flips it to "ready". + status: deferConversion ? "processing" : "ready", updated_at: new Date().toISOString(), }) .eq("id", docId); + if (deferConversion) { + await enqueueConversion({ + documentId: docId, + versionId: versionRow.id as string, + userId, + storagePath: key, + fileType: suffix, + }); + } + const { data: updated } = await db .from("documents") .select("*") diff --git a/backend/src/routes/tabular.ts b/backend/src/routes/tabular.ts index db2a8db9a..8dcf360e5 100644 --- a/backend/src/routes/tabular.ts +++ b/backend/src/routes/tabular.ts @@ -26,9 +26,11 @@ import { import { extractRowColumns } from "../lib/tabular/tabular.extractRow"; import { prepareTabularGenerate } from "../lib/tabular/tabular.generate"; import { + awaitCellTerminal, streamTabularGenerateAsync, streamTabularRunView, } from "../lib/tabular/tabular.generateStream"; +import { enqueueExtraction } from "../lib/queue/extractionQueue"; import { fetchSourceDocuments, loadReviewRows, @@ -960,6 +962,58 @@ tabularRouter.post( .eq("row_id", row.id) .eq("column_index", column_index); + // Async path: enqueue a single-cell job (deduped on + // extract:::) and wait for the cell to reach a + // terminal state, so the response keeps its synchronous JSON shape. + // The work itself is durable: if this request drops or times out the + // worker still finishes and the client catches up via the DB or the + // GET generate/stream view. + if (process.env.ASYNC_TABULAR_EXTRACTION === "true") { + try { + await enqueueExtraction({ + reviewId, + userId, + rowId: row.id, + columnIndex: column_index, + }); + } catch (err) { + console.error( + "[tabular/regenerate-cell] enqueue failed", + safeErrorLog(err), + ); + await db + .from("tabular_cells") + .update({ status: "error" }) + .eq("review_id", reviewId) + .eq("row_id", row.id) + .eq("column_index", column_index); + return void res + .status(500) + .json({ detail: "Generation failed" }); + } + + const terminal = await awaitCellTerminal({ + db, + reviewId, + rowId: row.id, + columnIndex: column_index, + log: console, + }); + if (terminal === null) + // Still running after the wait budget — the job survives this + // response; the client keeps the cell "generating" and picks + // the result up from the resume stream or a reload. + return void res.status(202).json({ + status: "generating", + detail: "Extraction still running", + }); + if (terminal.status === "error") + return void res + .status(500) + .json({ detail: "Generation failed" }); + return void res.json(terminal.content); + } + const markdown = await loadRowDocumentText(db, row); const result = await queryTabularCell( diff --git a/backend/src/workers/__tests__/conversionWorker.test.ts b/backend/src/workers/__tests__/conversionWorker.test.ts index a75ca9345..4873dc24a 100644 --- a/backend/src/workers/__tests__/conversionWorker.test.ts +++ b/backend/src/workers/__tests__/conversionWorker.test.ts @@ -97,6 +97,63 @@ describe("runConversionJob", () => { expect(docUpdate?.update.status).toBe("ready"); }); + it("writes the rendition to the payload's pdfKey when provided", async () => { + downloadFile.mockResolvedValue(new ArrayBuffer(8)); + docxToPdf.mockResolvedValue(Buffer.from("%PDF-1.7 fake")); + uploadFile.mockResolvedValue(undefined); + const db = makeDb(); + + await runConversionJob( + { ...JOB, pdfKey: "converted-pdfs/user-1/doc-1/slug.pdf" }, + db as never, + ); + + expect(uploadFile).toHaveBeenCalledWith( + "converted-pdfs/user-1/doc-1/slug.pdf", + expect.anything(), + "application/pdf", + ); + expect(db.calls).toContainEqual({ + table: "document_versions", + update: { + pdf_storage_path: "converted-pdfs/user-1/doc-1/slug.pdf", + }, + }); + }); + + it("never touches documents.status when finalizeDocumentStatus is false", async () => { + downloadFile.mockResolvedValue(new ArrayBuffer(8)); + docxToPdf.mockResolvedValue(Buffer.from("%PDF-1.7 fake")); + uploadFile.mockResolvedValue(undefined); + const db = makeDb(); + + await runConversionJob( + { ...JOB, finalizeDocumentStatus: false }, + db as never, + ); + + expect( + db.calls.some((c) => c.table === "documents"), + ).toBe(false); + // The version row still gets its rendition. + expect( + db.calls.some((c) => c.table === "document_versions"), + ).toBe(true); + }); + + it("leaves the document alone on conversion failure when finalizeDocumentStatus is false", async () => { + downloadFile.mockResolvedValue(new ArrayBuffer(8)); + docxToPdf.mockRejectedValue(new Error("soffice exploded")); + const db = makeDb(); + + await runConversionJob( + { ...JOB, finalizeDocumentStatus: false }, + db as never, + ); + + expect(db.calls).toHaveLength(0); + }); + it("throws when the original is missing so BullMQ retries", async () => { downloadFile.mockResolvedValue(null); const db = makeDb(); diff --git a/backend/src/workers/__tests__/extractionWorker.test.ts b/backend/src/workers/__tests__/extractionWorker.test.ts index 5c52c5ebe..f73fb5326 100644 --- a/backend/src/workers/__tests__/extractionWorker.test.ts +++ b/backend/src/workers/__tests__/extractionWorker.test.ts @@ -225,6 +225,43 @@ describe("runExtractionJob", () => { ).rejects.toThrow(/incomplete extraction/); }); + it("restricts a single-cell job (columnIndex) to its one column", async () => { + const publish = vi.fn(async () => {}); + const db = makeDb({ + tabular_reviews: { select: { data: { columns_config: COLUMNS } } }, + tabular_cells: { + select: { + data: [ + // Both cells are outstanding, but the job only owns col 1. + { id: "c0", column_index: 0, status: "pending", content: null }, + { id: "c1", column_index: 1, status: "generating", content: null }, + ], + }, + }, + }); + queryTabularAllColumns.mockImplementation( + async (_m, _f, _t, cols, onResult) => { + for (const c of cols) await onResult(c.index, CELL(c.index, {})); + }, + ); + + await runExtractionJob( + { ...DATA, columnIndex: 1 }, + { db: db as never, publish }, + ); + + // The LLM call was scoped to exactly one column. + const passedColumns = queryTabularAllColumns.mock.calls[0][3] as { + index: number; + }[]; + expect(passedColumns.map((c) => c.index)).toEqual([1]); + // Only column 1's cell was touched. + const updates = db.calls.filter((c) => c.op === "update"); + expect( + updates.every((c) => c.filters.column_index === 1 || c.filters.id === "c1"), + ).toBe(true); + }); + it("returns early when the review has no columns", async () => { const publish = vi.fn(async () => {}); const db = makeDb({ @@ -253,6 +290,32 @@ describe("runExtractionJob", () => { }); describe("markExtractionFailed", () => { + it("only touches its own column for a single-cell job", async () => { + const publish = vi.fn(async () => {}); + const db = makeDb({ + tabular_cells: { + select: { + data: [ + { id: "c0", column_index: 0, status: "generating", content: null }, + { id: "c1", column_index: 1, status: "generating", content: null }, + ], + }, + }, + }); + + await markExtractionFailed( + { ...DATA, columnIndex: 1 }, + { db: db as never, publish }, + ); + + const errorUpdates = db.calls.filter( + (c) => c.op === "update" && c.payload?.status === "error", + ); + expect(errorUpdates).toHaveLength(1); + expect(errorUpdates[0].filters.id).toBe("c1"); + expect(publish).toHaveBeenCalledTimes(1); + }); + it("marks only unfinished cells error and publishes them", async () => { const publish = vi.fn(async () => {}); const db = makeDb({ diff --git a/backend/src/workers/conversionWorker.ts b/backend/src/workers/conversionWorker.ts index 0f9dfefac..0d2825c81 100644 --- a/backend/src/workers/conversionWorker.ts +++ b/backend/src/workers/conversionWorker.ts @@ -24,6 +24,7 @@ export async function runConversionJob( db: Db = createServerSupabase(), ): Promise { const { documentId, versionId, userId, storagePath } = data; + const finalize = data.finalizeDocumentStatus !== false; const original = await downloadFile(storagePath); if (!original) { @@ -35,7 +36,7 @@ export async function runConversionJob( try { const pdfBuf = await docxToPdf(Buffer.from(original)); - const pdfKey = convertedPdfKey(userId, documentId); + const pdfKey = data.pdfKey ?? convertedPdfKey(userId, documentId); await uploadFile( pdfKey, pdfBuf.buffer.slice( @@ -48,20 +49,33 @@ export async function runConversionJob( .from("document_versions") .update({ pdf_storage_path: pdfKey }) .eq("id", versionId); - await db - .from("documents") - .update({ status: "ready", updated_at: new Date().toISOString() }) - .eq("id", documentId); + if (finalize) { + await db + .from("documents") + .update({ + status: "ready", + updated_at: new Date().toISOString(), + }) + .eq("id", documentId); + } console.log("[conversion-worker] converted", { documentId, versionId }); } catch (err) { + // Conversion failure is non-fatal (mirrors the sync path): the version + // stays usable without a PDF rendition. Only the initial-upload flow + // (finalize) needs the parked "processing" document flipped to ready. console.error( "[conversion-worker] DOCX→PDF failed; finalizing without a PDF rendition", { err, documentId, versionId }, ); - await db - .from("documents") - .update({ status: "ready", updated_at: new Date().toISOString() }) - .eq("id", documentId); + if (finalize) { + await db + .from("documents") + .update({ + status: "ready", + updated_at: new Date().toISOString(), + }) + .eq("id", documentId); + } } } @@ -123,8 +137,17 @@ export function createConversionWorker(): Worker { ); return; } - // Retries exhausted: the document is stuck "processing" with no PDF and - // no path forward — surface it to the user as a terminal "error". + // Retries exhausted. For the initial-upload flow the document is stuck + // "processing" with no path forward — surface it as a terminal + // "error". Version flows (finalizeDocumentStatus: false) belong to an + // already-healthy document: the version simply keeps no rendition. + if (job.data.finalizeDocumentStatus === false) { + console.error( + "[conversion-worker] version rendition permanently failed; document left untouched", + { jobId: job.id, versionId: job.data.versionId, err }, + ); + return; + } console.error( "[conversion-worker] job permanently failed; marking document error", { jobId: job.id, documentId: job.data.documentId, err }, diff --git a/backend/src/workers/extractionWorker.ts b/backend/src/workers/extractionWorker.ts index bb091414a..ec91f5551 100644 --- a/backend/src/workers/extractionWorker.ts +++ b/backend/src/workers/extractionWorker.ts @@ -44,16 +44,20 @@ export async function runExtractionJob( data: ExtractionJobData, deps: ExtractionDeps = defaultDeps(), ): Promise { - const { reviewId, userId, rowId } = data; + const { reviewId, userId, rowId, columnIndex } = data; const { db, publish } = deps; - // 1. Columns configured on the review. + // 1. Columns configured on the review. A single-cell job (regenerate) + // narrows to its one column; the cell was already flipped off "done" + // by the enqueuing route, so the shared core will re-extract it. const { data: review } = await db .from("tabular_reviews") .select("columns_config") .eq("id", reviewId) .single(); - const columns: Column[] = (review?.columns_config as Column[]) ?? []; + let columns: Column[] = (review?.columns_config as Column[]) ?? []; + if (columnIndex != null) + columns = columns.filter((c) => c.index === columnIndex); if (columns.length === 0) return; // 2. The row this job fills (with its source-document ids resolved). A row @@ -132,7 +136,7 @@ export async function markExtractionFailed( data: ExtractionJobData, deps: ExtractionDeps = defaultDeps(), ): Promise { - const { reviewId, rowId } = data; + const { reviewId, rowId, columnIndex } = data; const { db, publish } = deps; const { data: cells } = await db @@ -142,6 +146,8 @@ export async function markExtractionFailed( .eq("row_id", rowId); for (const cell of (cells ?? []) as Record[]) { + // Single-cell jobs only ever own their one column's terminal state. + if (columnIndex != null && cell.column_index !== columnIndex) continue; if (cell.status === "done" && cell.content) continue; await db .from("tabular_cells") diff --git a/frontend/src/app/components/documents/DocTable.tsx b/frontend/src/app/components/documents/DocTable.tsx index 050bdb90b..92134f225 100644 --- a/frontend/src/app/components/documents/DocTable.tsx +++ b/frontend/src/app/components/documents/DocTable.tsx @@ -20,6 +20,7 @@ import { } from "lucide-react"; import { deleteDocument, + getDocument, getDocumentUrl, downloadDocumentsZip, listDocumentVersions, @@ -655,6 +656,37 @@ export function DocTable({ return () => document.removeEventListener("dragend", handleDragEnd); }, []); + // Poll documents stuck in deferred conversion until the backend marks + // them "ready"/"error" (async conversion flips status server-side) + useEffect(() => { + const converting = documents.filter( + (d) => d.status === "pending" || d.status === "processing", + ); + if (converting.length === 0) return; + + let cancelled = false; + const interval = window.setInterval(() => { + for (const doc of converting) { + getDocument(doc.id) + .then((latest) => { + if (cancelled || latest.status === doc.status) return; + setDocuments((prev) => + prev.map((d) => + d.id === doc.id ? { ...d, ...latest } : d, + ), + ); + }) + .catch(() => { + // Transient fetch failure — keep polling + }); + } + }, 3000); + return () => { + cancelled = true; + window.clearInterval(interval); + }; + }, [documents, setDocuments]); + // Scroll new-folder input into view whenever it appears useEffect(() => { if (creatingFolderIn !== undefined) { diff --git a/frontend/src/app/components/tabular/TabularReviewView.tsx b/frontend/src/app/components/tabular/TabularReviewView.tsx index 13de213f5..ac904c431 100644 --- a/frontend/src/app/components/tabular/TabularReviewView.tsx +++ b/frontend/src/app/components/tabular/TabularReviewView.tsx @@ -27,6 +27,7 @@ import { listProjects, regenerateTabularCell, streamTabularGeneration, + streamTabularGenerationResume, updateTabularReview, uploadReviewDocument, } from "@/app/lib/mikeApi"; @@ -131,6 +132,8 @@ export function TRView({ reviewId, projectId }: Props) { useState(null); const actionsRef = useRef(null); const tableRef = useRef(null); + const generationAbortRef = useRef(null); + const resumeStreamOpenRef = useRef(false); const router = useRouter(); const { profile } = useUserProfile(); const apiKeys = profile?.apiKeys; @@ -162,6 +165,11 @@ export function TRView({ reviewId, projectId }: Props) { document.removeEventListener("mousedown", handleClickOutside); }, [actionsOpen]); + // Abort any in-flight generation/resume stream on unmount + useEffect(() => { + return () => generationAbortRef.current?.abort(); + }, []); + useEffect(() => { const fetches: Promise[] = [ getTabularReview(reviewId).then(({ review, cells, rows, documents }) => { @@ -170,6 +178,13 @@ export function TRView({ reviewId, projectId }: Props) { setRows(rows); setDocuments(documents); setColumns(review.columns_config || []); + // A run may still be executing server-side (e.g. after a + // refresh) — reattach to it via the resumable stream. + if (cells.some((c) => c.status === "generating")) { + resumeGenerationStream().catch((err) => + console.error("Generation resume failed", err), + ); + } }), ]; if (projectId) { @@ -279,6 +294,15 @@ export function TRView({ reviewId, projectId }: Props) { rowId, colIndex, ); + if ("status" in result) { + // HTTP 202 — the work continues in the background. Leave the + // cell "generating" and pick up the terminal state from the + // resumable stream. + resumeGenerationStream().catch((err) => + console.error("Generation resume failed", err), + ); + return; + } setCells((prev) => prev.map((c) => c.row_id === rowId && c.column_index === colIndex @@ -306,6 +330,75 @@ export function TRView({ reviewId, projectId }: Props) { } } + function getGenerationSignal(): AbortSignal { + if (!generationAbortRef.current) { + generationAbortRef.current = new AbortController(); + } + return generationAbortRef.current.signal; + } + + // Reads an SSE response and applies cell_update frames until [DONE] + async function consumeGenerationStream(response: Response) { + if (!response.body) throw new Error("No body"); + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + let finished = false; + + while (!finished) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() ?? ""; + + for (const line of lines) { + if (!line.startsWith("data:")) continue; + const dataStr = line.slice(5).trim(); + if (dataStr === "[DONE]") { + finished = true; + break; + } + try { + const data = JSON.parse(dataStr); + if (data.type === "cell_update") { + setCells((prev) => + prev.map((c) => + c.row_id === data.row_id && + c.column_index === data.column_index + ? { + ...c, + content: data.content, + status: data.status, + } + : c, + ), + ); + } + } catch {} + } + } + } + + // Reattaches to a run still executing server-side via the reconnectable + // SSE view. Guarded so only one resume stream is open at a time. + async function resumeGenerationStream() { + if (resumeStreamOpenRef.current) return; + resumeStreamOpenRef.current = true; + try { + const response = await streamTabularGenerationResume( + reviewId, + getGenerationSignal(), + ); + if (!response.ok) { + throw new Error(`Resume failed: ${response.status}`); + } + await consumeGenerationStream(response); + } finally { + resumeStreamOpenRef.current = false; + } + } + async function handleGenerate() { if (!review || generating) return; @@ -320,7 +413,10 @@ export function TRView({ reviewId, projectId }: Props) { setGenerating(true); try { - const response = await streamTabularGeneration(reviewId); + const response = await streamTabularGeneration( + reviewId, + getGenerationSignal(), + ); if (!response.ok) { const payload = await response.json().catch(() => null); const provider = @@ -369,39 +465,16 @@ export function TRView({ reviewId, projectId }: Props) { ), ); - const reader = response.body.getReader(); - const decoder = new TextDecoder(); - let buffer = ""; - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - buffer += decoder.decode(value, { stream: true }); - const lines = buffer.split("\n"); - buffer = lines.pop() ?? ""; - - for (const line of lines) { - if (!line.startsWith("data:")) continue; - const dataStr = line.slice(5).trim(); - if (dataStr === "[DONE]") break; - try { - const data = JSON.parse(dataStr); - if (data.type === "cell_update") { - setCells((prev) => - prev.map((c) => - c.row_id === data.row_id && - c.column_index === data.column_index - ? { - ...c, - content: data.content, - status: data.status, - } - : c, - ), - ); - } - } catch {} - } + try { + await consumeGenerationStream(response); + } catch (streamErr) { + // Stream dropped mid-generate — the run keeps executing + // server-side, so try one reconnect before giving up. + console.error( + "Generation stream interrupted, reconnecting", + streamErr, + ); + await resumeGenerationStream(); } } catch (err) { console.error("Generation failed", err); diff --git a/frontend/src/app/lib/mikeApi.test.ts b/frontend/src/app/lib/mikeApi.test.ts index c11935afa..4503a2645 100644 --- a/frontend/src/app/lib/mikeApi.test.ts +++ b/frontend/src/app/lib/mikeApi.test.ts @@ -893,7 +893,7 @@ describe("tabular cell operations", () => { const cell = await regenerateTabularCell("r1", "row-1", 2); - expect(cell.flag).toBe("green"); + expect(cell).toEqual({ summary: "s", flag: "green", reasoning: "r" }); const { url, init } = lastFetchCall(); expect(url).toBe( "http://localhost:3001/tabular-review/r1/regenerate-cell", diff --git a/frontend/src/app/lib/mikeApi.ts b/frontend/src/app/lib/mikeApi.ts index c143c1c8f..bbbc80577 100644 --- a/frontend/src/app/lib/mikeApi.ts +++ b/frontend/src/app/lib/mikeApi.ts @@ -828,6 +828,10 @@ export async function listStandaloneDocuments(): Promise { return apiRequest("/single-documents"); } +export async function getDocument(documentId: string): Promise { + return apiRequest(`/single-documents/${documentId}`); +} + export async function deleteDocument(documentId: string): Promise { await apiRequest(`/single-documents/${documentId}`, { method: "DELETE" }); } @@ -1198,11 +1202,24 @@ export async function deleteTabularReview(reviewId: string): Promise { export async function streamTabularGeneration( reviewId: string, + signal?: AbortSignal, ): Promise { const authHeaders = await getAuthHeader(); return fetch(`${API_BASE}/tabular-review/${reviewId}/generate`, { method: "POST", headers: { ...authHeaders }, + signal: signal ?? undefined, + }); +} + +export async function streamTabularGenerationResume( + reviewId: string, + signal?: AbortSignal, +): Promise { + const authHeaders = await getAuthHeader(); + return fetch(`${API_BASE}/tabular-review/${reviewId}/generate/stream`, { + headers: { ...authHeaders }, + signal: signal ?? undefined, }); } @@ -1323,11 +1340,15 @@ export async function regenerateTabularCell( reviewId: string, rowId: string, columnIndex: number, -): Promise<{ - summary: string; - flag: "green" | "grey" | "yellow" | "red"; - reasoning: string; -}> { +): Promise< + | { + summary: string; + flag: "green" | "grey" | "yellow" | "red"; + reasoning: string; + } + // HTTP 202 — regeneration continues in the background + | { status: "generating" } +> { return apiRequest(`/tabular-review/${reviewId}/regenerate-cell`, { method: "POST", headers: { "Content-Type": "application/json" }, From adaf14e2a4a01de315b89493452b07c43addcd7f Mon Sep 17 00:00:00 2001 From: Amal Date: Wed, 5 Aug 2026 20:29:46 -0700 Subject: [PATCH 3/3] refactor: decompose route monoliths into per-domain modules with service layers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WHY THIS MATTERS The backend's route files were monoliths — routes/tabular.ts (1,649 lines), routes/documents.ts (1,504), routes/projects.ts (1,139), routes/user.ts (1,132) — each interleaving HTTP parsing, auth checks, storage IO, and business logic inline in a dozen unrelated handlers. In a monolith, every change lands in a giant file where the blast radius is unclear, business logic can only be exercised through a live HTTP stack, and a new contributor cannot tell which lines are "the endpoint" and which are "the feature". For an open-source repo this is the difference between a drive-by contributor shipping a fix and giving up: small files with one concern each are reviewable; 1,600-line route files are not. WHAT IS A SERVICE LAYER A service layer separates WHAT the application does from HOW it is reached. The route (HTTP layer) owns request parsing, validation, and mapping results onto status codes; the service owns business logic and data access, takes its database handle as an explicit parameter, returns typed results (discriminated unions like { ok: false, kind: "not_found" } instead of writing to `res`), and never touches the HTTP request or response. That inversion is what makes logic unit-testable (call the function with a fake db — no server needed), reusable (the async extraction worker calls the exact same functions the SSE route calls), and safe to change (the compiler knows every result shape a route must handle). HOW IT WORKS - src/routes/*.ts (11 files, 7,853 lines) is replaced by src/modules// — chat, project-chat, projects, documents, tabular, user, workflows, library, downloads, case-law, models — each a thin .routes.ts plus .service.ts. Large domains split the service into topic files behind a named-re-export facade (documents: access/upload/versions/download/edits; projects: crud/folders/documents/chats; user: profile/mfa/apiKeys/mcp/account/ export; tabular: reviews/rows/extract/extractRow/generate/ generateStream/chats) so intra-module helpers cannot leak. - lib/tabular/* (from the durable-queues change this builds on) moves into modules/tabular/ — the domain's extraction core, row loaders and route layer now live together; src/lib/ keeps only cross-domain infrastructure (storage, llm, chat, queue, access...). - Streaming endpoints keep their SSE loops in the routes file; only their non-streaming prepare/persist logic moved into services — streaming lifetime and client-abort handling are HTTP concerns. - Pure motion, verified three ways: the endpoint inventory (method+path multiset, 67 endpoints) is byte-identical before and after; tsc is clean; the full suite — 510 tests, including the 11 route-level integration suites that exercise the real express app — passes unchanged. Handler bodies moved verbatim; the only rewrites are the mechanical seam (res.status(...) inside moved code became typed returns mapped back to the identical status/JSON in the route). - DRY within domains only: helpers duplicated across handlers in the same domain (shared_with normalization in projects, the doc-access guard sequence in documents, findSystemWorkflow) now have one copy in their service; similar-but-not-identical code was left alone rather than force-merged. - Zero new dependencies. No logging framework, no validation framework, no observability hooks — organization only, so the diff is reviewable as motion and each future concern can be its own decision. Re-derived against this branch's code from the fork's service-layer refactor (amal66/mike#42, running in the amal66 fork), whose module boundaries and routes/service contract this follows; the fork's pino/OTel/zod adoption was deliberately NOT ported to keep this dependency-free pure motion. Co-Authored-By: Claude Fable 5 --- backend/src/app.ts | 22 +- .../case-law/caseLaw.routes.ts} | 8 +- backend/src/modules/chat/chat.routes.ts | 375 ++++ backend/src/modules/chat/chat.service.ts | 533 ++++++ .../src/modules/documents/documents.access.ts | 142 ++ .../modules/documents/documents.download.ts | 221 +++ .../src/modules/documents/documents.edits.ts | 257 +++ .../src/modules/documents/documents.routes.ts | 550 ++++++ .../modules/documents/documents.service.ts | 56 + .../src/modules/documents/documents.shared.ts | 62 + .../src/modules/documents/documents.upload.ts | 179 ++ .../modules/documents/documents.versions.ts | 789 ++++++++ .../downloads/downloads.routes.ts} | 12 +- backend/src/modules/library/library.routes.ts | 178 ++ .../src/modules/library/library.service.ts | 377 ++++ .../models/models.routes.ts} | 4 +- .../project-chat/projectChat.routes.ts} | 185 +- .../project-chat/projectChat.service.ts | 235 +++ .../src/modules/projects/projects.chats.ts | 28 + backend/src/modules/projects/projects.crud.ts | 380 ++++ .../modules/projects/projects.documents.ts | 461 +++++ .../src/modules/projects/projects.folders.ts | 199 ++ .../src/modules/projects/projects.routes.ts | 449 +++++ .../src/modules/projects/projects.service.ts | 59 + .../src/modules/projects/projects.shared.ts | 189 ++ .../__tests__/tabular.extractRow.test.ts | 0 .../__tests__/tabular.generateStream.test.ts | 0 backend/src/modules/tabular/tabular.chats.ts | 104 ++ .../tabular/tabular.extract.ts | 14 +- .../tabular/tabular.extractRow.ts | 2 +- .../tabular/tabular.generate.ts | 6 +- .../tabular/tabular.generateStream.ts | 10 +- .../tabular/tabular.prompt.ts | 0 .../src/modules/tabular/tabular.reviews.ts | 284 +++ .../tabular/tabular.routes.ts} | 429 +---- .../{lib => modules}/tabular/tabular.rows.ts | 6 +- .../src/modules/tabular/tabular.service.ts | 50 + .../tabular/tabular.shared.ts | 4 +- backend/src/modules/user/user.account.ts | 84 + backend/src/modules/user/user.apiKeys.ts | 45 + backend/src/modules/user/user.export.ts | 63 + backend/src/modules/user/user.mcp.ts | 208 +++ backend/src/modules/user/user.mfa.ts | 72 + backend/src/modules/user/user.profile.ts | 414 +++++ backend/src/modules/user/user.routes.ts | 582 ++++++ backend/src/modules/user/user.service.ts | 76 + backend/src/modules/user/user.shared.ts | 32 + .../src/modules/workflows/workflows.routes.ts | 279 +++ .../workflows/workflows.service.ts} | 621 +++---- backend/src/routes/chat.ts | 669 ------- backend/src/routes/documents.ts | 1595 ----------------- backend/src/routes/library.ts | 414 ----- backend/src/routes/projects.ts | 1159 ------------ backend/src/routes/user.ts | 1132 ------------ .../__tests__/extractionWorker.test.ts | 4 +- backend/src/workers/extractionWorker.ts | 6 +- 56 files changed, 8441 insertions(+), 5873 deletions(-) rename backend/src/{routes/caseLaw.ts => modules/case-law/caseLaw.routes.ts} (91%) create mode 100644 backend/src/modules/chat/chat.routes.ts create mode 100644 backend/src/modules/chat/chat.service.ts create mode 100644 backend/src/modules/documents/documents.access.ts create mode 100644 backend/src/modules/documents/documents.download.ts create mode 100644 backend/src/modules/documents/documents.edits.ts create mode 100644 backend/src/modules/documents/documents.routes.ts create mode 100644 backend/src/modules/documents/documents.service.ts create mode 100644 backend/src/modules/documents/documents.shared.ts create mode 100644 backend/src/modules/documents/documents.upload.ts create mode 100644 backend/src/modules/documents/documents.versions.ts rename backend/src/{routes/downloads.ts => modules/downloads/downloads.routes.ts} (84%) create mode 100644 backend/src/modules/library/library.routes.ts create mode 100644 backend/src/modules/library/library.service.ts rename backend/src/{routes/models.ts => modules/models/models.routes.ts} (89%) rename backend/src/{routes/projectChat.ts => modules/project-chat/projectChat.routes.ts} (54%) create mode 100644 backend/src/modules/project-chat/projectChat.service.ts create mode 100644 backend/src/modules/projects/projects.chats.ts create mode 100644 backend/src/modules/projects/projects.crud.ts create mode 100644 backend/src/modules/projects/projects.documents.ts create mode 100644 backend/src/modules/projects/projects.folders.ts create mode 100644 backend/src/modules/projects/projects.routes.ts create mode 100644 backend/src/modules/projects/projects.service.ts create mode 100644 backend/src/modules/projects/projects.shared.ts rename backend/src/{lib => modules}/tabular/__tests__/tabular.extractRow.test.ts (100%) rename backend/src/{lib => modules}/tabular/__tests__/tabular.generateStream.test.ts (100%) create mode 100644 backend/src/modules/tabular/tabular.chats.ts rename backend/src/{lib => modules}/tabular/tabular.extract.ts (97%) rename backend/src/{lib => modules}/tabular/tabular.extractRow.ts (98%) rename backend/src/{lib => modules}/tabular/tabular.generate.ts (96%) rename backend/src/{lib => modules}/tabular/tabular.generateStream.ts (98%) rename backend/src/{lib => modules}/tabular/tabular.prompt.ts (100%) create mode 100644 backend/src/modules/tabular/tabular.reviews.ts rename backend/src/{routes/tabular.ts => modules/tabular/tabular.routes.ts} (78%) rename backend/src/{lib => modules}/tabular/tabular.rows.ts (96%) create mode 100644 backend/src/modules/tabular/tabular.service.ts rename backend/src/{lib => modules}/tabular/tabular.shared.ts (97%) create mode 100644 backend/src/modules/user/user.account.ts create mode 100644 backend/src/modules/user/user.apiKeys.ts create mode 100644 backend/src/modules/user/user.export.ts create mode 100644 backend/src/modules/user/user.mcp.ts create mode 100644 backend/src/modules/user/user.mfa.ts create mode 100644 backend/src/modules/user/user.profile.ts create mode 100644 backend/src/modules/user/user.routes.ts create mode 100644 backend/src/modules/user/user.service.ts create mode 100644 backend/src/modules/user/user.shared.ts create mode 100644 backend/src/modules/workflows/workflows.routes.ts rename backend/src/{routes/workflows.ts => modules/workflows/workflows.service.ts} (63%) delete mode 100644 backend/src/routes/chat.ts delete mode 100644 backend/src/routes/documents.ts delete mode 100644 backend/src/routes/library.ts delete mode 100644 backend/src/routes/projects.ts delete mode 100644 backend/src/routes/user.ts diff --git a/backend/src/app.ts b/backend/src/app.ts index f647f2df6..714ca309a 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -3,17 +3,17 @@ import express from "express"; import cors from "cors"; import helmet from "helmet"; import rateLimit from "express-rate-limit"; -import { chatRouter } from "./routes/chat"; -import { projectsRouter } from "./routes/projects"; -import { projectChatRouter } from "./routes/projectChat"; -import { documentsRouter } from "./routes/documents"; -import { libraryRouter } from "./routes/library"; -import { tabularRouter } from "./routes/tabular"; -import { workflowsRouter } from "./routes/workflows"; -import { userRouter } from "./routes/user"; -import { modelsRouter } from "./routes/models"; -import { downloadsRouter } from "./routes/downloads"; -import { caseLawRouter } from "./routes/caseLaw"; +import { chatRouter } from "./modules/chat/chat.routes"; +import { projectsRouter } from "./modules/projects/projects.routes"; +import { projectChatRouter } from "./modules/project-chat/projectChat.routes"; +import { documentsRouter } from "./modules/documents/documents.routes"; +import { libraryRouter } from "./modules/library/library.routes"; +import { tabularRouter } from "./modules/tabular/tabular.routes"; +import { workflowsRouter } from "./modules/workflows/workflows.routes"; +import { userRouter } from "./modules/user/user.routes"; +import { modelsRouter } from "./modules/models/models.routes"; +import { downloadsRouter } from "./modules/downloads/downloads.routes"; +import { caseLawRouter } from "./modules/case-law/caseLaw.routes"; import { manifestPublicKey } from "./lib/manifestSigning"; import { safeErrorLog } from "./lib/safeError"; diff --git a/backend/src/routes/caseLaw.ts b/backend/src/modules/case-law/caseLaw.routes.ts similarity index 91% rename from backend/src/routes/caseLaw.ts rename to backend/src/modules/case-law/caseLaw.routes.ts index 4be389858..7ad5a8896 100644 --- a/backend/src/routes/caseLaw.ts +++ b/backend/src/modules/case-law/caseLaw.routes.ts @@ -1,8 +1,8 @@ import { Router } from "express"; -import { requireAuth } from "../middleware/auth"; -import { getCourtlistenerCaseOpinions } from "../lib/courtlistener"; -import { createServerSupabase } from "../lib/supabase"; -import { getUserModelSettings } from "../lib/userSettings"; +import { requireAuth } from "../../middleware/auth"; +import { getCourtlistenerCaseOpinions } from "../../lib/courtlistener"; +import { createServerSupabase } from "../../lib/supabase"; +import { getUserModelSettings } from "../../lib/userSettings"; export const caseLawRouter = Router(); diff --git a/backend/src/modules/chat/chat.routes.ts b/backend/src/modules/chat/chat.routes.ts new file mode 100644 index 000000000..cc816b469 --- /dev/null +++ b/backend/src/modules/chat/chat.routes.ts @@ -0,0 +1,375 @@ +// HTTP layer for the chat module. +// +// Route handlers parse params/query/body, call the chat.service functions, +// and map their typed results onto status codes and JSON. The SSE streaming +// loop for POST /chat (header flush, runLLMStream, abort handling, +// assistant-message persistence) stays here — its ordering is delicate; the +// pre-stream preparation lives in chat.service.ts. + +import { Router } from "express"; +import { requireAuth } from "../../middleware/auth"; +import { createServerSupabase } from "../../lib/supabase"; +import { + appendAssistantEventsToLastAssistantMessage, + AssistantStreamError, + buildCancelledAssistantMessage, + extractCitations, + isAbortError, + runLLMStream, + stripTransientAssistantEvents, + parseChatMessages, + parseOptionalAskInputsResponse, + parseOptionalChatId, + parseOptionalModel, + parseOptionalProjectId, +} from "../../lib/chat"; +import { safeErrorLog, safeErrorMessage } from "../../lib/safeError"; +import { + createChat, + deleteChat, + devLog, + generateChatTitle, + getChatWithMessages, + listChats, + prepareChatStream, + updateChatTitle, +} from "./chat.service"; + +export const chatRouter = Router(); + +// GET /chat +// Visible chats = the user's own chats + every chat under a project the +// user owns (so a project owner sees all collaborator chats in their +// own projects in the global recent-chats list). Chats in projects that +// are merely *shared with* the user are NOT included here — those are +// listed per-project via GET /projects/:projectId/chats. +chatRouter.get("/", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const db = createServerSupabase(); + const requestedLimit = Number.parseInt(String(req.query.limit ?? ""), 10); + const limit = Number.isFinite(requestedLimit) + ? Math.min(Math.max(requestedLimit, 1), 100) + : null; + + const result = await listChats(db, { userId, limit }); + if (!result.ok) + return void res.status(500).json({ detail: result.detail }); + res.json(result.data); +}); + +// POST /chat/create +chatRouter.post("/create", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const parsedProjectId = parseOptionalProjectId(req.body?.project_id); + if (!parsedProjectId.ok) { + return void res.status(400).json({ detail: parsedProjectId.detail }); + } + const projectId = parsedProjectId.value.projectId; + const db = createServerSupabase(); + + const result = await createChat(db, { userId, userEmail, projectId }); + if (!result.ok) + return void res + .status(result.status) + .json({ detail: result.detail }); + res.json({ id: result.id }); +}); + +// GET /chat/:chatId +chatRouter.get("/:chatId", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { chatId } = req.params; + const db = createServerSupabase(); + + const result = await getChatWithMessages(db, { chatId, userId, userEmail }); + if (!result.ok) + return void res.status(404).json({ detail: "Chat not found" }); + res.json({ chat: result.chat, messages: result.messages }); +}); + +// PATCH /chat/:chatId +chatRouter.patch("/:chatId", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const { chatId } = req.params; + const title = (req.body.title ?? "").trim(); + if (!title) + return void res.status(400).json({ detail: "title is required" }); + + const db = createServerSupabase(); + const result = await updateChatTitle(db, { chatId, userId, title }); + if (!result.ok) + return void res.status(404).json({ detail: "Chat not found" }); + res.json(result.data); +}); + +// DELETE /chat/:chatId +chatRouter.delete("/:chatId", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const { chatId } = req.params; + const db = createServerSupabase(); + const result = await deleteChat(db, { chatId, userId }); + if (!result.ok) + return void res.status(500).json({ detail: result.detail }); + res.status(204).send(); +}); + +// POST /chat/:chatId/generate-title +chatRouter.post("/:chatId/generate-title", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { chatId } = req.params; + const message = + typeof req.body?.message === "string" ? req.body.message.trim() : ""; + if (!message) + return void res.status(400).json({ detail: "message is required" }); + + const db = createServerSupabase(); + const result = await generateChatTitle(db, { + chatId, + userId, + userEmail, + message, + }); + if (!result.ok) { + if (result.kind === "not_found") + return void res.status(404).json({ detail: "Chat not found" }); + return void res + .status(500) + .json({ detail: "Failed to generate title" }); + } + res.json({ title: result.title }); +}); + +// POST /chat — streaming +chatRouter.post("/", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const body = + req.body && typeof req.body === "object" && !Array.isArray(req.body) + ? (req.body as Record) + : {}; + const parsedMessages = parseChatMessages(body.messages); + if (!parsedMessages.ok) { + return void res.status(400).json({ detail: parsedMessages.detail }); + } + const parsedChatId = parseOptionalChatId(body.chat_id); + if (!parsedChatId.ok) { + return void res.status(400).json({ detail: parsedChatId.detail }); + } + const parsedProjectId = parseOptionalProjectId(body.project_id); + if (!parsedProjectId.ok) { + return void res.status(400).json({ detail: parsedProjectId.detail }); + } + const parsedModel = parseOptionalModel(body.model); + if (!parsedModel.ok) { + return void res.status(400).json({ detail: parsedModel.detail }); + } + const parsedAskInputsResponse = parseOptionalAskInputsResponse( + body.ask_inputs_response, + ); + if (!parsedAskInputsResponse.ok) { + return void res + .status(400) + .json({ detail: parsedAskInputsResponse.detail }); + } + + const messages = parsedMessages.value; + const chat_id = parsedChatId.value; + const project_id = parsedProjectId.value.projectId; + const model = parsedModel.value; + const askInputsResponse = parsedAskInputsResponse.value; + + devLog("[chat/stream] incoming request", { + userId, + chat_id, + project_id, + model, + messageCount: messages?.length, + }); + + const userEmail = res.locals.userEmail as string | undefined; + const db = createServerSupabase(); + + const prep = await prepareChatStream(db, { + userId, + userEmail, + messages, + chatId: chat_id ?? null, + projectIdProvided: parsedProjectId.value.provided, + projectId: parsedProjectId.value.projectId, + askInputsResponse, + }); + if (!prep.ok) + return void res.status(prep.status).json({ detail: prep.detail }); + + const { + chatId, + chatTitle, + lastUser, + resolvedProjectId, + docIndex, + docStore, + apiMessages, + workflowStore, + legalResearchUs, + apiKeys, + nonce, + } = prep.prepared; + + devLog("[chat/stream] starting LLM stream", { + apiMessageCount: apiMessages.length, + docCount: Object.keys(docIndex).length, + workflowCount: Object.keys(workflowStore).length, + }); + + res.setHeader("Content-Type", "text/event-stream"); + res.setHeader("Cache-Control", "no-cache"); + res.setHeader("Connection", "keep-alive"); + res.setHeader("X-Accel-Buffering", "no"); + res.flushHeaders(); + + const write = (line: string) => res.write(line); + const streamAbort = new AbortController(); + let streamFinished = false; + res.on("close", () => { + if (!streamFinished) streamAbort.abort(); + }); + + try { + write(`data: ${JSON.stringify({ type: "chat_id", chatId })}\n\n`); + + const { fullText, events, citations } = await runLLMStream({ + apiMessages, + docStore, + docIndex, + userId, + db, + write, + workflowStore, + includeResearchTools: legalResearchUs, + model, + apiKeys, + signal: streamAbort.signal, + projectId: resolvedProjectId, + nonce, + }); + + devLog("[chat/stream] LLM stream finished", { + fullTextLen: fullText?.length ?? 0, + eventCount: events?.length ?? 0, + }); + + const persistedEvents = stripTransientAssistantEvents(events); + if (askInputsResponse) { + await appendAssistantEventsToLastAssistantMessage( + db, + chatId, + persistedEvents, + citations, + ); + } else { + await db.from("chat_messages").insert({ + chat_id: chatId, + role: "assistant", + content: persistedEvents.length ? persistedEvents : null, + citations: citations.length ? citations : null, + }); + } + + if (!chatTitle && lastUser?.content) { + await db + .from("chats") + .update({ title: lastUser.content.slice(0, 120) }) + .eq("id", chatId); + } + } catch (err) { + if (isAbortError(err)) { + devLog("[chat/stream] client aborted stream", { chatId }); + if (err instanceof AssistantStreamError) { + const partial = buildCancelledAssistantMessage({ + fullText: err.fullText, + events: err.events, + buildCitations: (fullText, events) => + extractCitations(fullText, docIndex, events), + }); + const saveError = askInputsResponse + ? null + : ( + await db.from("chat_messages").insert({ + chat_id: chatId, + role: "assistant", + content: partial.events.length + ? partial.events + : null, + citations: partial.citations.length + ? partial.citations + : null, + }) + ).error; + if (askInputsResponse) { + await appendAssistantEventsToLastAssistantMessage( + db, + chatId, + partial.events, + partial.citations, + ); + } + if (saveError) { + console.error( + "[chat/stream] failed to save aborted stream", + saveError, + ); + } + } + return; + } + console.error("[chat/stream] error:", safeErrorLog(err)); + const message = safeErrorMessage(err, "Stream error"); + const errorEvents = err instanceof AssistantStreamError + ? stripTransientAssistantEvents(err.events) + : [{ type: "error" as const, message }]; + const errorFullText = + err instanceof AssistantStreamError ? err.fullText : ""; + try { + const citations = extractCitations( + errorFullText, + docIndex, + errorEvents, + ); + const saveError = askInputsResponse + ? null + : ( + await db.from("chat_messages").insert({ + chat_id: chatId, + role: "assistant", + content: errorEvents.length ? errorEvents : null, + citations: citations.length ? citations : null, + }) + ).error; + if (askInputsResponse) { + await appendAssistantEventsToLastAssistantMessage( + db, + chatId, + errorEvents, + citations, + ); + } + if (saveError) + console.error("[chat/stream] failed to save error", saveError); + } catch (saveErr) { + console.error("[chat/stream] failed to save error", saveErr); + } + try { + write( + `data: ${JSON.stringify({ type: "error", message })}\n\n`, + ); + write("data: [DONE]\n\n"); + } catch { + /* ignore */ + } + } finally { + streamFinished = true; + res.end(); + } +}); diff --git a/backend/src/modules/chat/chat.service.ts b/backend/src/modules/chat/chat.service.ts new file mode 100644 index 000000000..b4f4583e8 --- /dev/null +++ b/backend/src/modules/chat/chat.service.ts @@ -0,0 +1,533 @@ +// Business logic + data-access for the chat module. +// +// These functions are the service layer behind chat.routes.ts. They take an +// explicit Supabase client (`db`) plus request-derived primitives, perform the +// chat orchestration / DB work, and RETURN values or typed error results. They +// never touch req/res — the thin route handlers map the results onto HTTP +// status codes, headers, and response bodies. +// +// IMPORTANT: the SSE streaming loop (header flush, runLLMStream, abort +// handling, assistant-message persistence) deliberately stays in the route — +// its ordering is delicate. Only the NON-streaming logic and the pre-stream +// DB preparation live here. `prepareChatStream` returns the prepared data the +// route needs to run the stream; it does not stream. + +import { createServerSupabase } from "../../lib/supabase"; +import { + buildDocContext, + buildMessages, + enrichWithPriorEvents, + buildWorkflowStore, + appendAskInputsResponseToLastAssistantMessage, + generateSpotlightNonce, + type AskInputsResponseRequest, + type ChatMessage, +} from "../../lib/chat"; +import { completeText } from "../../lib/llm"; +import { + getUserModelSettings, +} from "../../lib/userSettings"; +import { checkProjectAccess } from "../../lib/access"; +import { safeErrorLog } from "../../lib/safeError"; + +type Db = ReturnType; + +const isDev = process.env.NODE_ENV !== "production"; +export const devLog = (...args: Parameters) => { + if (isDev) console.log(...args); +}; + +const TITLE_FALLBACK = "Misc. Query"; + +function normalizeGeneratedTitle(raw: string): string { + const title = raw.trim().replace(/^["'`]+|["'`.,:;!?]+$/g, "").trim(); + if (!title) return TITLE_FALLBACK; + return title.slice(0, 80); +} + +type AccessibleChat = { + id: string; + title: string | null; + user_id: string; + project_id: string | null; +} & Record; + +async function validateAccessibleProjectId( + projectId: string | null, + userId: string, + userEmail: string | null | undefined, + db: Db, +): Promise<{ ok: true } | { ok: false; status: number; detail: string }> { + if (!projectId) return { ok: true }; + const access = await checkProjectAccess(projectId, userId, userEmail, db); + if (!access.ok) + return { ok: false, status: 404, detail: "Project not found" }; + return { ok: true }; +} + +async function getAccessibleChat( + chatId: string, + userId: string, + userEmail: string | null | undefined, + db: Db, +): Promise { + const { data: chat, error } = await db + .from("chats") + .select("*") + .eq("id", chatId) + .maybeSingle(); + if (error || !chat) return null; + + const row = chat as AccessibleChat; + if (row.user_id === userId) return row; + + if (row.project_id) { + const access = await checkProjectAccess( + row.project_id, + userId, + userEmail, + db, + ); + if (access.ok) return row; + } + + return null; +} + +// Stored doc_edited events capture the `status` at the time the assistant +// produced the edit (always "pending"). If the user later accepts or rejects, +// `document_edits.status` is updated but the stored event is not. On chat load +// we merge the current DB status in so EditCards render with the real state. +async function hydrateEditStatuses( + messages: Record[], + db: ReturnType, +): Promise[]> { + const editIds = new Set(); + const versionIds = new Set(); + const collectFromAnnList = (list: unknown) => { + if (!Array.isArray(list)) return; + for (const a of list as Record[]) { + if (typeof a?.edit_id === "string") editIds.add(a.edit_id); + if (typeof a?.version_id === "string") + versionIds.add(a.version_id); + } + }; + for (const m of messages) { + const content = m.content; + if (Array.isArray(content)) { + for (const ev of content as Record[]) { + if (ev?.type === "doc_edited") { + collectFromAnnList(ev.annotations); + if (typeof ev.version_id === "string") + versionIds.add(ev.version_id); + } + } + } + } + if (editIds.size === 0 && versionIds.size === 0) return messages; + + // Edit status patch. + const statusById = new Map(); + if (editIds.size > 0) { + const { data: rows } = await db + .from("document_edits") + .select("id, status") + .in("id", Array.from(editIds)); + for (const r of (rows ?? []) as { id: string; status: string }[]) { + if ( + r.status === "pending" || + r.status === "accepted" || + r.status === "rejected" + ) { + statusById.set(r.id, r.status); + } + } + } + + // Version-number patch — old stored events don't carry `version_number` + // because they predate the schema change. Look it up from + // document_versions so the UI can render "V3" chips + download filenames. + const versionNumberById = new Map(); + if (versionIds.size > 0) { + const { data: vrows } = await db + .from("document_versions") + .select("id, version_number") + .in("id", Array.from(versionIds)); + for (const r of (vrows ?? []) as { + id: string; + version_number: number | null; + }[]) { + versionNumberById.set(r.id, r.version_number ?? null); + } + } + + const patchAnnList = (list: unknown): unknown => { + if (!Array.isArray(list)) return list; + return (list as Record[]).map((a) => { + let next = a; + if (typeof a?.edit_id === "string" && statusById.has(a.edit_id)) { + next = { ...next, status: statusById.get(a.edit_id) }; + } + if ( + typeof a?.version_id === "string" && + versionNumberById.has(a.version_id) + ) { + next = { + ...next, + version_number: versionNumberById.get(a.version_id) ?? null, + }; + } + return next; + }); + }; + return messages.map((m) => { + const next: Record = { ...m }; + if (Array.isArray(m.content)) { + next.content = (m.content as Record[]).map( + (ev) => { + if (ev?.type !== "doc_edited") return ev; + let patched: Record = { + ...ev, + annotations: patchAnnList(ev.annotations), + }; + if ( + typeof ev.version_id === "string" && + versionNumberById.has(ev.version_id) + ) { + patched = { + ...patched, + version_number: + versionNumberById.get(ev.version_id) ?? null, + }; + } + return patched; + }, + ); + } + return next; + }); +} + +// --------------------------------------------------------------------------- +// Non-streaming endpoints +// --------------------------------------------------------------------------- + +// GET /chat +export async function listChats( + db: Db, + args: { userId: string; limit: number | null }, +): Promise<{ ok: true; data: unknown[] } | { ok: false; detail: string }> { + const { data, error } = await db.rpc("get_chats_overview", { + p_user_id: args.userId, + p_limit: args.limit, + }); + if (error) return { ok: false, detail: error.message }; + return { ok: true, data: data ?? [] }; +} + +// POST /chat/create +export async function createChat( + db: Db, + args: { + userId: string; + userEmail: string | undefined; + projectId: string | null; + }, +): Promise< + { ok: true; id: string } | { ok: false; status: number; detail: string } +> { + const projectAccess = await validateAccessibleProjectId( + args.projectId, + args.userId, + args.userEmail, + db, + ); + if (!projectAccess.ok) + return { + ok: false, + status: projectAccess.status, + detail: projectAccess.detail, + }; + + const { data, error } = await db + .from("chats") + .insert({ user_id: args.userId, project_id: args.projectId ?? null }) + .select("id") + .single(); + + if (error) return { ok: false, status: 500, detail: error.message }; + return { ok: true, id: data.id }; +} + +// GET /chat/:chatId +export async function getChatWithMessages( + db: Db, + args: { chatId: string; userId: string; userEmail: string | undefined }, +): Promise< + | { ok: true; chat: AccessibleChat; messages: Record[] } + | { ok: false } +> { + const chat = await getAccessibleChat( + args.chatId, + args.userId, + args.userEmail, + db, + ); + if (!chat) return { ok: false }; + + const { data: messages } = await db + .from("chat_messages") + .select("*") + .eq("chat_id", args.chatId) + .order("created_at", { ascending: true }); + + const hydrated = await hydrateEditStatuses(messages ?? [], db); + return { ok: true, chat, messages: hydrated }; +} + +// PATCH /chat/:chatId +export async function updateChatTitle( + db: Db, + args: { chatId: string; userId: string; title: string }, +): Promise<{ ok: true; data: { id: string; title: string } } | { ok: false }> { + const { data, error } = await db + .from("chats") + .update({ title: args.title }) + .eq("id", args.chatId) + .eq("user_id", args.userId) + .select("id, title") + .single(); + + if (error || !data) return { ok: false }; + return { ok: true, data }; +} + +// DELETE /chat/:chatId +export async function deleteChat( + db: Db, + args: { chatId: string; userId: string }, +): Promise<{ ok: true } | { ok: false; detail: string }> { + const { error } = await db + .from("chats") + .delete() + .eq("id", args.chatId) + .eq("user_id", args.userId); + + if (error) return { ok: false, detail: error.message }; + return { ok: true }; +} + +// POST /chat/:chatId/generate-title +export async function generateChatTitle( + db: Db, + args: { + chatId: string; + userId: string; + userEmail: string | undefined; + message: string; + }, +): Promise< + | { ok: true; title: string } + | { ok: false; kind: "not_found" } + | { ok: false; kind: "error" } +> { + const chat = await getAccessibleChat( + args.chatId, + args.userId, + args.userEmail, + db, + ); + if (!chat) return { ok: false, kind: "not_found" }; + + try { + const { title_model, api_keys } = await getUserModelSettings( + args.userId, + db, + ); + const titleText = await completeText({ + model: title_model, + user: `Generate a concise title (3–6 words) for a chat in an AI Legal Platform that starts with this message. The title should describe the topic or document — do NOT include words like "Legal Assistant", "AI", "Chat", or any similar prefix. If there is not enough information to generate a title, return exactly "${TITLE_FALLBACK}". Return only the title, no quotes or punctuation.\n\nMessage: ${args.message.slice(0, 500)}`, + maxTokens: 64, + apiKeys: api_keys, + }); + const title = normalizeGeneratedTitle(titleText); + + await db + .from("chats") + .update({ title }) + .eq("id", args.chatId); + + return { ok: true, title }; + } catch (err) { + console.error("[generate-title]", safeErrorLog(err)); + return { ok: false, kind: "error" }; + } +} + +// --------------------------------------------------------------------------- +// Pre-stream preparation for POST /chat (streaming) +// --------------------------------------------------------------------------- +// +// This performs the DB work that precedes the SSE stream: resolving or creating +// the chat, persisting the user message, building doc context + messages, and +// assembling the workflow store. It RETURNS the prepared data; the route owns +// the header flush, runLLMStream loop, and persistence. + +export type PreparedChatStream = { + chatId: string; + chatTitle: string | null; + lastUser: ChatMessage | undefined; + resolvedProjectId: string | null; + docIndex: Awaited>["docIndex"]; + docStore: Awaited>["docStore"]; + apiMessages: ReturnType; + workflowStore: Awaited>; + legalResearchUs: boolean; + apiKeys: Awaited>["api_keys"]; + nonce: ReturnType; +}; + +export async function prepareChatStream( + db: Db, + args: { + userId: string; + userEmail: string | undefined; + messages: ChatMessage[]; + chatId: string | null; + projectIdProvided: boolean; + projectId: string | null; + // Parsed `ask_inputs_response` payload (answers to an ask_inputs + // event emitted by the assistant in a prior turn). When present, the + // user's answers are appended onto the previous assistant message + // instead of being stored as a new user message. + askInputsResponse: AskInputsResponseRequest | null; + }, +): Promise< + | { ok: true; prepared: PreparedChatStream } + | { ok: false; status: number; detail: string } +> { + const { userId, userEmail, messages } = args; + let chatId = args.chatId; + let chatTitle: string | null = null; + let resolvedProjectId: string | null = args.projectId; + + if (chatId) { + const existing = await getAccessibleChat(chatId, userId, userEmail, db); + if (!existing) + return { ok: false, status: 404, detail: "Chat not found" }; + + const existingProjectId = existing.project_id ?? null; + if ( + args.projectIdProvided && + args.projectId !== existingProjectId + ) { + return { + ok: false, + status: 400, + detail: "project_id does not match chat", + }; + } + resolvedProjectId = existingProjectId; + chatTitle = existing.title; + } + + if (!chatId) { + // If creating a chat tied to a project, the user must have access + // to the project (own or shared). + const projectAccess = await validateAccessibleProjectId( + resolvedProjectId, + userId, + userEmail, + db, + ); + if (!projectAccess.ok) + return { + ok: false, + status: projectAccess.status, + detail: projectAccess.detail, + }; + + const { data: newChat, error } = await db + .from("chats") + .insert({ user_id: userId, project_id: resolvedProjectId }) + .select("id, title") + .single(); + if (error || !newChat) { + console.error("[chat/stream] failed to create chat", error); + return { ok: false, status: 500, detail: "Failed to create chat" }; + } + chatId = newChat.id as string; + chatTitle = newChat.title; + } + + devLog("[chat/stream] resolved chatId", chatId); + + const lastUser = [...messages].reverse().find((m) => m.role === "user"); + if (args.askInputsResponse) { + await appendAskInputsResponseToLastAssistantMessage( + db, + chatId, + args.askInputsResponse, + ); + } else if (lastUser) { + await db.from("chat_messages").insert({ + chat_id: chatId, + role: "user", + content: lastUser.content, + files: lastUser.files ?? null, + workflow: lastUser.workflow ?? null, + }); + } + + const { docIndex, docStore } = await buildDocContext( + messages, + userId, + db, + chatId, + ); + const docAvailability = Object.entries(docIndex).map(([doc_id, info]) => ({ + doc_id, + filename: info.filename, + })); + // Generate the nonce before enriching prior events so document filenames + // and workflow titles replayed from earlier turns are fenced as well. + const nonce = generateSpotlightNonce(); + const enrichedMessages = await enrichWithPriorEvents( + messages, + chatId, + db, + docIndex, + nonce, + ); + const { + api_keys: apiKeys, + legal_research_us: legalResearchUs, + } = await getUserModelSettings(userId, db); + const apiMessages = buildMessages( + enrichedMessages, + docAvailability, + undefined, + undefined, + legalResearchUs, + nonce, + ); + + const workflowStore = await buildWorkflowStore(userId, userEmail, db); + + return { + ok: true, + prepared: { + chatId, + chatTitle, + lastUser, + resolvedProjectId, + docIndex, + docStore, + apiMessages, + workflowStore, + legalResearchUs, + apiKeys, + nonce, + }, + }; +} diff --git a/backend/src/modules/documents/documents.access.ts b/backend/src/modules/documents/documents.access.ts new file mode 100644 index 000000000..b1d89f0b1 --- /dev/null +++ b/backend/src/modules/documents/documents.access.ts @@ -0,0 +1,142 @@ +// Document access guards plus the list/delete operations that are pure +// row-level concerns (no version/storage orchestration beyond cleanup). + +import { + attachActiveVersionPaths, + attachLatestVersionNumbers, +} from "../../lib/documentVersions"; +import { ensureDocAccess } from "../../lib/access"; +import { deleteDocumentAndVersionFiles, type Db } from "./documents.shared"; + +type DocRow = { + id: string; + user_id: string; + project_id: string | null; + current_version_id?: string | null; +}; + +/** + * Load a document row and verify the caller can access it. Returns the row + * (with whatever columns `select` requested) and the owner flag, or + * `{ ok: false }` when the document is missing / inaccessible / (when + * `ownerOnly`) not owned by the caller. + */ +export async function ensureDocumentAccess( + documentId: string, + userId: string, + userEmail: string | undefined, + db: Db, + opts: { select?: string; ownerOnly?: boolean } = {}, +): Promise<{ ok: true; doc: DocRow; isOwner: boolean } | { ok: false }> { + const { data: doc } = await db + .from("documents") + .select(opts.select ?? "id, user_id, project_id") + .eq("id", documentId) + .single(); + if (!doc) return { ok: false }; + // `select` is a dynamic string, so supabase-js can't derive the row type. + const d = doc as unknown as DocRow; + const access = await ensureDocAccess(d, userId, userEmail, db); + if (!access.ok) return { ok: false }; + if (opts.ownerOnly && !access.isOwner) return { ok: false }; + return { ok: true, doc: d, isOwner: access.isOwner }; +} + +/** + * Boolean access guard for route handlers that interleave the access check + * with HTTP-layer validation (file presence, extension checks) and therefore + * run the check inline rather than inside a higher-level service function. + */ +export async function checkDocumentAccess( + documentId: string, + userId: string, + userEmail: string | undefined, + db: Db, + opts: { select?: string; ownerOnly?: boolean } = {}, +): Promise { + const access = await ensureDocumentAccess( + documentId, + userId, + userEmail, + db, + opts, + ); + return access.ok; +} + +// --------------------------------------------------------------------------- +// List +// --------------------------------------------------------------------------- + +export async function listSingleDocuments( + userId: string, + db: Db, +): Promise< + | { ok: true; docs: { id: string; current_version_id?: string | null }[] } + | { ok: false; detail: string } +> { + const { data, error } = await db + .from("documents") + .select("*") + .eq("user_id", userId) + .is("project_id", null) + .or("library_kind.eq.file,library_kind.is.null") + .order("created_at", { ascending: false }); + if (error) return { ok: false, detail: error.message }; + const docs = (data ?? []) as unknown as { + id: string; + current_version_id?: string | null; + }[]; + await attachLatestVersionNumbers(db, docs); + await attachActiveVersionPaths(db, docs); + return { ok: true, docs }; +} + +/** + * One document, same shape as a list entry. Exists so the client can poll a + * single document's status while a deferred conversion runs, instead of + * refetching the whole collection. + */ +export async function getDocument( + documentId: string, + userId: string, + userEmail: string | undefined, + db: Db, +): Promise< + | { ok: true; doc: Record } + | { ok: false; kind: "not_found" } +> { + const access = await ensureDocumentAccess(documentId, userId, userEmail, db, { + select: "*", + }); + if (!access.ok) return { ok: false, kind: "not_found" }; + + const docs = [access.doc] as unknown as { + id: string; + current_version_id?: string | null; + }[]; + await attachLatestVersionNumbers(db, docs); + await attachActiveVersionPaths(db, docs); + return { ok: true, doc: docs[0] as unknown as Record }; +} + +// --------------------------------------------------------------------------- +// Delete document +// --------------------------------------------------------------------------- + +export async function deleteDocument( + documentId: string, + userId: string, + db: Db, +): Promise<{ ok: true } | { ok: false }> { + const { data: doc, error } = await db + .from("documents") + .select("id") + .eq("id", documentId) + .eq("user_id", userId) + .single(); + if (error || !doc) return { ok: false }; + + await deleteDocumentAndVersionFiles(db, documentId); + return { ok: true }; +} diff --git a/backend/src/modules/documents/documents.download.ts b/backend/src/modules/documents/documents.download.ts new file mode 100644 index 000000000..30089da4f --- /dev/null +++ b/backend/src/modules/documents/documents.download.ts @@ -0,0 +1,221 @@ +// Read/serve paths for documents: inline display bytes, zip bundling, signed +// download URLs, and raw DOCX bytes. + +import { downloadFile, getSignedUrl } from "../../lib/storage"; +import { loadActiveVersion } from "../../lib/documentVersions"; +import { ensureDocAccess } from "../../lib/access"; +import { + contentTypeForDocumentType, + shouldConvertToPdf, +} from "../../lib/documentTypes"; +import { downloadFilenameForVersion, type Db } from "./documents.shared"; +import { ensureDocumentAccess } from "./documents.access"; + +// --------------------------------------------------------------------------- +// Display +// --------------------------------------------------------------------------- + +/** + * Resolve the bytes + content-type to serve inline for a document's display + * view. The route sets the headers and sends `bytes`. All failures here map + * to 404 in the route, so we return the exact detail strings. + */ +export async function getDisplayableVersion( + documentId: string, + userId: string, + userEmail: string, + versionIdParam: string | null, + db: Db, +): Promise< + | { ok: true; bytes: ArrayBuffer; contentType: string; filename: string } + | { ok: false; detail: string } +> { + const access = await ensureDocumentAccess(documentId, userId, userEmail, db); + if (!access.ok) return { ok: false, detail: "Document not found" }; + + const active = await loadActiveVersion(documentId, db, versionIdParam); + if (!active) return { ok: false, detail: "No file available" }; + + const fileType = active.file_type ?? ""; + const isConvertibleOffice = shouldConvertToPdf(fileType); + const displayFilename = downloadFilenameForVersion( + active.filename, + active.version_number, + active.source === "assistant_edit", + ); + + // For Office files, prefer the per-version PDF rendition if one exists. + const servePath = + isConvertibleOffice && active.pdf_storage_path + ? active.pdf_storage_path + : active.storage_path; + const raw = await downloadFile(servePath); + if (!raw) return { ok: false, detail: "Document not found in storage" }; + + if (fileType === "pdf" || (isConvertibleOffice && active.pdf_storage_path)) { + return { + ok: true, + bytes: raw, + contentType: "application/pdf", + filename: displayFilename, + }; + } else { + // Fallback: serve raw Office bytes when PDF conversion was unavailable. + return { + ok: true, + bytes: raw, + contentType: contentTypeForDocumentType(fileType), + filename: displayFilename, + }; + } +} + +// --------------------------------------------------------------------------- +// Download zip +// --------------------------------------------------------------------------- + +/** + * Build the zip archive for the given document ids, filtered to those the + * caller can access. The route validates the id list, sets the headers, and + * sends the returned buffer. + */ +export async function buildZipForDocuments( + documentIds: string[], + userId: string, + userEmail: string | undefined, + db: Db, +): Promise< + | { ok: true; content: Buffer } + | { ok: false; kind: "db"; detail: string } + | { ok: false; kind: "empty" } +> { + const { data: rawDocs, error } = await db + .from("documents") + .select("id, current_version_id, user_id, project_id") + .in("id", documentIds); + + if (error) return { ok: false, kind: "db", detail: error.message }; + // Filter to docs the user actually has access to (own + shared-project). + const accessChecks = await Promise.all( + (rawDocs ?? []).map(async (d) => ({ + doc: d, + access: await ensureDocAccess( + d as { user_id: string; project_id: string | null }, + userId, + userEmail, + db, + ), + })), + ); + const docs = accessChecks + .filter((x) => x.access.ok) + .map((x) => x.doc as { id: string }); + if (!docs || docs.length === 0) return { ok: false, kind: "empty" }; + + const JSZip = (await import("jszip")).default; + const zip = new JSZip(); + + await Promise.all( + docs.map(async (doc) => { + const active = await loadActiveVersion(doc.id, db); + if (!active) return; + const raw = await downloadFile(active.storage_path); + if (!raw) return; + zip.file( + downloadFilenameForVersion( + active.filename, + active.version_number, + active.source === "assistant_edit", + ), + Buffer.from(raw), + ); + }), + ); + + const content = await zip.generateAsync({ type: "nodebuffer", compression: "DEFLATE" }); + return { ok: true, content }; +} + +// --------------------------------------------------------------------------- +// Signed download URL +// --------------------------------------------------------------------------- + +export async function getDownloadUrl( + documentId: string, + userId: string, + userEmail: string | undefined, + versionIdParam: string | null, + db: Db, +): Promise< + | { ok: true; payload: Record } + | { ok: false; kind: "not_found"; detail: string } + | { ok: false; kind: "storage"; detail: string } +> { + const access = await ensureDocumentAccess(documentId, userId, userEmail, db); + if (!access.ok) + return { ok: false, kind: "not_found", detail: "Document not found" }; + + const active = await loadActiveVersion(documentId, db, versionIdParam); + if (!active) + return { ok: false, kind: "not_found", detail: "No file available" }; + + const downloadFilename = downloadFilenameForVersion( + active.filename, + active.version_number, + active.source === "assistant_edit", + ); + const url = await getSignedUrl( + active.storage_path, + 3600, + downloadFilename, + ); + if (!url) + return { ok: false, kind: "storage", detail: "Storage not configured" }; + + return { + ok: true, + payload: { + url, + document_id: documentId, + filename: downloadFilename, + version_id: active.id, + // Lets the frontend decide between DocView (PDF.js) and DocxView + // (docx-preview) without a follow-up round-trip. + has_pdf_rendition: !!active.pdf_storage_path, + }, + }; +} + +// --------------------------------------------------------------------------- +// Raw DOCX bytes +// --------------------------------------------------------------------------- + +export async function getDocxBytes( + documentId: string, + userId: string, + userEmail: string | undefined, + versionIdParam: string | null, + db: Db, +): Promise< + | { ok: true; bytes: ArrayBuffer; filename: string } + | { ok: false; detail: string } +> { + const access = await ensureDocumentAccess(documentId, userId, userEmail, db); + if (!access.ok) return { ok: false, detail: "Document not found" }; + + const active = await loadActiveVersion(documentId, db, versionIdParam); + if (!active) return { ok: false, detail: "No file available" }; + + const raw = await downloadFile(active.storage_path); + if (!raw) return { ok: false, detail: "Document bytes not available" }; + + return { + ok: true, + bytes: raw, + filename: downloadFilenameForVersion( + active.filename, + active.version_number, + active.source === "assistant_edit", + ), + }; +} diff --git a/backend/src/modules/documents/documents.edits.ts b/backend/src/modules/documents/documents.edits.ts new file mode 100644 index 000000000..187af77c4 --- /dev/null +++ b/backend/src/modules/documents/documents.edits.ts @@ -0,0 +1,257 @@ +// Tracked-change (assistant edit) operations: listing change ids embedded in +// the active DOCX and accepting / rejecting an individual edit. + +import { downloadFile, uploadFile } from "../../lib/storage"; +import { + extractTrackedChangeIds, + resolveTrackedChange, +} from "../../lib/docxTrackedChanges"; +import { buildDownloadUrl } from "../../lib/downloadTokens"; +import { contentSha256, loadActiveVersion } from "../../lib/documentVersions"; +import { ensureDocAccess } from "../../lib/access"; +import { downloadFilenameForVersion, type Db } from "./documents.shared"; +import { ensureDocumentAccess } from "./documents.access"; + +const isDev = process.env.NODE_ENV !== "production"; +const devLog = (...args: Parameters) => { + if (isDev) console.log(...args); +}; + +// --------------------------------------------------------------------------- +// Tracked-change ids +// --------------------------------------------------------------------------- + +export async function getTrackedChangeIds( + documentId: string, + userId: string, + userEmail: string | undefined, + versionIdParam: string | null, + db: Db, +): Promise<{ ok: true; ids: unknown } | { ok: false; detail: string }> { + const access = await ensureDocumentAccess(documentId, userId, userEmail, db); + if (!access.ok) return { ok: false, detail: "Document not found" }; + + const active = await loadActiveVersion(documentId, db, versionIdParam); + if (!active) return { ok: false, detail: "No file available" }; + + const raw = await downloadFile(active.storage_path); + if (!raw) return { ok: false, detail: "Document bytes not available" }; + + const ids = await extractTrackedChangeIds(Buffer.from(raw)); + return { ok: true, ids }; +} + +// --------------------------------------------------------------------------- +// Accept / reject a tracked-change edit +// --------------------------------------------------------------------------- + +export async function resolveEdit( + mode: "accept" | "reject", + documentId: string, + editId: string, + userId: string, + userEmail: string | undefined, + db: Db, +): Promise< + | { ok: true; body: Record } + | { ok: false; detail: string } +> { + devLog(`[edit-resolution] incoming ${mode}`, { + userId, + documentId, + editId, + }); + + const { data: edit, error: editErr } = await db + .from("document_edits") + .select("id, document_id, change_id, del_w_id, ins_w_id, status") + .eq("id", editId) + .eq("document_id", documentId) + .single(); + devLog(`[edit-resolution] fetched edit row`, { edit, editErr }); + if (!edit) { + devLog(`[edit-resolution] edit not found, returning 404`); + return { ok: false, detail: "Edit not found" }; + } + // Idempotent: if the edit is already resolved, return the current doc + // state so stale UI (e.g. an old chat reloaded in a new session) can + // reconcile without throwing. + if (edit.status !== "pending") { + devLog(`[edit-resolution] edit already resolved`, { + editId, + status: edit.status, + }); + const { data: doc } = await db + .from("documents") + .select("current_version_id, user_id, project_id") + .eq("id", documentId) + .single(); + if (!doc) { + devLog(`[edit-resolution] doc not found for resolved edit`); + return { ok: false, detail: "Document not found" }; + } + const accessResolved = await ensureDocAccess(doc, userId, userEmail, db); + if (!accessResolved.ok) { + devLog(`[edit-resolution] doc access denied for resolved edit`); + return { ok: false, detail: "Document not found" }; + } + const activeForResolved = await loadActiveVersion(documentId, db); + const payload = { + ok: true, + already_resolved: true, + status: edit.status, + version_id: doc.current_version_id ?? null, + download_url: activeForResolved + ? buildDownloadUrl( + activeForResolved.storage_path, + downloadFilenameForVersion( + activeForResolved.filename, + activeForResolved.version_number, + activeForResolved.source === "assistant_edit", + ), + ) + : null, + remaining_pending: 0, + }; + devLog(`[edit-resolution] returning already-resolved payload`, payload); + return { ok: true, body: payload }; + } + + const { data: doc, error: docErr } = await db + .from("documents") + .select("id, current_version_id, user_id, project_id") + .eq("id", documentId) + .single(); + devLog(`[edit-resolution] fetched doc`, { doc, docErr }); + if (!doc) return { ok: false, detail: "Document not found" }; + const access = await ensureDocAccess(doc, userId, userEmail, db); + if (!access.ok) return { ok: false, detail: "Document not found" }; + + const active = await loadActiveVersion(documentId, db); + const latestPath = active?.storage_path ?? null; + devLog(`[edit-resolution] resolved latestPath`, { + latestPath, + current_version_id: doc.current_version_id, + }); + if (!latestPath) return { ok: false, detail: "No file to edit" }; + + const raw = await downloadFile(latestPath); + devLog(`[edit-resolution] downloaded bytes`, { + byteLength: raw?.byteLength ?? 0, + }); + if (!raw) return { ok: false, detail: "Document bytes not available" }; + + const wIds = [edit.del_w_id, edit.ins_w_id].filter( + (v): v is string => typeof v === "string" && v.length > 0, + ); + const { bytes: resolvedBytes, found } = await resolveTrackedChange( + Buffer.from(raw), + wIds, + mode, + ); + devLog(`[edit-resolution] resolveTrackedChange result`, { + mode, + change_id: edit.change_id, + wIds, + found, + resolvedByteLength: resolvedBytes?.byteLength ?? 0, + }); + if (!found) { + devLog( + `[edit-resolution] change_id not found in docx — updating status only`, + ); + // Still update DB status so the UI reflects the decision — the change + // may have been auto-consumed by a previous accept/reject pass. + const { error: updErr } = await db + .from("document_edits") + .update({ status: mode === "accept" ? "accepted" : "rejected", resolved_at: new Date().toISOString() }) + .eq("id", editId); + devLog(`[edit-resolution] status-only update`, { updErr }); + const payload = { + ok: true, + version_id: doc.current_version_id, + download_url: buildDownloadUrl( + latestPath, + downloadFilenameForVersion( + active?.filename, + active?.version_number ?? null, + active?.source === "assistant_edit", + ), + ), + remaining_pending: 0, + }; + devLog(`[edit-resolution] returning not-found payload`, payload); + return { ok: true, body: payload }; + } + + // Overwrite bytes in place at the current version's storage path — + // accept/reject mutates the existing version rather than spawning a + // new row. This keeps document_versions lean (one row per assistant + // edit, not one per accept/reject click) and avoids the N-versions- + // per-doc churn as users resolve pending changes. + const ab = resolvedBytes.buffer.slice( + resolvedBytes.byteOffset, + resolvedBytes.byteOffset + resolvedBytes.byteLength, + ) as ArrayBuffer; + + // Clear the hash before the bytes change, and set it again after. The stored + // object and the hash live in different systems, so they cannot be written + // atomically; ordering it this way means a failure in between leaves the + // version unhashed, which the manifest reports as unverifiable. The + // alternative ordering can leave a hash attesting to content the version no + // longer holds, which is the one thing the manifest must never do. + await db + .from("document_versions") + .update({ content_sha256: null }) + .eq("id", doc.current_version_id); + + devLog(`[edit-resolution] overwriting bytes in place`, { + latestPath, + byteLength: ab.byteLength, + }); + await uploadFile( + latestPath, + ab, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ); + + await db + .from("document_versions") + .update({ content_sha256: contentSha256(ab) }) + .eq("id", doc.current_version_id); + + const { error: statusErr } = await db + .from("document_edits") + .update({ + status: mode === "accept" ? "accepted" : "rejected", + resolved_at: new Date().toISOString(), + }) + .eq("id", editId); + devLog(`[edit-resolution] updated document_edits status`, { + editId, + newStatus: mode === "accept" ? "accepted" : "rejected", + statusErr, + }); + const { count: remainingPending } = await db + .from("document_edits") + .select("id", { count: "exact", head: true }) + .eq("document_id", documentId) + .eq("status", "pending"); + devLog(`[edit-resolution] remaining pending count`, { remainingPending }); + + const payload = { + ok: true, + version_id: doc.current_version_id, + download_url: buildDownloadUrl( + latestPath, + downloadFilenameForVersion( + active?.filename, + active?.version_number ?? null, + active?.source === "assistant_edit", + ), + ), + remaining_pending: remainingPending ?? 0, + }; + devLog(`[edit-resolution] returning success payload`, payload); + return { ok: true, body: payload }; +} diff --git a/backend/src/modules/documents/documents.routes.ts b/backend/src/modules/documents/documents.routes.ts new file mode 100644 index 000000000..0d6f4feee --- /dev/null +++ b/backend/src/modules/documents/documents.routes.ts @@ -0,0 +1,550 @@ +import { Router } from "express"; +import { requireAuth } from "../../middleware/auth"; +import { createServerSupabase } from "../../lib/supabase"; +import { buildContentDisposition } from "../../lib/storage"; +import { singleFileUpload } from "../../lib/upload"; +import { + ALLOWED_DOCUMENT_TYPES, + ALLOWED_DOCUMENT_TYPES_LABEL, +} from "../../lib/documentTypes"; +import { + listSingleDocuments, + getDocument, + createDocumentFromUpload, + deleteDocument, + getDisplayableVersion, + buildZipForDocuments, + getDownloadUrl, + getDocxBytes, + listVersions, + createVersionFromDocument, + addUploadedVersion, + renameVersion, + loadReplaceTarget, + writeReplacementVersion, + deleteVersion, + getTrackedChangeIds, + resolveEdit, + checkDocumentAccess, +} from "./documents.service"; + +export const documentsRouter = Router(); + +// GET /single-documents +documentsRouter.get("/", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const db = createServerSupabase(); + const result = await listSingleDocuments(userId, db); + if (!result.ok) return void res.status(500).json({ detail: result.detail }); + res.json(result.docs); +}); + +// GET /single-documents/:documentId +// One document, same shape as a list entry — the client polls this while a +// deferred conversion runs instead of refetching the whole collection. +documentsRouter.get("/:documentId", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { documentId } = req.params; + const db = createServerSupabase(); + + const result = await getDocument(documentId, userId, userEmail, db); + if (!result.ok) + return void res.status(404).json({ detail: "Document not found" }); + res.json(result.doc); +}); + +// POST /single-documents +documentsRouter.post( + "/", + requireAuth, + singleFileUpload("file"), + async (req, res) => { + const userId = res.locals.userId as string; + const db = createServerSupabase(); + + const file = req.file; + if (!file) + return void res.status(400).json({ detail: "file is required" }); + + const filename = file.originalname; + const suffix = filename.includes(".") + ? filename.split(".").pop()!.toLowerCase() + : ""; + if (!ALLOWED_DOCUMENT_TYPES.has(suffix)) + return void res + .status(400) + .json({ + detail: `Unsupported file type: ${suffix}. Allowed: ${ALLOWED_DOCUMENT_TYPES_LABEL}`, + }); + + const result = await createDocumentFromUpload( + { + userId, + projectId: null, + filename, + suffix, + content: file.buffer, + libraryKind: "file", + }, + db, + ); + if (!result.ok) { + if (result.kind === "create_failed") + return void res + .status(500) + .json({ detail: "Failed to create document record" }); + return void res + .status(500) + .json({ detail: `Document processing failed: ${result.detail}` }); + } + res.status(201).json(result.doc); + }, +); + +// DELETE /single-documents/:documentId +documentsRouter.delete("/:documentId", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const { documentId } = req.params; + const db = createServerSupabase(); + + const result = await deleteDocument(documentId, userId, db); + if (!result.ok) + return void res.status(404).json({ detail: "Document not found" }); + res.status(204).send(); +}); + +// GET /single-documents/:documentId/display +// Optional ?version_id= renders a historical version. Defaults to the +// document's current_version_id. +documentsRouter.get("/:documentId/display", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string; + const { documentId } = req.params; + const versionIdParam = + typeof req.query.version_id === "string" ? req.query.version_id : null; + const db = createServerSupabase(); + + const result = await getDisplayableVersion( + documentId, + userId, + userEmail, + versionIdParam, + db, + ); + if (!result.ok) + return void res.status(404).json({ detail: result.detail }); + + res.setHeader("Content-Type", result.contentType); + res.setHeader( + "Content-Disposition", + buildContentDisposition("inline", result.filename), + ); + res.send(Buffer.from(result.bytes)); +}); + +// POST /single-documents/download-zip +documentsRouter.post("/download-zip", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { document_ids } = req.body as { document_ids?: string[] }; + + if (!Array.isArray(document_ids) || document_ids.length === 0) + return void res.status(400).json({ detail: "document_ids is required" }); + + const db = createServerSupabase(); + const result = await buildZipForDocuments( + document_ids, + userId, + userEmail, + db, + ); + if (!result.ok) { + if (result.kind === "db") + return void res.status(500).json({ detail: result.detail }); + return void res.status(404).json({ detail: "No documents found" }); + } + + res.setHeader("Content-Type", "application/zip"); + res.setHeader("Content-Disposition", 'attachment; filename="documents.zip"'); + res.send(result.content); +}); + +// GET /single-documents/:documentId/url +// Optional ?version_id= selects a specific tracked-changes version. +// Otherwise falls back to documents.current_version_id, else the original upload. +documentsRouter.get("/:documentId/url", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { documentId } = req.params; + const versionIdParam = + typeof req.query.version_id === "string" ? req.query.version_id : null; + const db = createServerSupabase(); + + const result = await getDownloadUrl( + documentId, + userId, + userEmail, + versionIdParam, + db, + ); + if (!result.ok) { + const status = result.kind === "storage" ? 503 : 404; + return void res.status(status).json({ detail: result.detail }); + } + res.json(result.payload); +}); + +// GET /single-documents/:documentId/docx +// Streams the raw .docx bytes for the given document, optionally at a +// specific tracked-changes version. Unlike /url, this bypasses R2 (avoids +// the browser CORS problem on signed URLs) so the frontend docx-preview +// viewer can load tracked-change documents directly. +documentsRouter.get("/:documentId/docx", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { documentId } = req.params; + const versionIdParam = + typeof req.query.version_id === "string" ? req.query.version_id : null; + const db = createServerSupabase(); + + const result = await getDocxBytes( + documentId, + userId, + userEmail, + versionIdParam, + db, + ); + if (!result.ok) + return void res.status(404).json({ detail: result.detail }); + + res.setHeader( + "Content-Type", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ); + res.setHeader( + "Content-Disposition", + buildContentDisposition("inline", result.filename), + ); + res.send(Buffer.from(result.bytes)); +}); + +// GET /single-documents/:documentId/versions +// Returns every version row for the document in document order, with +// the human-friendly version number when present. +documentsRouter.get("/:documentId/versions", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { documentId } = req.params; + const db = createServerSupabase(); + + const result = await listVersions(documentId, userId, userEmail, db); + if (!result.ok) + return void res.status(404).json({ detail: result.detail }); + + res.json({ + current_version_id: result.current_version_id, + versions: result.versions, + }); +}); + +// POST /single-documents/:documentId/versions/from-document +// Create a new version of documentId from another existing document's active +// bytes. This keeps signed storage URLs out of the browser fetch path. +documentsRouter.post( + "/:documentId/versions/from-document", + requireAuth, + async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { documentId } = req.params; + const sourceDocumentId = + typeof req.body?.source_document_id === "string" + ? req.body.source_document_id + : ""; + const db = createServerSupabase(); + + if (!sourceDocumentId) { + return void res + .status(400) + .json({ detail: "source_document_id is required" }); + } + if (sourceDocumentId === documentId) { + return void res + .status(400) + .json({ detail: "Source and target documents must be different." }); + } + + const result = await createVersionFromDocument( + { + documentId, + sourceDocumentId, + requestedFilename: + typeof req.body?.filename === "string" + ? req.body.filename + : null, + userId, + userEmail, + }, + db, + ); + if (!result.ok) { + const status = + result.kind === "source_not_owner" + ? 403 + : result.kind === "target_not_found" || + result.kind === "source_not_found" || + result.kind === "source_no_active" || + result.kind === "source_bytes" + ? 404 + : 500; + return void res.status(status).json({ detail: result.detail }); + } + res.status(201).json(result.version); + }, +); + +// POST /single-documents/:documentId/versions +// Upload a brand-new version of an existing document. The uploaded file +// becomes the new current_version_id. filename defaults to the +// uploaded filename; client may override via the `filename` form field. +documentsRouter.post( + "/:documentId/versions", + requireAuth, + singleFileUpload("file"), + async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { documentId } = req.params; + const db = createServerSupabase(); + + const file = req.file; + if (!file) + return void res.status(400).json({ detail: "file is required" }); + + const hasAccess = await checkDocumentAccess( + documentId, + userId, + userEmail, + db, + { select: "id, user_id, project_id, current_version_id" }, + ); + if (!hasAccess) + return void res.status(404).json({ detail: "Document not found" }); + + const suffix = file.originalname.includes(".") + ? file.originalname.split(".").pop()!.toLowerCase() + : ""; + if (!ALLOWED_DOCUMENT_TYPES.has(suffix)) { + return void res.status(400).json({ + detail: `Unsupported file type: ${suffix}. Allowed: ${ALLOWED_DOCUMENT_TYPES_LABEL}`, + }); + } + + const result = await addUploadedVersion( + { + userId, + documentId, + file, + suffix, + requestedFilename: req.body?.filename, + }, + db, + ); + if (!result.ok) + return void res.status(500).json({ detail: result.detail }); + res.status(201).json(result.version); + }, +); + +// PATCH /single-documents/:documentId/versions/:versionId +// Rename a version's filename. Pass `{ "filename": "…" }`. +documentsRouter.patch( + "/:documentId/versions/:versionId", + requireAuth, + async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { documentId, versionId } = req.params; + const db = createServerSupabase(); + + const result = await renameVersion( + { + documentId, + versionId, + rawFilename: req.body?.filename, + userId, + userEmail, + }, + db, + ); + if (!result.ok) + return void res.status(404).json({ detail: result.detail }); + res.json(result.version); + }, +); + +// PUT /single-documents/:documentId/versions/:versionId/file +// Replace the file bytes and metadata for an existing version while keeping +// its version number and id. This is destructive and owner-only. +documentsRouter.put( + "/:documentId/versions/:versionId/file", + requireAuth, + singleFileUpload("file"), + async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { documentId, versionId } = req.params; + const db = createServerSupabase(); + + const file = req.file; + if (!file) + return void res.status(400).json({ detail: "file is required" }); + + const hasAccess = await checkDocumentAccess( + documentId, + userId, + userEmail, + db, + { ownerOnly: true }, + ); + if (!hasAccess) + return void res.status(404).json({ detail: "Document not found" }); + + const targetResult = await loadReplaceTarget(documentId, versionId, db); + if (!targetResult.ok) { + const status = targetResult.kind === "version_not_found" ? 404 : 400; + return void res.status(status).json({ detail: targetResult.detail }); + } + const target = targetResult.target; + + const suffix = file.originalname.includes(".") + ? file.originalname.split(".").pop()!.toLowerCase() + : ""; + if (!ALLOWED_DOCUMENT_TYPES.has(suffix)) { + return void res.status(400).json({ + detail: `Unsupported file type: ${suffix}. Allowed: ${ALLOWED_DOCUMENT_TYPES_LABEL}`, + }); + } + if (target.file_type && target.file_type !== suffix) { + return void res.status(400).json({ + detail: `Uploaded file type (${suffix}) does not match version type (${target.file_type}).`, + }); + } + + const result = await writeReplacementVersion( + { + userId, + documentId, + versionId, + file, + suffix, + requestedFilename: req.body?.filename, + target, + }, + db, + ); + if (!result.ok) + return void res.status(500).json({ detail: result.detail }); + res.json(result.version); + }, +); + +// DELETE /single-documents/:documentId/versions/:versionId +// Delete one version. The last remaining version cannot be deleted; if the +// deleted version is current, the newest remaining version becomes current. +documentsRouter.delete( + "/:documentId/versions/:versionId", + requireAuth, + async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { documentId, versionId } = req.params; + const db = createServerSupabase(); + + const result = await deleteVersion( + documentId, + versionId, + userId, + userEmail, + db, + ); + if (!result.ok) { + const status = + result.kind === "doc_not_found" || + result.kind === "version_not_found" + ? 404 + : result.kind === "only_version" + ? 400 + : 500; + return void res.status(status).json({ detail: result.detail }); + } + res.json(result.payload); + }, +); + +// GET /single-documents/:documentId/tracked-change-ids +// Returns the ordered list of { kind, w_id } for every w:ins / w:del in +// the current (or specified) version's document.xml. The frontend uses +// this to tag each rendered / with data-w-id, since +// docx-preview drops the w:id attribute during parsing. +documentsRouter.get( + "/:documentId/tracked-change-ids", + requireAuth, + async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { documentId } = req.params; + const versionIdParam = + typeof req.query.version_id === "string" ? req.query.version_id : null; + const db = createServerSupabase(); + + const result = await getTrackedChangeIds( + documentId, + userId, + userEmail, + versionIdParam, + db, + ); + if (!result.ok) + return void res.status(404).json({ detail: result.detail }); + res.json({ ids: result.ids }); + }, +); + +// POST /single-documents/:documentId/edits/:editId/accept +// POST /single-documents/:documentId/edits/:editId/reject +async function handleEditResolution( + req: import("express").Request, + res: import("express").Response, + mode: "accept" | "reject", +) { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { documentId, editId } = req.params; + const db = createServerSupabase(); + + const result = await resolveEdit( + mode, + documentId, + editId, + userId, + userEmail, + db, + ); + if (!result.ok) + return void res.status(404).json({ detail: result.detail }); + res.json(result.body); +} + +documentsRouter.post( + "/:documentId/edits/:editId/accept", + requireAuth, + (req, res) => void handleEditResolution(req, res, "accept"), +); + +documentsRouter.post( + "/:documentId/edits/:editId/reject", + requireAuth, + (req, res) => void handleEditResolution(req, res, "reject"), +); diff --git a/backend/src/modules/documents/documents.service.ts b/backend/src/modules/documents/documents.service.ts new file mode 100644 index 000000000..f729613fe --- /dev/null +++ b/backend/src/modules/documents/documents.service.ts @@ -0,0 +1,56 @@ +// Business logic + data-access for the documents module. +// +// These functions are the service layer behind documents.routes.ts. They take +// an explicit Supabase client (`db`) plus request-derived primitives, perform +// the storage / version / conversion orchestration, and RETURN values or +// typed error results. They never touch req/res — the thin route handlers map +// the results onto HTTP status codes, headers, and response bodies. +// +// This file is the module's stable facade: the implementation is decomposed +// into cohesive sibling files and re-exported here so importers never change. +// +// documents.shared.ts — shared types and helpers +// documents.access.ts — access guards + list/delete document +// documents.download.ts — display bytes, zip bundling, signed URLs, raw docx +// documents.versions.ts — version lifecycle (list/create/rename/replace/delete) +// documents.edits.ts — tracked-change ids + accept/reject edits +// documents.upload.ts — initial document creation from an uploaded file + +export { + deleteDocumentAndVersionFiles, + downloadFilenameForVersion, + countPdfPages, + type Db, + type UploadedFile, +} from "./documents.shared"; + +export { + checkDocumentAccess, + getDocument, + listSingleDocuments, + deleteDocument, +} from "./documents.access"; + +export { + getDisplayableVersion, + buildZipForDocuments, + getDownloadUrl, + getDocxBytes, +} from "./documents.download"; + +export { + listVersions, + createVersionFromDocument, + addUploadedVersion, + renameVersion, + loadReplaceTarget, + writeReplacementVersion, + deleteVersion, +} from "./documents.versions"; + +export { + getTrackedChangeIds, + resolveEdit, +} from "./documents.edits"; + +export { createDocumentFromUpload } from "./documents.upload"; diff --git a/backend/src/modules/documents/documents.shared.ts b/backend/src/modules/documents/documents.shared.ts new file mode 100644 index 000000000..0aeda8097 --- /dev/null +++ b/backend/src/modules/documents/documents.shared.ts @@ -0,0 +1,62 @@ +// Shared types and helpers for the documents module's service files. +// Everything public here is re-exported through documents.service.ts, +// which remains the module's stable facade. + +import { createServerSupabase } from "../../lib/supabase"; +import { deleteFile } from "../../lib/storage"; + +export type Db = ReturnType; + +// Structural slice of Express.Multer.File — only these two fields are read. +export type UploadedFile = { buffer: Buffer; originalname: string }; + +export async function deleteDocumentAndVersionFiles( + db: Db, + documentId: string, +) { + // Storage lives on document_versions — fan out and delete each version's + // bytes (source + PDF rendition) before dropping the document row. + const { data: versions } = await db + .from("document_versions") + .select("storage_path, pdf_storage_path") + .eq("document_id", documentId); + await Promise.all( + (versions ?? []).flatMap((v) => + [v.storage_path, v.pdf_storage_path] + .filter((p): p is string => typeof p === "string" && p.length > 0) + .map((p) => deleteFile(p).catch(() => {})), + ), + ); + return db.from("documents").delete().eq("id", documentId); +} + +// Produce the filename a download should present to the user. Version +// filenames are expected to include the real extension. +export function downloadFilenameForVersion( + filename: string | null | undefined, + versionNumber: number | null, + edited = false, +): string { + const resolved = filename?.trim() || "Untitled document.docx"; + if (!edited || !versionNumber || versionNumber < 1) return resolved; + const dot = resolved.lastIndexOf("."); + const stem = dot > 0 ? resolved.slice(0, dot) : resolved; + const ext = dot > 0 ? resolved.slice(dot) : ""; + return `${stem} [Edited V${versionNumber}]${ext}`; +} + +export async function countPdfPages(buf: ArrayBuffer): Promise { + try { + const pdfjsLib = await import("pdfjs-dist/legacy/build/pdf.mjs" as string); + const pdf = await ( + pdfjsLib as unknown as { + getDocument: (opts: unknown) => { + promise: Promise<{ numPages: number }>; + }; + } + ).getDocument({ data: new Uint8Array(buf) }).promise; + return pdf.numPages; + } catch { + return null; + } +} diff --git a/backend/src/modules/documents/documents.upload.ts b/backend/src/modules/documents/documents.upload.ts new file mode 100644 index 000000000..cfb12d653 --- /dev/null +++ b/backend/src/modules/documents/documents.upload.ts @@ -0,0 +1,179 @@ +// Initial document creation from an uploaded file. + +import { storageKey, uploadFile } from "../../lib/storage"; +import { docxToPdf, convertedPdfKey } from "../../lib/convert"; +import { enqueueConversion } from "../../lib/queue/conversionQueue"; +import { contentSha256 } from "../../lib/documentVersions"; +import { + contentTypeForDocumentType, + shouldConvertToPdf, +} from "../../lib/documentTypes"; +import { countPdfPages, type Db } from "./documents.shared"; + +// --------------------------------------------------------------------------- +// Create a document from an uploaded file (initial upload pipeline) +// --------------------------------------------------------------------------- + +export async function createDocumentFromUpload( + params: { + userId: string; + projectId: string | null; + filename: string; + suffix: string; + content: Buffer; + libraryKind?: "file" | "template"; + libraryFolderId?: string | null; + }, + db: Db, +): Promise< + | { ok: true; doc: unknown } + | { ok: false; kind: "create_failed" } + | { ok: false; kind: "processing_failed"; detail: string } +> { + const { userId, projectId, filename, suffix, content } = params; + + const { data: doc, error: insertErr } = await db + .from("documents") + .insert({ + project_id: projectId, + user_id: userId, + status: "processing", + library_kind: params.libraryKind ?? "file", + library_folder_id: params.libraryFolderId ?? null, + }) + .select("*") + .single(); + + if (insertErr || !doc) + console.error("[single-documents/upload] failed to create document row", { + userId, + projectId, + filename, + suffix, + error: insertErr, + }); + if (insertErr || !doc) return { ok: false, kind: "create_failed" }; + + try { + const docId = doc.id as string; + const key = storageKey(userId, docId, filename); + const contentType = contentTypeForDocumentType(suffix); + await uploadFile( + key, + content.buffer.slice( + content.byteOffset, + content.byteOffset + content.byteLength, + ) as ArrayBuffer, + contentType, + ); + + const rawBuf = content.buffer.slice( + content.byteOffset, + content.byteOffset + content.byteLength, + ) as ArrayBuffer; + const pageCount = suffix === "pdf" ? await countPdfPages(rawBuf) : null; + + // When the job queue is enabled, defer Office → PDF conversion to the + // BullMQ worker instead of blocking the upload request on LibreOffice. + const deferConversion = + shouldConvertToPdf(suffix) && + process.env.ASYNC_DOCUMENT_CONVERSION === "true"; + + // Convert Office files → PDF for display. PDFs are their own rendition. + let pdfStoragePath: string | null = null; + if (!deferConversion && shouldConvertToPdf(suffix)) { + try { + const pdfBuf = await docxToPdf(content); + const pdfKey = convertedPdfKey(userId, docId); + await uploadFile( + pdfKey, + pdfBuf.buffer.slice( + pdfBuf.byteOffset, + pdfBuf.byteOffset + pdfBuf.byteLength, + ) as ArrayBuffer, + "application/pdf", + ); + pdfStoragePath = pdfKey; + } catch (err) { + console.error( + `[upload] Office→PDF conversion failed for ${filename}:`, + err, + ); + } + } else if (suffix === "pdf") { + pdfStoragePath = key; + } + + // storage_path / pdf_storage_path live on document_versions now — + // create the V1 "upload" row and point documents.current_version_id + // at it. + const { data: versionRow, error: verErr } = await db + .from("document_versions") + .insert({ + document_id: docId, + storage_path: key, + pdf_storage_path: pdfStoragePath, + source: "upload", + version_number: 1, + filename: filename, + file_type: suffix, + size_bytes: content.byteLength, + page_count: pageCount, + content_sha256: contentSha256(content), + }) + .select("id") + .single(); + if (verErr || !versionRow) { + throw new Error( + `Failed to record upload version: ${verErr?.message ?? "unknown"}`, + ); + } + + await db + .from("documents") + .update({ + current_version_id: versionRow.id, + // Deferred conversion leaves the doc "processing" until the worker + // produces the PDF and flips it to "ready". + status: deferConversion ? "processing" : "ready", + updated_at: new Date().toISOString(), + }) + .eq("id", docId); + + if (deferConversion) { + await enqueueConversion({ + documentId: docId, + versionId: versionRow.id, + userId, + storagePath: key, + fileType: suffix, + }); + } + + const { data: updated } = await db + .from("documents") + .select("*") + .eq("id", docId) + .single(); + // Surface storage paths to the caller for backward compatibility. + const responseDoc = updated + ? { + ...updated, + filename, + storage_path: key, + pdf_storage_path: pdfStoragePath, + folder_id: + (updated.library_folder_id as string | null | undefined) ?? + null, + file_type: suffix, + size_bytes: content.byteLength, + page_count: pageCount, + active_version_number: 1, + } + : updated; + return { ok: true, doc: responseDoc }; + } catch (e) { + await db.from("documents").update({ status: "error" }).eq("id", doc.id); + return { ok: false, kind: "processing_failed", detail: String(e) }; + } +} diff --git a/backend/src/modules/documents/documents.versions.ts b/backend/src/modules/documents/documents.versions.ts new file mode 100644 index 000000000..76d8c627e --- /dev/null +++ b/backend/src/modules/documents/documents.versions.ts @@ -0,0 +1,789 @@ +// Version lifecycle for documents: listing, creating (from another document +// or an uploaded file), renaming, replacing bytes, and deleting versions. + +import { + downloadFile, + deleteFile, + uploadFile, + versionStorageKey, +} from "../../lib/storage"; +import { docxToPdf } from "../../lib/convert"; +import { enqueueConversion } from "../../lib/queue/conversionQueue"; +import { contentSha256, loadActiveVersion } from "../../lib/documentVersions"; +import { + contentTypeForDocumentType, + shouldConvertToPdf, +} from "../../lib/documentTypes"; +import { + countPdfPages, + deleteDocumentAndVersionFiles, + type Db, + type UploadedFile, +} from "./documents.shared"; +import { ensureDocumentAccess } from "./documents.access"; + +// --------------------------------------------------------------------------- +// Versions list +// --------------------------------------------------------------------------- + +export async function listVersions( + documentId: string, + userId: string, + userEmail: string | undefined, + db: Db, +): Promise< + | { ok: true; current_version_id: string | null; versions: unknown[] } + | { ok: false; detail: string } +> { + const access = await ensureDocumentAccess(documentId, userId, userEmail, db, { + select: "id, current_version_id, user_id, project_id", + }); + if (!access.ok) return { ok: false, detail: "Document not found" }; + + const { data: rows } = await db + .from("document_versions") + .select( + "id, version_number, source, created_at, filename, file_type, size_bytes, page_count, deleted_at, deleted_by", + ) + .eq("document_id", documentId) + .order("created_at", { ascending: true }); + + return { + ok: true, + current_version_id: access.doc.current_version_id ?? null, + versions: rows ?? [], + }; +} + +// --------------------------------------------------------------------------- +// Create version from another document +// --------------------------------------------------------------------------- + +export async function createVersionFromDocument( + params: { + documentId: string; + sourceDocumentId: string; + requestedFilename: string | null; + userId: string; + userEmail: string | undefined; + }, + db: Db, +): Promise< + | { ok: true; version: unknown } + | { + ok: false; + kind: + | "target_not_found" + | "source_not_found" + | "source_not_owner" + | "source_no_active" + | "source_bytes" + | "storage_write" + | "version_insert" + | "doc_update" + | "source_delete"; + detail: string; + } +> { + const { documentId, sourceDocumentId, requestedFilename, userId, userEmail } = + params; + + const targetAccess = await ensureDocumentAccess( + documentId, + userId, + userEmail, + db, + ); + if (!targetAccess.ok) + return { + ok: false, + kind: "target_not_found", + detail: "Document not found", + }; + const targetDoc = targetAccess.doc; + + const sourceAccess = await ensureDocumentAccess( + sourceDocumentId, + userId, + userEmail, + db, + ); + if (!sourceAccess.ok) + return { + ok: false, + kind: "source_not_found", + detail: "Source document not found", + }; + const sourceDoc = sourceAccess.doc; + const willDeleteSource = + (sourceDoc.project_id && + targetDoc.project_id && + sourceDoc.project_id === targetDoc.project_id) || + (!sourceDoc.project_id && + !targetDoc.project_id && + sourceDoc.user_id === userId && + targetDoc.user_id === userId); + if (willDeleteSource && !sourceAccess.isOwner) { + return { + ok: false, + kind: "source_not_owner", + detail: "Only the source document owner can move it into a version.", + }; + } + + const active = await loadActiveVersion(sourceDocumentId, db); + if (!active) + return { + ok: false, + kind: "source_no_active", + detail: "Source document has no active version.", + }; + const sourceType = active.file_type ?? ""; + + const bytes = await downloadFile(active.storage_path); + if (!bytes) + return { + ok: false, + kind: "source_bytes", + detail: "Source document bytes not available.", + }; + + const filename = + requestedFilename && requestedFilename.trim() + ? requestedFilename.trim().slice(0, 200) + : active.filename?.trim() || "Untitled document"; + const suffix = + sourceType || + (filename.includes(".") ? filename.split(".").pop()!.toLowerCase() : ""); + const versionSlug = crypto.randomUUID().replace(/-/g, ""); + const key = versionStorageKey(userId, documentId, versionSlug, filename); + const contentType = contentTypeForDocumentType(suffix); + + try { + await uploadFile(key, bytes, contentType); + } catch (e) { + console.error("[versions/copy] storage write failed", e); + return { + ok: false, + kind: "storage_write", + detail: "Failed to create new version.", + }; + } + + let pdfStoragePath: string | null = null; + let deferConversion = false; + if (suffix === "pdf") { + pdfStoragePath = key; + } else if (active.pdf_storage_path) { + if (active.pdf_storage_path === active.storage_path) { + pdfStoragePath = key; + } else { + const pdfBytes = await downloadFile(active.pdf_storage_path); + if (pdfBytes) { + const pdfKey = `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`; + await uploadFile(pdfKey, pdfBytes, "application/pdf"); + pdfStoragePath = pdfKey; + } + } + } else if (shouldConvertToPdf(suffix)) { + // Only reached when the source has no rendition to copy — this is the + // one branch of the copy flow that pays for LibreOffice, so it's the + // branch the conversion queue takes over when the flag is on. + if (process.env.ASYNC_DOCUMENT_CONVERSION === "true") { + deferConversion = true; + } else { + try { + const pdfBuf = await docxToPdf(Buffer.from(bytes)); + const pdfKey = `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`; + await uploadFile( + pdfKey, + pdfBuf.buffer.slice( + pdfBuf.byteOffset, + pdfBuf.byteOffset + pdfBuf.byteLength, + ) as ArrayBuffer, + "application/pdf", + ); + pdfStoragePath = pdfKey; + } catch (err) { + console.error( + `[versions/copy] Office→PDF conversion failed for ${filename}:`, + err, + ); + } + } + } + + const { data: maxRow } = await db + .from("document_versions") + .select("version_number") + .eq("document_id", documentId) + .in("source", ["upload", "user_upload", "assistant_edit"]) + .order("version_number", { ascending: false, nullsFirst: false }) + .limit(1) + .maybeSingle(); + const nextVersionNumber = + ((maxRow?.version_number as number | null) ?? 1) + 1; + + const { data: versionRow, error: verErr } = await db + .from("document_versions") + .insert({ + document_id: documentId, + storage_path: key, + pdf_storage_path: pdfStoragePath, + source: "user_upload", + version_number: nextVersionNumber, + filename: filename, + file_type: sourceType || null, + size_bytes: active.size_bytes ?? bytes.byteLength, + page_count: active.page_count, + content_sha256: contentSha256(bytes), + }) + .select("id, version_number, source, created_at, filename") + .single(); + if (verErr || !versionRow) { + console.error("[versions/copy] insert failed", verErr); + return { + ok: false, + kind: "version_insert", + detail: "Failed to record new version.", + }; + } + + const { error: updateDocErr } = await db + .from("documents") + .update({ + current_version_id: versionRow.id, + }) + .eq("id", documentId); + if (updateDocErr) { + console.error("[versions/copy] current version update failed", updateDocErr); + return { + ok: false, + kind: "doc_update", + detail: "Failed to update document current version.", + }; + } + + if (deferConversion) { + await enqueueConversion({ + documentId, + versionId: versionRow.id as string, + userId, + storagePath: key, + fileType: suffix, + pdfKey: `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`, + finalizeDocumentStatus: false, + }); + } + + if (willDeleteSource) { + const { error: deleteErr } = await deleteDocumentAndVersionFiles( + db, + sourceDocumentId, + ); + if (deleteErr) { + console.error("[versions/copy] source document delete failed", deleteErr); + return { + ok: false, + kind: "source_delete", + detail: "Failed to delete source document.", + }; + } + } + + return { ok: true, version: versionRow }; +} + +// --------------------------------------------------------------------------- +// Create version from an uploaded file (orchestration after HTTP validation) +// --------------------------------------------------------------------------- + +export async function addUploadedVersion( + params: { + userId: string; + documentId: string; + file: UploadedFile; + suffix: string; + requestedFilename: unknown; + }, + db: Db, +): Promise< + | { ok: true; version: unknown } + | { ok: false; detail: string } +> { + const { userId, documentId, file, suffix } = params; + + // Peg the new version into a predictable /versions/:id path under the + // existing document folder so ops can spot the history in storage. + const versionSlug = crypto.randomUUID().replace(/-/g, ""); + const key = versionStorageKey( + userId, + documentId, + versionSlug, + file.originalname, + ); + const contentType = contentTypeForDocumentType(suffix); + try { + await uploadFile( + key, + file.buffer.buffer.slice( + file.buffer.byteOffset, + file.buffer.byteOffset + file.buffer.byteLength, + ) as ArrayBuffer, + contentType, + ); + } catch (e) { + console.error("[versions/upload] storage write failed", e); + return { ok: false, detail: "Failed to upload new version." }; + } + + // Render this version's bytes to PDF up front so /display can show + // historical versions without on-demand conversion. Same logic as the + // initial-upload pipeline; failures don't block the version row. + // With the job queue enabled the LibreOffice work is deferred to the + // conversion worker instead of blocking this request; the version row is + // created with pdf_storage_path null and the worker fills it in. + const deferConversion = + shouldConvertToPdf(suffix) && + process.env.ASYNC_DOCUMENT_CONVERSION === "true"; + let pdfStoragePath: string | null = null; + if (!deferConversion && shouldConvertToPdf(suffix)) { + try { + const pdfBuf = await docxToPdf(file.buffer); + const pdfKey = `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`; + await uploadFile( + pdfKey, + pdfBuf.buffer.slice( + pdfBuf.byteOffset, + pdfBuf.byteOffset + pdfBuf.byteLength, + ) as ArrayBuffer, + "application/pdf", + ); + pdfStoragePath = pdfKey; + } catch (err) { + console.error( + `[versions/upload] Office→PDF conversion failed for ${file.originalname}:`, + err, + ); + } + } else if (suffix === "pdf") { + // For PDF uploads, the uploaded bytes are themselves the PDF rendition. + pdfStoragePath = key; + } + + const rawBuf = file.buffer.buffer.slice( + file.buffer.byteOffset, + file.buffer.byteOffset + file.buffer.byteLength, + ) as ArrayBuffer; + const pageCount = suffix === "pdf" ? await countPdfPages(rawBuf) : null; + + // Per-document sequential version_number — the upload is V1 and + // user_upload + assistant_edit count forward from there. + const { data: maxRow } = await db + .from("document_versions") + .select("version_number") + .eq("document_id", documentId) + .in("source", ["upload", "user_upload", "assistant_edit"]) + .order("version_number", { ascending: false, nullsFirst: false }) + .limit(1) + .maybeSingle(); + const nextVersionNumber = + ((maxRow?.version_number as number | null) ?? 1) + 1; + + const requestedFilename = + typeof params.requestedFilename === "string" && + params.requestedFilename.trim() + ? params.requestedFilename.trim().slice(0, 200) + : file.originalname; + + const { data: versionRow, error: verErr } = await db + .from("document_versions") + .insert({ + document_id: documentId, + storage_path: key, + pdf_storage_path: pdfStoragePath, + source: "user_upload", + version_number: nextVersionNumber, + filename: requestedFilename, + file_type: suffix, + size_bytes: file.buffer.byteLength, + page_count: pageCount, + content_sha256: contentSha256(file.buffer), + }) + .select("id, version_number, source, created_at, filename") + .single(); + if (verErr || !versionRow) { + console.error("[versions/upload] insert failed", verErr); + return { ok: false, detail: "Failed to record new version." }; + } + + const { error: updateDocErr } = await db + .from("documents") + .update({ + current_version_id: versionRow.id, + }) + .eq("id", documentId); + if (updateDocErr) { + console.error( + "[versions/upload] current version update failed", + updateDocErr, + ); + return { ok: false, detail: "Failed to update document current version." }; + } + + if (deferConversion) { + // The document itself stays "ready" — only this version's rendition is + // pending, so the worker must not touch documents.status. + await enqueueConversion({ + documentId, + versionId: versionRow.id as string, + userId, + storagePath: key, + fileType: suffix, + pdfKey: `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`, + finalizeDocumentStatus: false, + }); + } + + return { ok: true, version: versionRow }; +} + +// --------------------------------------------------------------------------- +// Rename a version +// --------------------------------------------------------------------------- + +export async function renameVersion( + params: { + documentId: string; + versionId: string; + rawFilename: unknown; + userId: string; + userEmail: string | undefined; + }, + db: Db, +): Promise<{ ok: true; version: unknown } | { ok: false; detail: string }> { + const { documentId, versionId, rawFilename, userId, userEmail } = params; + + const access = await ensureDocumentAccess(documentId, userId, userEmail, db); + if (!access.ok) return { ok: false, detail: "Document not found" }; + + const filename = + typeof rawFilename === "string" && rawFilename.trim() + ? rawFilename.trim().slice(0, 200) + : null; + + const { data: updated, error } = await db + .from("document_versions") + .update({ filename }) + .eq("id", versionId) + .eq("document_id", documentId) + .is("deleted_at", null) + .select( + "id, version_number, source, created_at, filename, file_type, size_bytes, page_count", + ) + .single(); + if (error || !updated) { + return { ok: false, detail: "Version not found" }; + } + return { ok: true, version: updated }; +} + +// --------------------------------------------------------------------------- +// Replace a version's file bytes (owner-only; destructive) +// --------------------------------------------------------------------------- + +/** + * Load the version targeted by a replace request and verify it exists and is + * not deleted. Returns the version's existing storage paths (needed for + * cleanup) plus its declared file_type (so the route can run the + * extension-then-type-mismatch validation in its original order). + */ +export async function loadReplaceTarget( + documentId: string, + versionId: string, + db: Db, +): Promise< + | { + ok: true; + target: { + storage_path: string | null; + pdf_storage_path: string | null; + file_type: string | null; + }; + } + | { ok: false; kind: "version_not_found" | "deleted"; detail: string } +> { + const { data: target, error: targetErr } = await db + .from("document_versions") + .select("id, storage_path, pdf_storage_path, file_type, deleted_at") + .eq("id", versionId) + .eq("document_id", documentId) + .single(); + if (targetErr || !target) + return { + ok: false, + kind: "version_not_found", + detail: "Version not found", + }; + if (target.deleted_at) + return { ok: false, kind: "deleted", detail: "Version is deleted." }; + return { + ok: true, + target: { + storage_path: target.storage_path as string | null, + pdf_storage_path: target.pdf_storage_path as string | null, + file_type: target.file_type as string | null, + }, + }; +} + +export async function writeReplacementVersion( + params: { + userId: string; + documentId: string; + versionId: string; + file: UploadedFile; + suffix: string; + requestedFilename: unknown; + target: { storage_path: string | null; pdf_storage_path: string | null }; + }, + db: Db, +): Promise<{ ok: true; version: unknown } | { ok: false; detail: string }> { + const { userId, documentId, versionId, file, suffix, target } = params; + + const versionSlug = crypto.randomUUID().replace(/-/g, ""); + const key = versionStorageKey( + userId, + documentId, + versionSlug, + file.originalname, + ); + const contentType = contentTypeForDocumentType(suffix); + + try { + await uploadFile( + key, + file.buffer.buffer.slice( + file.buffer.byteOffset, + file.buffer.byteOffset + file.buffer.byteLength, + ) as ArrayBuffer, + contentType, + ); + } catch (e) { + console.error("[versions/replace] storage write failed", e); + return { ok: false, detail: "Failed to upload replacement version." }; + } + + // Same queue deferral as version uploads: the replacement's rendition is + // produced by the conversion worker when the flag is on. The old rendition + // is deleted below either way, so /display briefly falls back until the + // worker writes the new one. + const deferConversion = + shouldConvertToPdf(suffix) && + process.env.ASYNC_DOCUMENT_CONVERSION === "true"; + let pdfStoragePath: string | null = null; + if (!deferConversion && shouldConvertToPdf(suffix)) { + try { + const pdfBuf = await docxToPdf(file.buffer); + const pdfKey = `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`; + await uploadFile( + pdfKey, + pdfBuf.buffer.slice( + pdfBuf.byteOffset, + pdfBuf.byteOffset + pdfBuf.byteLength, + ) as ArrayBuffer, + "application/pdf", + ); + pdfStoragePath = pdfKey; + } catch (err) { + console.error( + `[versions/replace] Office→PDF conversion failed for ${file.originalname}:`, + err, + ); + } + } else if (suffix === "pdf") { + pdfStoragePath = key; + } + + const rawBuf = file.buffer.buffer.slice( + file.buffer.byteOffset, + file.buffer.byteOffset + file.buffer.byteLength, + ) as ArrayBuffer; + const pageCount = suffix === "pdf" ? await countPdfPages(rawBuf) : null; + const requestedFilename = + typeof params.requestedFilename === "string" && + params.requestedFilename.trim() + ? params.requestedFilename.trim().slice(0, 200) + : file.originalname; + const uploadedAt = new Date().toISOString(); + + const { data: updated, error: updateErr } = await db + .from("document_versions") + .update({ + storage_path: key, + pdf_storage_path: pdfStoragePath, + filename: requestedFilename, + file_type: suffix, + size_bytes: file.buffer.byteLength, + page_count: pageCount, + content_sha256: contentSha256(file.buffer), + created_at: uploadedAt, + }) + .eq("id", versionId) + .eq("document_id", documentId) + .select( + "id, version_number, source, created_at, filename, file_type, size_bytes, page_count", + ) + .single(); + if (updateErr || !updated) { + await Promise.all( + [key, pdfStoragePath] + .filter((path): path is string => !!path) + .map((path) => deleteFile(path).catch(() => {})), + ); + return { + ok: false, + detail: updateErr?.message ?? "Failed to replace version.", + }; + } + + await Promise.all( + [target.storage_path, target.pdf_storage_path] + .filter((path): path is string => !!path) + .map((path) => deleteFile(path).catch(() => {})), + ); + + if (deferConversion) { + // Replace reuses the versionId, which is exactly why terminal jobs are + // removed from the queue immediately — this enqueue must not be deduped + // against a completed job for the same version. + await enqueueConversion({ + documentId, + versionId, + userId, + storagePath: key, + fileType: suffix, + pdfKey: `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`, + finalizeDocumentStatus: false, + }); + } + + return { ok: true, version: updated }; +} + +// --------------------------------------------------------------------------- +// Delete a version +// --------------------------------------------------------------------------- + +export async function deleteVersion( + documentId: string, + versionId: string, + userId: string, + userEmail: string | undefined, + db: Db, +): Promise< + | { ok: true; payload: Record } + | { + ok: false; + kind: "doc_not_found" | "version_not_found" | "only_version" | "db"; + detail: string; + } +> { + const access = await ensureDocumentAccess(documentId, userId, userEmail, db, { + select: "id, user_id, project_id, current_version_id", + ownerOnly: true, + }); + if (!access.ok) + return { ok: false, kind: "doc_not_found", detail: "Document not found" }; + const doc = access.doc; + + const { data: versions, error: versionsErr } = await db + .from("document_versions") + .select( + "id, storage_path, pdf_storage_path, version_number, created_at, deleted_at", + ) + .eq("document_id", documentId) + .is("deleted_at", null); + if (versionsErr) { + return { ok: false, kind: "db", detail: versionsErr.message }; + } + + const rows = (versions ?? []) as { + id: string; + storage_path: string | null; + pdf_storage_path: string | null; + version_number: number | null; + created_at: string | null; + deleted_at?: string | null; + }[]; + const target = rows.find((row) => row.id === versionId); + if (!target) + return { ok: false, kind: "version_not_found", detail: "Version not found" }; + if (rows.length <= 1) { + return { + ok: false, + kind: "only_version", + detail: "Cannot delete the only document version.", + }; + } + + const remaining = rows + .filter((row) => row.id !== versionId) + .sort((a, b) => { + const versionDelta = + (b.version_number ?? -1) - (a.version_number ?? -1); + if (versionDelta !== 0) return versionDelta; + return ( + new Date(b.created_at ?? 0).getTime() - + new Date(a.created_at ?? 0).getTime() + ); + }); + const nextCurrentVersionId = + doc.current_version_id === versionId + ? (remaining[0]?.id ?? null) + : doc.current_version_id; + const deletedAt = new Date().toISOString(); + + if (doc.current_version_id === versionId) { + const { error: updateErr } = await db + .from("documents") + .update({ + current_version_id: nextCurrentVersionId, + updated_at: new Date().toISOString(), + }) + .eq("id", documentId); + if (updateErr) { + return { ok: false, kind: "db", detail: updateErr.message }; + } + } + + const { error: deleteErr } = await db + .from("document_versions") + .update({ + storage_path: null, + pdf_storage_path: null, + deleted_at: deletedAt, + deleted_by: userId, + }) + .eq("id", versionId) + .eq("document_id", documentId) + .is("deleted_at", null); + if (deleteErr) { + return { ok: false, kind: "db", detail: deleteErr.message }; + } + + await Promise.all( + [target.storage_path, target.pdf_storage_path] + .filter((path): path is string => !!path) + .map((path) => deleteFile(path).catch(() => {})), + ); + + return { + ok: true, + payload: { + deleted_version_id: versionId, + current_version_id: nextCurrentVersionId, + deleted_at: deletedAt, + }, + }; +} diff --git a/backend/src/routes/downloads.ts b/backend/src/modules/downloads/downloads.routes.ts similarity index 84% rename from backend/src/routes/downloads.ts rename to backend/src/modules/downloads/downloads.routes.ts index 9726f86e5..ea9610055 100644 --- a/backend/src/routes/downloads.ts +++ b/backend/src/modules/downloads/downloads.routes.ts @@ -1,10 +1,10 @@ import { Router } from "express"; -import { requireAuth } from "../middleware/auth"; -import { createServerSupabase } from "../lib/supabase"; -import { buildContentDisposition, downloadFile } from "../lib/storage"; -import { verifyDownload } from "../lib/downloadTokens"; -import { ensureDocAccess } from "../lib/access"; -import { contentTypeForDocumentType } from "../lib/documentTypes"; +import { requireAuth } from "../../middleware/auth"; +import { createServerSupabase } from "../../lib/supabase"; +import { buildContentDisposition, downloadFile } from "../../lib/storage"; +import { verifyDownload } from "../../lib/downloadTokens"; +import { ensureDocAccess } from "../../lib/access"; +import { contentTypeForDocumentType } from "../../lib/documentTypes"; export const downloadsRouter = Router(); diff --git a/backend/src/modules/library/library.routes.ts b/backend/src/modules/library/library.routes.ts new file mode 100644 index 000000000..780ff82c2 --- /dev/null +++ b/backend/src/modules/library/library.routes.ts @@ -0,0 +1,178 @@ +// HTTP surface for the library module. +// +// GET /library/:kind — documents + folders +// POST /library/:kind/documents — upload a document +// POST /library/:kind/folders — create a folder +// PATCH /library/:kind/folders/:folderId — rename / move a folder +// DELETE /library/:kind/folders/:folderId — delete a folder (+ docs) +// PATCH /library/:kind/documents/:documentId/folder — move a document +// PATCH /library/:kind/documents/:documentId — rename a document +// +// `:kind` is "files" | "templates" and maps to library_kind "file" | "template". + +import { Router } from "express"; +import { requireAuth } from "../../middleware/auth"; +import { createServerSupabase } from "../../lib/supabase"; +import { singleFileUpload } from "../../lib/upload"; +import { + ALLOWED_DOCUMENT_TYPES, + ALLOWED_DOCUMENT_TYPES_LABEL, +} from "../../lib/documentTypes"; +import { createDocumentFromUpload } from "../documents/documents.service"; +import { + normalizeLibraryKind, + getLibrary, + createLibraryFolder, + updateLibraryFolder, + deleteLibraryFolder, + moveLibraryDocument, + renameLibraryDocument, +} from "./library.service"; + +export const libraryRouter = Router(); + +// GET /library/:kind +libraryRouter.get("/:kind", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const kind = normalizeLibraryKind(req.params.kind); + if (!kind) return void res.status(404).json({ detail: "Library not found" }); + + const db = createServerSupabase(); + const result = await getLibrary(db, userId, kind); + if (!result.ok) + return void res.status(result.status).json({ detail: result.detail }); + res.json(result.data); +}); + +// POST /library/:kind/documents +libraryRouter.post( + "/:kind/documents", + requireAuth, + singleFileUpload("file"), + async (req, res) => { + const userId = res.locals.userId as string; + const kind = normalizeLibraryKind(req.params.kind); + if (!kind) return void res.status(404).json({ detail: "Library not found" }); + const db = createServerSupabase(); + + const file = req.file; + if (!file) return void res.status(400).json({ detail: "file is required" }); + + const filename = file.originalname; + const suffix = filename.includes(".") + ? filename.split(".").pop()!.toLowerCase() + : ""; + if (!ALLOWED_DOCUMENT_TYPES.has(suffix)) + return void res.status(400).json({ + detail: `Unsupported file type: ${suffix}. Allowed: ${ALLOWED_DOCUMENT_TYPES_LABEL}`, + }); + + const result = await createDocumentFromUpload( + { + userId, + projectId: null, + filename, + suffix, + content: file.buffer, + libraryKind: kind, + }, + db, + ); + if (!result.ok) { + if (result.kind === "create_failed") + return void res + .status(500) + .json({ detail: "Failed to create document record" }); + return void res + .status(500) + .json({ detail: `Document processing failed: ${result.detail}` }); + } + res.status(201).json(result.doc); + }, +); + +// POST /library/:kind/folders +libraryRouter.post("/:kind/folders", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const kind = normalizeLibraryKind(req.params.kind); + if (!kind) return void res.status(404).json({ detail: "Library not found" }); + + const body = req.body as { name?: string; parent_folder_id?: string | null }; + const db = createServerSupabase(); + const result = await createLibraryFolder(db, userId, kind, body); + if (!result.ok) + return void res.status(result.status).json({ detail: result.detail }); + res.status(201).json(result.data); +}); + +// PATCH /library/:kind/folders/:folderId +libraryRouter.patch("/:kind/folders/:folderId", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const kind = normalizeLibraryKind(req.params.kind); + if (!kind) return void res.status(404).json({ detail: "Library not found" }); + + const { folderId } = req.params; + const body = req.body as { name?: string; parent_folder_id?: string | null }; + const db = createServerSupabase(); + const result = await updateLibraryFolder(db, userId, kind, folderId, body); + if (!result.ok) + return void res.status(result.status).json({ detail: result.detail }); + res.json(result.data); +}); + +// DELETE /library/:kind/folders/:folderId +libraryRouter.delete("/:kind/folders/:folderId", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const kind = normalizeLibraryKind(req.params.kind); + if (!kind) return void res.status(404).json({ detail: "Library not found" }); + + const { folderId } = req.params; + const db = createServerSupabase(); + const result = await deleteLibraryFolder(db, userId, kind, folderId); + if (!result.ok) + return void res.status(result.status).json({ detail: result.detail }); + res.status(204).send(); +}); + +// PATCH /library/:kind/documents/:documentId/folder +libraryRouter.patch( + "/:kind/documents/:documentId/folder", + requireAuth, + async (req, res) => { + const userId = res.locals.userId as string; + const kind = normalizeLibraryKind(req.params.kind); + if (!kind) return void res.status(404).json({ detail: "Library not found" }); + + const { documentId } = req.params; + const { folder_id } = req.body as { folder_id: string | null }; + const db = createServerSupabase(); + const result = await moveLibraryDocument(db, userId, kind, documentId, folder_id); + if (!result.ok) + return void res.status(result.status).json({ detail: result.detail }); + res.json(result.data); + }, +); + +// PATCH /library/:kind/documents/:documentId +libraryRouter.patch( + "/:kind/documents/:documentId", + requireAuth, + async (req, res) => { + const userId = res.locals.userId as string; + const kind = normalizeLibraryKind(req.params.kind); + if (!kind) return void res.status(404).json({ detail: "Library not found" }); + + const { documentId } = req.params; + const db = createServerSupabase(); + const result = await renameLibraryDocument( + db, + userId, + kind, + documentId, + req.body?.filename, + ); + if (!result.ok) + return void res.status(result.status).json({ detail: result.detail }); + res.json(result.data); + }, +); diff --git a/backend/src/modules/library/library.service.ts b/backend/src/modules/library/library.service.ts new file mode 100644 index 000000000..0deae310a --- /dev/null +++ b/backend/src/modules/library/library.service.ts @@ -0,0 +1,377 @@ +// Business logic + data access for the library module. +// +// The library organises a user's standalone (project_id === null) documents +// into two collections — "files" and "templates" — each with an optional +// folder tree (library_folders). These functions take an explicit Supabase +// client (`db`) plus request-derived primitives and RETURN typed results; +// the thin route handlers in library.routes.ts map them onto HTTP responses. + +import { createServerSupabase } from "../../lib/supabase"; +import { deleteFile } from "../../lib/storage"; +import { + attachActiveVersionPaths, + attachLatestVersionNumbers, +} from "../../lib/documentVersions"; + +type Db = ReturnType; + +export type LibraryKind = "file" | "template"; + +export function normalizeLibraryKind(value: unknown): LibraryKind | null { + if (value === "file" || value === "files") return "file"; + if (value === "template" || value === "templates") return "template"; + return null; +} + +function normalizeDocumentFilename(nextName: unknown, currentName: string) { + if (typeof nextName !== "string") return null; + const trimmed = nextName.trim().slice(0, 200); + if (!trimmed) return null; + if (/\.[a-z0-9]{1,6}$/i.test(trimmed)) return trimmed; + const ext = currentName.match(/\.[a-z0-9]{1,6}$/i)?.[0] ?? ""; + return `${trimmed}${ext}`; +} + +function mapLibraryDocument>(doc: T) { + return { + ...doc, + folder_id: (doc.library_folder_id as string | null | undefined) ?? null, + }; +} + +async function loadLibraryFolder( + db: Db, + userId: string, + kind: LibraryKind, + folderId: string, +): Promise<{ id: string; parent_folder_id: string | null } | null> { + const { data } = await db + .from("library_folders") + .select("id, parent_folder_id") + .eq("id", folderId) + .eq("user_id", userId) + .eq("library_kind", kind) + .maybeSingle(); + return (data as { id: string; parent_folder_id: string | null } | null) ?? null; +} + +async function deleteLibraryDocumentsAndVersionFiles( + db: Db, + userId: string, + kind: LibraryKind, + documentIds: string[], +) { + if (documentIds.length === 0) return null; + const { data: versions, error: versionsError } = await db + .from("document_versions") + .select("storage_path, pdf_storage_path") + .in("document_id", documentIds); + if (versionsError) return versionsError; + + const paths = new Set(); + for (const version of versions ?? []) { + if (typeof version.storage_path === "string" && version.storage_path) { + paths.add(version.storage_path); + } + if ( + typeof version.pdf_storage_path === "string" && + version.pdf_storage_path + ) { + paths.add(version.pdf_storage_path); + } + } + await Promise.all([...paths].map((path) => deleteFile(path).catch(() => {}))); + + let deleteQuery = db + .from("documents") + .delete() + .eq("user_id", userId) + .is("project_id", null); + deleteQuery = + kind === "file" + ? deleteQuery.or("library_kind.eq.file,library_kind.is.null") + : deleteQuery.eq("library_kind", kind); + const { error } = await deleteQuery.in("id", documentIds); + return error ?? null; +} + +export type ServiceOk = { ok: true; data: T }; +export type ServiceErr = { ok: false; status: number; detail: string }; +export type ServiceResult = ServiceOk | ServiceErr; + +const ok = (data: T): ServiceOk => ({ ok: true, data }); +const err = (status: number, detail: string): ServiceErr => ({ + ok: false, + status, + detail, +}); + +export async function getLibrary( + db: Db, + userId: string, + kind: LibraryKind, +): Promise> { + let documentsQuery = db + .from("documents") + .select("*") + .eq("user_id", userId) + .is("project_id", null); + documentsQuery = + kind === "file" + ? documentsQuery.or("library_kind.eq.file,library_kind.is.null") + : documentsQuery.eq("library_kind", kind); + const [{ data: docs, error: docsError }, { data: folders, error: foldersError }] = + await Promise.all([ + documentsQuery.order("created_at", { ascending: true }), + db + .from("library_folders") + .select("*") + .eq("user_id", userId) + .eq("library_kind", kind) + .order("created_at", { ascending: true }), + ]); + if (docsError) return err(500, docsError.message); + if (foldersError) return err(500, foldersError.message); + + const docsTyped = (docs ?? []).map(mapLibraryDocument) as { + id: string; + current_version_id?: string | null; + }[]; + await attachLatestVersionNumbers(db, docsTyped); + await attachActiveVersionPaths(db, docsTyped); + return ok({ documents: docsTyped, folders: folders ?? [] }); +} + +export async function createLibraryFolder( + db: Db, + userId: string, + kind: LibraryKind, + body: { name?: string; parent_folder_id?: string | null }, +): Promise> { + const { name, parent_folder_id } = body; + if (!name?.trim()) return err(400, "name is required"); + + if (parent_folder_id) { + const parent = await loadLibraryFolder(db, userId, kind, parent_folder_id); + if (!parent) return err(404, "Parent folder not found"); + } + + const { data, error } = await db + .from("library_folders") + .insert({ + user_id: userId, + library_kind: kind, + name: name.trim(), + parent_folder_id: parent_folder_id ?? null, + }) + .select("*") + .single(); + if (error) return err(500, error.message); + return ok(data); +} + +export async function updateLibraryFolder( + db: Db, + userId: string, + kind: LibraryKind, + folderId: string, + body: { name?: string; parent_folder_id?: string | null }, +): Promise> { + const folder = await loadLibraryFolder(db, userId, kind, folderId); + if (!folder) return err(404, "Folder not found"); + + const updates: Record = { + updated_at: new Date().toISOString(), + }; + if (body.name != null) { + const trimmed = body.name.trim(); + if (!trimmed) return err(400, "name is required"); + updates.name = trimmed; + } + if ("parent_folder_id" in body) { + if (body.parent_folder_id) { + let cur: string | null = body.parent_folder_id; + while (cur) { + if (cur === folderId) { + return err(400, "Cannot move a folder into itself or a descendant"); + } + const parent = await loadLibraryFolder(db, userId, kind, cur); + if (!parent) return err(404, "Parent folder not found"); + cur = parent.parent_folder_id ?? null; + } + } + updates.parent_folder_id = body.parent_folder_id ?? null; + } + + const { data, error } = await db + .from("library_folders") + .update(updates) + .eq("id", folderId) + .eq("user_id", userId) + .eq("library_kind", kind) + .select("*") + .single(); + if (error || !data) return err(404, "Folder not found"); + return ok(data); +} + +export async function deleteLibraryFolder( + db: Db, + userId: string, + kind: LibraryKind, + folderId: string, +): Promise> { + const { data: allFolders, error: foldersError } = await db + .from("library_folders") + .select("id, parent_folder_id") + .eq("user_id", userId) + .eq("library_kind", kind); + if (foldersError) return err(500, foldersError.message); + if (!(allFolders ?? []).some((folder) => folder.id === folderId)) { + return err(404, "Folder not found"); + } + + const childrenByParent = new Map(); + for (const folder of allFolders ?? []) { + const parentId = folder.parent_folder_id as string | null; + if (!parentId) continue; + const children = childrenByParent.get(parentId) ?? []; + children.push(folder.id as string); + childrenByParent.set(parentId, children); + } + + const folderIds = new Set(); + const stack = [folderId]; + while (stack.length > 0) { + const id = stack.pop()!; + if (folderIds.has(id)) continue; + folderIds.add(id); + stack.push(...(childrenByParent.get(id) ?? [])); + } + + let documentsInFolderQuery = db + .from("documents") + .select("id") + .eq("user_id", userId) + .is("project_id", null); + documentsInFolderQuery = + kind === "file" + ? documentsInFolderQuery.or("library_kind.eq.file,library_kind.is.null") + : documentsInFolderQuery.eq("library_kind", kind); + const { data: docs, error: docsError } = await documentsInFolderQuery.in( + "library_folder_id", + [...folderIds], + ); + if (docsError) return err(500, docsError.message); + + const docIds = (docs ?? []).map((doc) => doc.id as string); + const deleteDocsError = await deleteLibraryDocumentsAndVersionFiles( + db, + userId, + kind, + docIds, + ); + if (deleteDocsError) return err(500, deleteDocsError.message); + + const { error } = await db + .from("library_folders") + .delete() + .eq("id", folderId) + .eq("user_id", userId) + .eq("library_kind", kind); + if (error) return err(500, error.message); + return ok(null); +} + +export async function moveLibraryDocument( + db: Db, + userId: string, + kind: LibraryKind, + documentId: string, + folder_id: string | null, +): Promise> { + if (folder_id) { + const folder = await loadLibraryFolder(db, userId, kind, folder_id); + if (!folder) return err(404, "Folder not found"); + } + + let moveQuery = db + .from("documents") + .update({ + library_folder_id: folder_id ?? null, + updated_at: new Date().toISOString(), + }) + .eq("id", documentId) + .eq("user_id", userId) + .is("project_id", null); + moveQuery = + kind === "file" + ? moveQuery.or("library_kind.eq.file,library_kind.is.null") + : moveQuery.eq("library_kind", kind); + const { data, error } = await moveQuery + .select("*") + .single(); + if (error || !data) return err(404, "Document not found"); + return ok(mapLibraryDocument(data)); +} + +export async function renameLibraryDocument( + db: Db, + userId: string, + kind: LibraryKind, + documentId: string, + rawFilename: unknown, +): Promise> { + let docQuery = db + .from("documents") + .select("id, current_version_id") + .eq("id", documentId) + .eq("user_id", userId) + .is("project_id", null); + docQuery = + kind === "file" + ? docQuery.or("library_kind.eq.file,library_kind.is.null") + : docQuery.eq("library_kind", kind); + const { data: doc } = await docQuery.single(); + if (!doc) return err(404, "Document not found"); + + const active = doc.current_version_id + ? await db + .from("document_versions") + .select("filename") + .eq("id", doc.current_version_id) + .eq("document_id", documentId) + .single() + : null; + const currentName = + typeof active?.data?.filename === "string" && active.data.filename.trim() + ? active.data.filename.trim() + : "Untitled document"; + const filename = normalizeDocumentFilename(rawFilename, currentName); + if (!filename) return err(400, "filename is required"); + + let updateQuery = db + .from("documents") + .update({ updated_at: new Date().toISOString() }) + .eq("id", documentId) + .eq("user_id", userId) + .is("project_id", null); + updateQuery = + kind === "file" + ? updateQuery.or("library_kind.eq.file,library_kind.is.null") + : updateQuery.eq("library_kind", kind); + const { data: updated, error } = await updateQuery + .select("*") + .single(); + if (error || !updated) return err(404, "Document not found"); + + if (doc.current_version_id) { + await db + .from("document_versions") + .update({ filename }) + .eq("id", doc.current_version_id) + .eq("document_id", documentId); + } + + return ok(mapLibraryDocument({ ...updated, filename })); +} diff --git a/backend/src/routes/models.ts b/backend/src/modules/models/models.routes.ts similarity index 89% rename from backend/src/routes/models.ts rename to backend/src/modules/models/models.routes.ts index 3c3c36c0b..7bc72ffea 100644 --- a/backend/src/routes/models.ts +++ b/backend/src/modules/models/models.routes.ts @@ -1,6 +1,6 @@ import { Router } from "express"; -import { requireAuth } from "../middleware/auth"; -import { authHeaders } from "../lib/llm/ollama"; +import { requireAuth } from "../../middleware/auth"; +import { authHeaders } from "../../lib/llm/ollama"; export const modelsRouter = Router(); diff --git a/backend/src/routes/projectChat.ts b/backend/src/modules/project-chat/projectChat.routes.ts similarity index 54% rename from backend/src/routes/projectChat.ts rename to backend/src/modules/project-chat/projectChat.routes.ts index 8f0565feb..e30c552b6 100644 --- a/backend/src/routes/projectChat.ts +++ b/backend/src/modules/project-chat/projectChat.routes.ts @@ -1,20 +1,20 @@ +// HTTP layer for the project-chat module. +// +// The route handler parses the request body, calls +// prepareProjectChatStream for the pre-stream DB work, and owns the SSE +// streaming loop (header flush, runLLMStream, abort handling, +// assistant-message persistence) — its ordering is delicate. + import { Router } from "express"; -import { requireAuth } from "../middleware/auth"; -import { createServerSupabase } from "../lib/supabase"; +import { requireAuth } from "../../middleware/auth"; +import { createServerSupabase } from "../../lib/supabase"; import { - buildProjectDocContext, - buildMessages, - buildWorkflowStore, - enrichWithPriorEvents, - appendAskInputsResponseToLastAssistantMessage, appendAssistantEventsToLastAssistantMessage, AssistantStreamError, buildCancelledAssistantMessage, extractCitations, - generateSpotlightNonce, isAbortError, runLLMStream, - spotlightFilename, stripTransientAssistantEvents, PROJECT_EXTRA_TOOLS, parseChatMessages, @@ -23,21 +23,9 @@ import { parseOptionalChatId, parseOptionalDisplayedDoc, parseOptionalModel, - type ChatMessage, -} from "../lib/chat"; -import { - getUserModelSettings, -} from "../lib/userSettings"; -import { checkProjectAccess } from "../lib/access"; -import { safeErrorLog, safeErrorMessage } from "../lib/safeError"; - -const PROJECT_SYSTEM_PROMPT_EXTRA = `PROJECT CONTEXT: -You are operating within a project folder that contains a collection of legal documents the user has organised for a single matter. The user's questions will usually refer to one or more documents in this project — your job is to find the relevant files to work on. Use list_documents to see what is available and fetch_documents / read_document to pull in any documents you need before answering. - -A document may currently be displayed in the user's side panel; when provided, treat it as context for the user's likely focus, but do NOT assume it is the only or definitive document the user is asking about. If the request could apply to other files in the project, identify and read those as well. Prefer coverage across the relevant project documents over an over-narrow reading of only the displayed one. - -REPLICATING A DOCUMENT: -When the user wants to use an existing project document as a starting point for a new file (e.g. "use this NDA as a template", "make me a copy of the SOW so I can edit it", "duplicate this and adapt it for company X"), call the replicate_document tool with the source doc_id. This creates a byte-for-byte copy as a new project document, returns a fresh doc_id slug, and shows a download/open card in the UI. Then call edit_document on the returned slug to make the user's requested changes — do NOT call generate_docx for cases where the user clearly wants the existing document's structure and formatting preserved.`; +} from "../../lib/chat"; +import { safeErrorLog, safeErrorMessage } from "../../lib/safeError"; +import { prepareProjectChatStream } from "./projectChat.service"; export const projectChatRouter = Router({ mergeParams: true }); @@ -92,146 +80,31 @@ projectChatRouter.post("/", requireAuth, async (req, res) => { const db = createServerSupabase(); - // Verify the user has access to the project (owner or shared member). - const projectAccess = await checkProjectAccess( - projectId, + const prep = await prepareProjectChatStream(db, { userId, userEmail, - db, - ); - if (!projectAccess.ok) - return void res.status(404).json({ detail: "Project not found" }); - - let chatId = chat_id ?? null; - let chatTitle: string | null = null; - - if (chatId) { - const { data: existing } = await db - .from("chats") - .select("id, title, project_id") - .eq("id", chatId) - .single(); - const canUse = !!existing && existing.project_id === projectId; - if (!canUse) chatId = null; - else chatTitle = existing!.title; - } - - if (!chatId) { - const { data: newChat, error } = await db - .from("chats") - .insert({ user_id: userId, project_id: projectId }) - .select("id, title") - .single(); - if (error || !newChat) - return void res - .status(500) - .json({ detail: "Failed to create chat" }); - chatId = newChat.id as string; - chatTitle = newChat.title; - } - - const lastUser = [...messages].reverse().find((m) => m.role === "user"); - if (askInputsResponse) { - await appendAskInputsResponseToLastAssistantMessage( - db, - chatId, - askInputsResponse, - ); - } else if (lastUser) { - await db.from("chat_messages").insert({ - chat_id: chatId, - role: "user", - content: lastUser.content, - files: lastUser.files ?? null, - workflow: lastUser.workflow ?? null, - }); - } - - const { docIndex, docStore, folderPaths } = await buildProjectDocContext( projectId, - userId, - db, - ); - const docAvailability = Object.entries(docIndex).map(([doc_id, info]) => ({ - doc_id, - filename: info.filename, - folder_path: folderPaths.get(doc_id), - })); - const documentsById = new Map( - Object.entries(docIndex).map(([slug, document]) => [ - document.document_id, - { slug, filename: document.filename }, - ] as const), - ); - // Generate the nonce before adding request metadata or prior events so - // every document filename is fenced wherever it enters the prompt. - const nonce = generateSpotlightNonce(); - const documentPromptRef = ( - documentId: string, - requestFilename: string, - ) => { - const document = documentsById.get(documentId); - return { - slug: document?.slug, - filename: spotlightFilename( - document?.filename ?? requestFilename, - nonce, - ), - }; - }; - - const enrichedMessages = await enrichWithPriorEvents( messages, - chatId, - db, - docIndex, - nonce, - ); - const messagesForLLM: ChatMessage[] = displayed_doc - ? enrichedMessages.map((m, i) => { - if (i !== enrichedMessages.length - 1 || m.role !== "user") - return m; - const displayedDocument = documentPromptRef( - displayed_doc.document_id, - displayed_doc.filename, - ); - return { - ...m, - content: `${m.content}\n\ndisplayed_doc: ${displayedDocument.filename}, displayed_doc_id: ${displayed_doc.document_id}`, - }; - }) - : enrichedMessages; - - // The user-attached docs for this turn (dragged into / picked from - // the chat input) come in as a request-level field. Surface them in - // the system prompt with the current-turn doc_id slugs so the model - // knows which docs the user is highlighting *now*, distinct from - // the broader project doc list. - let systemPromptExtra = PROJECT_SYSTEM_PROMPT_EXTRA; - if (attached_documents?.length) { - const lines = attached_documents.map((d) => { - const document = documentPromptRef(d.document_id, d.filename); - return document.slug - ? `- ${document.slug}: ${document.filename}` - : `- ${document.filename}`; - }); - systemPromptExtra += `\n\nUSER-ATTACHED DOCUMENTS FOR THIS TURN:\nThe user has attached the following document(s) directly to their latest message. Treat these as the primary focus of the request unless their message clearly says otherwise.\n${lines.join("\n")}`; - } + chatId: chat_id ?? null, + displayed_doc, + attached_documents, + askInputsResponse, + }); + if (!prep.ok) + return void res.status(prep.status).json({ detail: prep.detail }); const { - api_keys: apiKeys, - legal_research_us: legalResearchUs, - } = await getUserModelSettings(userId, db); - const apiMessages = buildMessages( - messagesForLLM, - docAvailability, - systemPromptExtra, - undefined, + chatId, + chatTitle, + lastUser, + docIndex, + docStore, + apiMessages, + workflowStore, legalResearchUs, + apiKeys, nonce, - ); - - const workflowStore = await buildWorkflowStore(userId, userEmail, db); + } = prep.prepared; res.setHeader("Content-Type", "text/event-stream"); res.setHeader("Cache-Control", "no-cache"); diff --git a/backend/src/modules/project-chat/projectChat.service.ts b/backend/src/modules/project-chat/projectChat.service.ts new file mode 100644 index 000000000..90d426d40 --- /dev/null +++ b/backend/src/modules/project-chat/projectChat.service.ts @@ -0,0 +1,235 @@ +// Business logic + data-access for the project-chat module. +// +// Service layer behind projectChat.routes.ts. Takes an explicit Supabase client +// (`db`) plus request-derived primitives, does the pre-stream DB orchestration, +// and RETURNS the prepared data (or a typed error). It never touches req/res. +// +// IMPORTANT: the SSE streaming loop (header flush, runLLMStream, abort +// handling, assistant-message persistence) stays in the route — its ordering +// is delicate. Only the pre-stream preparation lives here. + +import { createServerSupabase } from "../../lib/supabase"; +import { + buildProjectDocContext, + buildMessages, + buildWorkflowStore, + enrichWithPriorEvents, + appendAskInputsResponseToLastAssistantMessage, + generateSpotlightNonce, + spotlightFilename, + type AskInputsResponseRequest, + type ChatDocumentReference, + type ChatMessage, +} from "../../lib/chat"; +import { + getUserModelSettings, +} from "../../lib/userSettings"; +import { checkProjectAccess } from "../../lib/access"; + +type Db = ReturnType; + +const PROJECT_SYSTEM_PROMPT_EXTRA = `PROJECT CONTEXT: +You are operating within a project folder that contains a collection of legal documents the user has organised for a single matter. The user's questions will usually refer to one or more documents in this project — your job is to find the relevant files to work on. Use list_documents to see what is available and fetch_documents / read_document to pull in any documents you need before answering. + +A document may currently be displayed in the user's side panel; when provided, treat it as context for the user's likely focus, but do NOT assume it is the only or definitive document the user is asking about. If the request could apply to other files in the project, identify and read those as well. Prefer coverage across the relevant project documents over an over-narrow reading of only the displayed one. + +REPLICATING A DOCUMENT: +When the user wants to use an existing project document as a starting point for a new file (e.g. "use this NDA as a template", "make me a copy of the SOW so I can edit it", "duplicate this and adapt it for company X"), call the replicate_document tool with the source doc_id. This creates a byte-for-byte copy as a new project document, returns a fresh doc_id slug, and shows a download/open card in the UI. Then call edit_document on the returned slug to make the user's requested changes — do NOT call generate_docx for cases where the user clearly wants the existing document's structure and formatting preserved.`; + +export type PreparedProjectChatStream = { + chatId: string; + chatTitle: string | null; + lastUser: ChatMessage | undefined; + docIndex: Awaited>["docIndex"]; + docStore: Awaited>["docStore"]; + apiMessages: ReturnType; + workflowStore: Awaited>; + legalResearchUs: boolean; + apiKeys: Awaited>["api_keys"]; + nonce: ReturnType; +}; + +export async function prepareProjectChatStream( + db: Db, + args: { + userId: string; + userEmail: string | undefined; + projectId: string; + messages: ChatMessage[]; + chatId: string | null; + displayed_doc: ChatDocumentReference | undefined; + attached_documents: ChatDocumentReference[] | undefined; + // Parsed `ask_inputs_response` payload (answers to an ask_inputs + // event emitted by the assistant in a prior turn). When present, the + // user's answers are appended onto the previous assistant message + // instead of being stored as a new user message. + askInputsResponse: AskInputsResponseRequest | null; + }, +): Promise< + | { ok: true; prepared: PreparedProjectChatStream } + | { ok: false; status: number; detail: string } +> { + const { + userId, + userEmail, + projectId, + messages, + displayed_doc, + attached_documents, + } = args; + + // Verify the user has access to the project (owner or shared member). + const projectAccess = await checkProjectAccess( + projectId, + userId, + userEmail, + db, + ); + if (!projectAccess.ok) + return { ok: false, status: 404, detail: "Project not found" }; + + let chatId = args.chatId; + let chatTitle: string | null = null; + + if (chatId) { + const { data: existing } = await db + .from("chats") + .select("id, title, project_id") + .eq("id", chatId) + .single(); + const canUse = !!existing && existing.project_id === projectId; + if (!canUse) chatId = null; + else chatTitle = existing!.title; + } + + if (!chatId) { + const { data: newChat, error } = await db + .from("chats") + .insert({ user_id: userId, project_id: projectId }) + .select("id, title") + .single(); + if (error || !newChat) + return { ok: false, status: 500, detail: "Failed to create chat" }; + chatId = newChat.id as string; + chatTitle = newChat.title; + } + + const lastUser = [...messages].reverse().find((m) => m.role === "user"); + if (args.askInputsResponse) { + await appendAskInputsResponseToLastAssistantMessage( + db, + chatId, + args.askInputsResponse, + ); + } else if (lastUser) { + await db.from("chat_messages").insert({ + chat_id: chatId, + role: "user", + content: lastUser.content, + files: lastUser.files ?? null, + workflow: lastUser.workflow ?? null, + }); + } + + const { docIndex, docStore, folderPaths } = await buildProjectDocContext( + projectId, + userId, + db, + ); + const docAvailability = Object.entries(docIndex).map(([doc_id, info]) => ({ + doc_id, + filename: info.filename, + folder_path: folderPaths.get(doc_id), + })); + const documentsById = new Map( + Object.entries(docIndex).map(([slug, document]) => [ + document.document_id, + { slug, filename: document.filename }, + ] as const), + ); + // Generate the nonce before adding request metadata or prior events so + // every document filename is fenced wherever it enters the prompt. + const nonce = generateSpotlightNonce(); + const documentPromptRef = ( + documentId: string, + requestFilename: string, + ) => { + const document = documentsById.get(documentId); + return { + slug: document?.slug, + filename: spotlightFilename( + document?.filename ?? requestFilename, + nonce, + ), + }; + }; + + const enrichedMessages = await enrichWithPriorEvents( + messages, + chatId, + db, + docIndex, + nonce, + ); + const messagesForLLM: ChatMessage[] = displayed_doc + ? enrichedMessages.map((m, i) => { + if (i !== enrichedMessages.length - 1 || m.role !== "user") + return m; + const displayedDocument = documentPromptRef( + displayed_doc.document_id, + displayed_doc.filename, + ); + return { + ...m, + content: `${m.content}\n\ndisplayed_doc: ${displayedDocument.filename}, displayed_doc_id: ${displayed_doc.document_id}`, + }; + }) + : enrichedMessages; + + // The user-attached docs for this turn (dragged into / picked from + // the chat input) come in as a request-level field. Surface them in + // the system prompt with the current-turn doc_id slugs so the model + // knows which docs the user is highlighting *now*, distinct from + // the broader project doc list. + let systemPromptExtra = PROJECT_SYSTEM_PROMPT_EXTRA; + if (attached_documents?.length) { + const lines = attached_documents.map((d) => { + const document = documentPromptRef(d.document_id, d.filename); + return document.slug + ? `- ${document.slug}: ${document.filename}` + : `- ${document.filename}`; + }); + systemPromptExtra += `\n\nUSER-ATTACHED DOCUMENTS FOR THIS TURN:\nThe user has attached the following document(s) directly to their latest message. Treat these as the primary focus of the request unless their message clearly says otherwise.\n${lines.join("\n")}`; + } + + const { + api_keys: apiKeys, + legal_research_us: legalResearchUs, + } = await getUserModelSettings(userId, db); + const apiMessages = buildMessages( + messagesForLLM, + docAvailability, + systemPromptExtra, + undefined, + legalResearchUs, + nonce, + ); + + const workflowStore = await buildWorkflowStore(userId, userEmail, db); + + return { + ok: true, + prepared: { + chatId, + chatTitle, + lastUser, + docIndex, + docStore, + apiMessages, + workflowStore, + legalResearchUs, + apiKeys, + nonce, + }, + }; +} diff --git a/backend/src/modules/projects/projects.chats.ts b/backend/src/modules/projects/projects.chats.ts new file mode 100644 index 000000000..018b5bc64 --- /dev/null +++ b/backend/src/modules/projects/projects.chats.ts @@ -0,0 +1,28 @@ +// Project chat service functions: list a project's assistant chats. + +import { checkProjectAccess } from "../../lib/access"; +import { type Db, attachChatCreatorLabels } from "./projects.shared"; + +export async function listProjectChats( + db: Db, + args: { projectId: string; userId: string; userEmail?: string }, +): Promise< + | { ok: true; chats: unknown[] } + | { ok: false; kind: "forbidden" } + | { ok: false; kind: "db_error"; detail: string } +> { + const { projectId, userId, userEmail } = args; + + const access = await checkProjectAccess(projectId, userId, userEmail, db); + if (!access.ok) return { ok: false, kind: "forbidden" }; + + const { data, error } = await db + .from("chats") + .select("*") + .eq("project_id", projectId) + .order("created_at", { ascending: false }); + if (error) return { ok: false, kind: "db_error", detail: error.message }; + const chats = data ?? []; + await attachChatCreatorLabels(db, chats); + return { ok: true, chats }; +} diff --git a/backend/src/modules/projects/projects.crud.ts b/backend/src/modules/projects/projects.crud.ts new file mode 100644 index 000000000..f5407ea7a --- /dev/null +++ b/backend/src/modules/projects/projects.crud.ts @@ -0,0 +1,380 @@ +// Project CRUD service functions: overview, create, detail, people, update, +// delete, and the tamper-evident export manifest. + +import { + attachActiveVersionPaths, + attachLatestVersionNumbers, +} from "../../lib/documentVersions"; +import { safeErrorLog } from "../../lib/safeError"; +import { + buildProjectExportManifest, + projectManifestFilename, +} from "../../lib/userDataExport"; +import { checkProjectAccess } from "../../lib/access"; +import { deleteUserProjects } from "../../lib/userDataCleanup"; +import { + findMissingUserEmails, + loadProfileUsersByEmail, +} from "../../lib/userLookup"; +import { + type Db, + attachDocumentOwnerLabels, + normalizeOptionalString, + normalizeSharedWith, +} from "./projects.shared"; + +// Pass includeDocuments to also receive each project's documents in the +// same response. The directory pickers (useDirectoryData) previously fanned +// out one GET /projects/:id per project to obtain those documents; with N +// projects that burst — auth check plus several DB queries per request — +// could overwhelm the Supabase gateway. Batching keeps it at one request +// and a fixed number of queries regardless of project count. +export async function getProjectsOverview( + db: Db, + args: { + userId: string; + userEmail?: string; + includeDocuments: boolean; + }, +): Promise<{ ok: true; data: unknown } | { ok: false; detail: string }> { + const { userId, userEmail, includeDocuments } = args; + + const { data, error } = await db.rpc("get_projects_overview", { + p_user_id: userId, + p_user_email: userEmail ?? null, + }); + if (error) return { ok: false, detail: error.message }; + + const projects = (data ?? []) as { id: string }[]; + if (!includeDocuments || projects.length === 0) { + return { ok: true, data: projects }; + } + + const projectIds = projects.map((p) => p.id); + const [ + { data: docs, error: docsError }, + { data: folders, error: foldersError }, + ] = await Promise.all([ + db + .from("documents") + .select("*") + .in("project_id", projectIds) + .order("created_at", { ascending: true }), + db + .from("project_subfolders") + .select("*") + .in("project_id", projectIds) + .order("created_at", { ascending: true }), + ]); + if (docsError) return { ok: false, detail: docsError.message }; + if (foldersError) return { ok: false, detail: foldersError.message }; + + const docsTyped = (docs ?? []) as unknown as { + id: string; + project_id?: string | null; + user_id?: string | null; + current_version_id?: string | null; + }[]; + await attachLatestVersionNumbers(db, docsTyped); + await attachActiveVersionPaths(db, docsTyped); + await attachDocumentOwnerLabels(db, docsTyped); + + const docsByProject = new Map(); + for (const doc of docsTyped) { + if (!doc.project_id) continue; + const bucket = docsByProject.get(doc.project_id); + if (bucket) bucket.push(doc); + else docsByProject.set(doc.project_id, [doc]); + } + const foldersByProject = new Map>(); + for (const folder of folders ?? []) { + const projectId = folder.project_id as string; + const bucket = foldersByProject.get(projectId); + if (bucket) bucket.push(folder); + else foldersByProject.set(projectId, [folder]); + } + return { + ok: true, + data: projects.map((p) => ({ + ...p, + documents: docsByProject.get(p.id) ?? [], + folders: foldersByProject.get(p.id) ?? [], + })), + }; +} + +export type CreateProjectResult = + | { ok: true; project: Record } + | { ok: false; kind: "validation" | "self_share"; detail: string } + | { ok: false; kind: "db_error"; detail: string }; + +export async function createProject( + db: Db, + args: { + userId: string; + userEmail?: string; + name: string; + cm_number?: string; + practice?: string; + shared_with?: string[]; + }, +): Promise { + const { userId, userEmail, name, cm_number, practice, shared_with } = args; + if (!name?.trim()) + return { ok: false, kind: "validation", detail: "name is required" }; + const normalizedUserEmail = userEmail?.trim().toLowerCase(); + const shared = normalizeSharedWith(shared_with, normalizedUserEmail); + if (!shared.ok) { + return { + ok: false, + kind: "self_share", + detail: "You cannot share a project with yourself.", + }; + } + const cleanedSharedWith = shared.cleaned; + + const missingSharedUsers = await findMissingUserEmails(db, cleanedSharedWith); + if (missingSharedUsers.length > 0) { + return { + ok: false, + kind: "validation", + detail: `${missingSharedUsers[0]} does not belong to a Mike user.`, + }; + } + + const { data, error } = await db + .from("projects") + .insert({ + user_id: userId, + name: name.trim(), + cm_number: normalizeOptionalString(cm_number), + practice: normalizeOptionalString(practice), + shared_with: cleanedSharedWith, + }) + .select("*") + .single(); + if (error) return { ok: false, kind: "db_error", detail: error.message }; + return { ok: true, project: { ...data, documents: [] } }; +} + +export async function getProjectDetail( + db: Db, + args: { projectId: string; userId: string; userEmail: string }, +): Promise<{ ok: true; body: Record } | { ok: false }> { + const { projectId, userId, userEmail } = args; + + const { data: project, error } = await db + .from("projects") + .select("*") + .eq("id", projectId) + .single(); + if (error || !project) return { ok: false }; + + const canAccess = + project.user_id === userId || + (userEmail && + Array.isArray(project.shared_with) && + project.shared_with.includes(userEmail)); + if (!canAccess) return { ok: false }; + + const [{ data: docs }, { data: folderData }] = await Promise.all([ + db.from("documents").select("*").eq("project_id", projectId).order("created_at", { ascending: true }), + db.from("project_subfolders").select("*").eq("project_id", projectId).order("created_at", { ascending: true }), + ]); + const docsTyped = (docs ?? []) as unknown as { + id: string; + user_id?: string | null; + current_version_id?: string | null; + }[]; + await attachLatestVersionNumbers(db, docsTyped); + await attachActiveVersionPaths(db, docsTyped); + await attachDocumentOwnerLabels(db, docsTyped); + return { + ok: true, + body: { + ...project, + is_owner: project.user_id === userId, + documents: docsTyped, + folders: folderData ?? [], + }, + }; +} + +export async function getProjectPeople( + db: Db, + args: { projectId: string; userId: string; userEmail?: string }, +): Promise< + | { + ok: true; + body: { + owner: { + user_id: unknown; + email: string | null; + display_name: string | null; + }; + members: { email: string; display_name: string | null }[]; + }; + } + | { ok: false } +> { + const { projectId, userId, userEmail } = args; + + const { data: project } = await db + .from("projects") + .select("id, user_id, shared_with") + .eq("id", projectId) + .single(); + if (!project) return { ok: false }; + + const isOwner = project.user_id === userId; + const sharedWith = (Array.isArray(project.shared_with) + ? (project.shared_with as string[]) + : [] + ).map((e) => e.toLowerCase()); + const isShared = + !!userEmail && sharedWith.includes(userEmail.toLowerCase()); + if (!isOwner && !isShared) return { ok: false }; + + // Use the mirrored profile email so sharing checks do not scan auth.users. + const { userByEmail, userById } = await loadProfileUsersByEmail(db); + + const ownerInfo = userById.get(project.user_id as string); + const owner = { + user_id: project.user_id, + email: ownerInfo?.email ?? null, + display_name: ownerInfo?.display_name ?? null, + }; + const members = sharedWith.map((email) => { + const u = userByEmail.get(email); + const display_name = u?.display_name ?? null; + return { email, display_name }; + }); + + return { ok: true, body: { owner, members } }; +} + +export type UpdateProjectResult = + | { ok: true; body: Record } + | { ok: false; kind: "self_share" | "missing_user"; detail: string } + | { ok: false; kind: "not_found" }; + +export async function updateProject( + db: Db, + args: { + projectId: string; + userId: string; + userEmail?: string; + body: Record; + }, +): Promise { + const { projectId, userId, userEmail, body } = args; + const updates: Record = {}; + if (body.name != null) updates.name = body.name; + if (body.cm_number != null) updates.cm_number = body.cm_number; + if ("practice" in body) { + updates.practice = normalizeOptionalString(body.practice); + } + if (Array.isArray(body.shared_with)) { + // Normalise: lowercase + dedupe + drop empties. + const normalizedUserEmail = userEmail?.trim().toLowerCase(); + const shared = normalizeSharedWith(body.shared_with, normalizedUserEmail); + if (!shared.ok) { + return { + ok: false, + kind: "self_share", + detail: "You cannot share a project with yourself.", + }; + } + updates.shared_with = shared.cleaned; + } + + if (Array.isArray(updates.shared_with)) { + const missingSharedUsers = await findMissingUserEmails( + db, + updates.shared_with as string[], + ); + if (missingSharedUsers.length > 0) { + return { + ok: false, + kind: "missing_user", + detail: `${missingSharedUsers[0]} does not belong to a Mike user.`, + }; + } + } + + const { data, error } = await db + .from("projects") + .update({ ...updates, updated_at: new Date().toISOString() }) + .eq("id", projectId) + .eq("user_id", userId) + .select("*") + .single(); + if (error || !data) return { ok: false, kind: "not_found" }; + + const [{ data: docs }, { data: folderData }] = await Promise.all([ + db.from("documents").select("*").eq("project_id", projectId).order("created_at", { ascending: true }), + db.from("project_subfolders").select("*").eq("project_id", projectId).order("created_at", { ascending: true }), + ]); + const docsTyped = (docs ?? []) as unknown as { + id: string; + user_id?: string | null; + current_version_id?: string | null; + }[]; + await attachActiveVersionPaths(db, docsTyped); + await attachDocumentOwnerLabels(db, docsTyped); + return { + ok: true, + body: { ...data, documents: docsTyped, folders: folderData ?? [] }, + }; +} + +export async function deleteProject( + db: Db, + userId: string, + projectId: string, +): Promise< + | { ok: true } + | { ok: false; kind: "not_found" } + | { ok: false; kind: "error"; detail: string } +> { + try { + const deletedCount = await deleteUserProjects(db, userId, [projectId]); + if (deletedCount === 0) return { ok: false, kind: "not_found" }; + return { ok: true }; + } catch (err) { + const detail = err instanceof Error ? err.message : String(err); + return { ok: false, kind: "error", detail }; + } +} + +// Tamper-evident manifest of the project's documents: every version with its +// content_sha256 plus the accept/reject trail, under a SHA-256 digest that is +// Ed25519-signed when the deployment has MANIFEST_SIGNING_KEY set. To check an +// export, recompute a downloaded file's SHA-256 and compare, then check the +// manifest's signature against the key served at GET /manifest-signing-key. +// See the README. +export type ExportProjectResult = + | { ok: true; data: unknown; filename: string } + | { ok: false; kind: "forbidden" } + | { ok: false; kind: "failed" }; + +export async function exportProjectManifest( + db: Db, + args: { projectId: string; userId: string; userEmail?: string }, +): Promise { + const { projectId, userId, userEmail } = args; + + const access = await checkProjectAccess(projectId, userId, userEmail, db); + if (!access.ok) return { ok: false, kind: "forbidden" }; + + try { + const data = await buildProjectExportManifest(db, projectId); + return { ok: true, data, filename: projectManifestFilename(projectId) }; + } catch (err) { + console.error("[projects/export] failed", { + projectId, + error: safeErrorLog(err), + }); + return { ok: false, kind: "failed" }; + } +} diff --git a/backend/src/modules/projects/projects.documents.ts b/backend/src/modules/projects/projects.documents.ts new file mode 100644 index 000000000..b4e902717 --- /dev/null +++ b/backend/src/modules/projects/projects.documents.ts @@ -0,0 +1,461 @@ +// Project document service functions: list, assign/copy an existing document +// into a project, rename, and the upload processing pipeline. + +import { + attachActiveVersionPaths, + contentSha256, +} from "../../lib/documentVersions"; +import { + deleteFile, + downloadFile, + uploadFile, + storageKey, +} from "../../lib/storage"; +import { docxToPdf, convertedPdfKey } from "../../lib/convert"; +import { enqueueConversion } from "../../lib/queue/conversionQueue"; +import { checkProjectAccess } from "../../lib/access"; +import { + contentTypeForDocumentType, + shouldConvertToPdf, +} from "../../lib/documentTypes"; +import { + type Db, + countPdfPages, + normalizeDocumentFilename, +} from "./projects.shared"; + +export async function listProjectDocuments( + db: Db, + args: { projectId: string; userId: string; userEmail?: string }, +): Promise<{ ok: true; docs: unknown } | { ok: false; kind: "forbidden" }> { + const { projectId, userId, userEmail } = args; + + const access = await checkProjectAccess(projectId, userId, userEmail, db); + if (!access.ok) return { ok: false, kind: "forbidden" }; + + const { data: docs } = await db + .from("documents") + .select("*") + .eq("project_id", projectId) + .order("created_at", { ascending: true }); + const docsTyped = (docs ?? []) as unknown as { + id: string; + current_version_id?: string | null; + }[]; + await attachActiveVersionPaths(db, docsTyped); + return { ok: true, docs: docsTyped }; +} + +export type AssignOrCopyResult = + | { ok: true; status: 200 | 201; doc: unknown } + | { ok: false; kind: "forbidden" } + | { ok: false; kind: "doc_not_found" } + | { ok: false; kind: "update_failed" } + | { ok: false; kind: "no_active_version" } + | { ok: false; kind: "read_failed" } + | { ok: false; kind: "copy_failed" }; + +export async function assignOrCopyDocument( + db: Db, + args: { + projectId: string; + documentId: string; + userId: string; + userEmail?: string; + }, +): Promise { + const { projectId, documentId, userId, userEmail } = args; + + const access = await checkProjectAccess(projectId, userId, userEmail, db); + if (!access.ok) return { ok: false, kind: "forbidden" }; + + // Adding-by-id pulls a doc into the project — only the doc's owner + // is allowed to do that, so other people's standalone docs can't be + // siphoned into a project the requester happens to share. + const { data: doc } = await db + .from("documents") + .select("*") + .eq("id", documentId) + .eq("user_id", userId) + .single(); + if (!doc) return { ok: false, kind: "doc_not_found" }; + await attachActiveVersionPaths( + db, + [doc as { id: string; current_version_id?: string | null }], + ); + + // Already in this project — idempotent + if (doc.project_id === projectId) return { ok: true, status: 200, doc }; + + if (doc.project_id === null) { + // Standalone → assign project_id + const { data: updated, error } = await db + .from("documents") + .update({ + project_id: projectId, + library_folder_id: null, + updated_at: new Date().toISOString(), + }) + .eq("id", documentId) + .select("*") + .single(); + if (error || !updated) return { ok: false, kind: "update_failed" }; + await attachActiveVersionPaths( + db, + [updated as { id: string; current_version_id?: string | null }], + ); + return { ok: true, status: 200, doc: updated }; + } else { + // Belongs to another project → duplicate record AND copy the + // underlying storage objects so each project's copy is fully + // independent (edits/version bumps on one don't leak into the + // other). + if (!doc.current_version_id) { + return { ok: false, kind: "no_active_version" }; + } + + const { data: srcV } = await db + .from("document_versions") + .select( + "storage_path, pdf_storage_path, version_number, filename, source, file_type, size_bytes, page_count", + ) + .eq("id", doc.current_version_id) + .single(); + if (!srcV?.storage_path) { + return { ok: false, kind: "no_active_version" }; + } + + const activeVersionFilename = + (srcV.filename as string | null)?.trim() || "Untitled document"; + const srcBytes = await downloadFile(srcV.storage_path); + if (!srcBytes) { + return { ok: false, kind: "read_failed" }; + } + + const { data: copy, error } = await db + .from("documents") + .insert({ + project_id: projectId, + user_id: userId, + status: doc.status, + }) + .select("*") + .single(); + if (error || !copy) return { ok: false, kind: "copy_failed" }; + + const newKey = storageKey( + userId, + copy.id as string, + activeVersionFilename, + ); + let newPdfPath: string | null = null; + try { + const contentType = contentTypeForDocumentType( + (srcV.file_type as string | null) ?? doc.file_type, + ); + await uploadFile(newKey, srcBytes, contentType); + + // PDFs share one object for source + display rendition. DOCX + // store the converted PDF at a separate `converted-pdfs/` key — + // copy that too if it exists so the copy renders without going + // back through libreoffice. + if (srcV.pdf_storage_path) { + if (srcV.pdf_storage_path === srcV.storage_path) { + newPdfPath = newKey; + } else { + const pdfBytes = await downloadFile(srcV.pdf_storage_path); + if (pdfBytes) { + const newPdfKey = convertedPdfKey(userId, copy.id as string); + await uploadFile(newPdfKey, pdfBytes, "application/pdf"); + newPdfPath = newPdfKey; + } + } + } + + const { data: newV, error: newVError } = await db + .from("document_versions") + .insert({ + document_id: copy.id, + storage_path: newKey, + pdf_storage_path: newPdfPath, + source: (srcV.source as string | null) ?? "upload", + version_number: srcV.version_number ?? 1, + filename: activeVersionFilename, + file_type: (srcV.file_type as string | null) ?? doc.file_type, + size_bytes: + (srcV.size_bytes as number | null) ?? doc.size_bytes ?? null, + page_count: + (srcV.page_count as number | null) ?? doc.page_count ?? null, + content_sha256: contentSha256(srcBytes), + }) + .select("id") + .single(); + const copyVersionRowId = (newV?.id as string | null) ?? null; + if (newVError || !copyVersionRowId) { + throw new Error( + `Failed to create copied document version: ${newVError?.message ?? "unknown"}`, + ); + } + + const { data: updatedCopy, error: updateCopyError } = await db + .from("documents") + .update({ + current_version_id: copyVersionRowId, + }) + .eq("id", copy.id) + .select("*") + .single(); + if (updateCopyError || !updatedCopy) { + throw new Error( + `Failed to activate copied document version: ${updateCopyError?.message ?? "unknown"}`, + ); + } + + await attachActiveVersionPaths( + db, + [updatedCopy as { id: string; current_version_id?: string | null }], + ); + return { ok: true, status: 201, doc: updatedCopy }; + } catch (err) { + console.error("[projects/documents/copy] failed", err); + await Promise.all([ + deleteFile(newKey).catch(() => {}), + newPdfPath && newPdfPath !== newKey + ? deleteFile(newPdfPath).catch(() => {}) + : Promise.resolve(), + db.from("documents").delete().eq("id", copy.id), + ]); + return { ok: false, kind: "copy_failed" }; + } + } +} + +export type RenameDocumentResult = + | { ok: true; doc: Record } + | { ok: false; kind: "forbidden" } + | { ok: false; kind: "doc_not_found" } + | { ok: false; kind: "validation"; detail: string }; + +export async function renameProjectDocument( + db: Db, + args: { + projectId: string; + documentId: string; + userId: string; + userEmail?: string; + filename: unknown; + }, +): Promise { + const { projectId, documentId, userId, userEmail } = args; + + const access = await checkProjectAccess(projectId, userId, userEmail, db); + if (!access.ok) return { ok: false, kind: "forbidden" }; + + const { data: doc } = await db + .from("documents") + .select("id, current_version_id") + .eq("id", documentId) + .eq("project_id", projectId) + .single(); + if (!doc) return { ok: false, kind: "doc_not_found" }; + + const active = doc.current_version_id + ? await db + .from("document_versions") + .select("filename") + .eq("id", doc.current_version_id) + .eq("document_id", documentId) + .single() + : null; + const currentName = + typeof active?.data?.filename === "string" && + active.data.filename.trim() + ? active.data.filename.trim() + : "Untitled document"; + const filename = normalizeDocumentFilename(args.filename, currentName); + if (!filename) + return { ok: false, kind: "validation", detail: "filename is required" }; + + const { data: updated, error } = await db + .from("documents") + .update({ updated_at: new Date().toISOString() }) + .eq("id", documentId) + .eq("project_id", projectId) + .select("*") + .single(); + if (error || !updated) return { ok: false, kind: "doc_not_found" }; + + if (doc.current_version_id) { + await db + .from("document_versions") + .update({ filename }) + .eq("id", doc.current_version_id) + .eq("document_id", documentId); + } + + return { + ok: true, + doc: { + ...updated, + filename, + }, + }; +} + +export async function ensureProjectUploadAccess( + db: Db, + args: { projectId: string; userId: string; userEmail?: string }, +): Promise<{ ok: true } | { ok: false }> { + const { projectId, userId, userEmail } = args; + const access = await checkProjectAccess(projectId, userId, userEmail, db); + return access.ok ? { ok: true } : { ok: false }; +} + +export type UploadDocumentResult = + | { ok: true; doc: unknown } + | { ok: false; kind: "create_failed" } + | { ok: false; kind: "processing_failed"; detail: string }; + +export async function processProjectDocumentUpload( + db: Db, + args: { + userId: string; + projectId: string | null; + filename: string; + suffix: string; + content: Buffer; + }, +): Promise { + const { userId, projectId, filename, suffix, content } = args; + + const { data: doc, error: insertErr } = await db + .from("documents") + .insert({ + project_id: projectId, + user_id: userId, + status: "processing", + }) + .select("*") + .single(); + + if (insertErr || !doc) return { ok: false, kind: "create_failed" }; + + try { + const docId = doc.id as string; + const key = storageKey(userId, docId, filename); + const contentType = contentTypeForDocumentType(suffix); + await uploadFile( + key, + content.buffer.slice( + content.byteOffset, + content.byteOffset + content.byteLength, + ) as ArrayBuffer, + contentType, + ); + + const rawBuf = content.buffer.slice( + content.byteOffset, + content.byteOffset + content.byteLength, + ) as ArrayBuffer; + const pageCount = suffix === "pdf" ? await countPdfPages(rawBuf) : null; + + // When the job queue is enabled, defer Office → PDF conversion to the + // BullMQ worker instead of blocking the upload request on LibreOffice — + // the same deferral the single-document upload path makes. + const deferConversion = + shouldConvertToPdf(suffix) && + process.env.ASYNC_DOCUMENT_CONVERSION === "true"; + + // Convert Office files → PDF for display. PDFs are their own rendition. + let pdfStoragePath: string | null = null; + if (!deferConversion && shouldConvertToPdf(suffix)) { + try { + const pdfBuf = await docxToPdf(content); + const pdfKey = convertedPdfKey(userId, docId); + await uploadFile( + pdfKey, + pdfBuf.buffer.slice( + pdfBuf.byteOffset, + pdfBuf.byteOffset + pdfBuf.byteLength, + ) as ArrayBuffer, + "application/pdf", + ); + pdfStoragePath = pdfKey; + } catch (err) { + console.error( + `[upload] Office→PDF conversion failed for ${filename}:`, + err, + ); + } + } else if (suffix === "pdf") { + pdfStoragePath = key; + } + + // Storage paths live on document_versions — create the V1 row and + // point documents.current_version_id at it. + const { data: versionRow, error: verErr } = await db + .from("document_versions") + .insert({ + document_id: docId, + storage_path: key, + pdf_storage_path: pdfStoragePath, + source: "upload", + version_number: 1, + filename, + file_type: suffix, + size_bytes: content.byteLength, + page_count: pageCount, + content_sha256: contentSha256(content), + }) + .select("id") + .single(); + if (verErr || !versionRow) { + throw new Error( + `Failed to record upload version: ${verErr?.message ?? "unknown"}`, + ); + } + + await db + .from("documents") + .update({ + current_version_id: versionRow.id, + // Deferred conversion leaves the doc "processing" until the worker + // produces the PDF and flips it to "ready". + status: deferConversion ? "processing" : "ready", + updated_at: new Date().toISOString(), + }) + .eq("id", docId); + + if (deferConversion) { + await enqueueConversion({ + documentId: docId, + versionId: versionRow.id as string, + userId, + storagePath: key, + fileType: suffix, + }); + } + + const { data: updated } = await db + .from("documents") + .select("*") + .eq("id", docId) + .single(); + const responseDoc = updated + ? { + ...updated, + filename, + storage_path: key, + pdf_storage_path: pdfStoragePath, + file_type: suffix, + size_bytes: content.byteLength, + page_count: pageCount, + active_version_number: 1, + } + : updated; + return { ok: true, doc: responseDoc }; + } catch (e) { + await db.from("documents").update({ status: "error" }).eq("id", doc.id); + return { ok: false, kind: "processing_failed", detail: String(e) }; + } +} diff --git a/backend/src/modules/projects/projects.folders.ts b/backend/src/modules/projects/projects.folders.ts new file mode 100644 index 000000000..bdd500bf8 --- /dev/null +++ b/backend/src/modules/projects/projects.folders.ts @@ -0,0 +1,199 @@ +// Project subfolder service functions: create, rename/move (with cycle +// check), recursive delete, and moving documents between folders. + +import { checkProjectAccess } from "../../lib/access"; +import { + type Db, + deleteProjectDocumentsAndVersionFiles, + loadProjectFolder, +} from "./projects.shared"; + +export type CreateFolderResult = + | { ok: true; folder: unknown } + | { ok: false; kind: "forbidden" } + | { ok: false; kind: "parent_not_found" } + | { ok: false; kind: "db_error"; detail: string }; + +export async function createProjectFolder( + db: Db, + args: { + projectId: string; + userId: string; + userEmail?: string; + name: string; + parent_folder_id?: string | null; + }, +): Promise { + const { projectId, userId, userEmail, name, parent_folder_id } = args; + + const access = await checkProjectAccess(projectId, userId, userEmail, db); + if (!access.ok) return { ok: false, kind: "forbidden" }; + + // Verify parent folder belongs to this project + if (parent_folder_id) { + const { data: parent } = await db.from("project_subfolders").select("id").eq("id", parent_folder_id).eq("project_id", projectId).single(); + if (!parent) return { ok: false, kind: "parent_not_found" }; + } + + const { data, error } = await db.from("project_subfolders").insert({ + project_id: projectId, + user_id: userId, + name: name.trim(), + parent_folder_id: parent_folder_id ?? null, + }).select("*").single(); + if (error) return { ok: false, kind: "db_error", detail: error.message }; + return { ok: true, folder: data }; +} + +export type UpdateFolderResult = + | { ok: true; folder: unknown } + | { ok: false; kind: "forbidden" } + | { ok: false; kind: "parent_not_found" } + | { ok: false; kind: "cycle" } + | { ok: false; kind: "not_found" }; + +export async function updateProjectFolder( + db: Db, + args: { + projectId: string; + folderId: string; + userId: string; + userEmail?: string; + body: { name?: string; parent_folder_id?: string | null }; + }, +): Promise { + const { projectId, folderId, userId, userEmail, body } = args; + + const access = await checkProjectAccess(projectId, userId, userEmail, db); + if (!access.ok) return { ok: false, kind: "forbidden" }; + + const updates: Record = { updated_at: new Date().toISOString() }; + if (body.name != null) updates.name = body.name.trim(); + if ("parent_folder_id" in body) { + // Cycle check: walk up the tree from the proposed parent to ensure folderId is not an ancestor + if (body.parent_folder_id) { + const parent = await loadProjectFolder(db, projectId, body.parent_folder_id); + if (!parent) return { ok: false, kind: "parent_not_found" }; + + let cur: string | null = body.parent_folder_id; + while (cur) { + if (cur === folderId) return { ok: false, kind: "cycle" }; + const p = await loadProjectFolder(db, projectId, cur); + if (!p) return { ok: false, kind: "parent_not_found" }; + cur = p?.parent_folder_id ?? null; + } + } + updates.parent_folder_id = body.parent_folder_id ?? null; + } + + const { data, error } = await db.from("project_subfolders") + .update(updates) + .eq("id", folderId).eq("project_id", projectId) + .select("*").single(); + if (error || !data) return { ok: false, kind: "not_found" }; + return { ok: true, folder: data }; +} + +export type DeleteFolderResult = + | { ok: true } + | { ok: false; kind: "forbidden" } + | { ok: false; kind: "not_found" } + | { ok: false; kind: "db_error"; detail: string }; + +export async function deleteProjectFolder( + db: Db, + args: { + projectId: string; + folderId: string; + userId: string; + userEmail?: string; + }, +): Promise { + const { projectId, folderId, userId, userEmail } = args; + + const access = await checkProjectAccess(projectId, userId, userEmail, db); + if (!access.ok) return { ok: false, kind: "forbidden" }; + if (!access.isOwner) return { ok: false, kind: "forbidden" }; + + const { data: allFolders, error: foldersError } = await db + .from("project_subfolders") + .select("id, parent_folder_id") + .eq("project_id", projectId); + if (foldersError) + return { ok: false, kind: "db_error", detail: foldersError.message }; + if (!(allFolders ?? []).some((f) => f.id === folderId)) + return { ok: false, kind: "not_found" }; + + const childrenByParent = new Map(); + for (const f of allFolders ?? []) { + const parentId = f.parent_folder_id as string | null; + if (!parentId) continue; + const children = childrenByParent.get(parentId) ?? []; + children.push(f.id as string); + childrenByParent.set(parentId, children); + } + + const folderIds = new Set(); + const stack = [folderId]; + while (stack.length > 0) { + const id = stack.pop()!; + if (folderIds.has(id)) continue; + folderIds.add(id); + stack.push(...(childrenByParent.get(id) ?? [])); + } + + const { data: docs, error: docsError } = await db + .from("documents") + .select("id") + .eq("project_id", projectId) + .in("folder_id", [...folderIds]); + if (docsError) return { ok: false, kind: "db_error", detail: docsError.message }; + + const docIds = (docs ?? []).map((d) => d.id as string); + const deleteDocsError = await deleteProjectDocumentsAndVersionFiles( + db, + projectId, + docIds, + ); + if (deleteDocsError) + return { ok: false, kind: "db_error", detail: deleteDocsError.message }; + + const { error } = await db.from("project_subfolders") + .delete().eq("id", folderId).eq("project_id", projectId); + if (error) return { ok: false, kind: "db_error", detail: error.message }; + return { ok: true }; +} + +export type MoveDocumentResult = + | { ok: true; doc: unknown } + | { ok: false; kind: "forbidden" } + | { ok: false; kind: "folder_not_found" } + | { ok: false; kind: "doc_not_found" }; + +export async function moveProjectDocument( + db: Db, + args: { + projectId: string; + documentId: string; + userId: string; + userEmail?: string; + folder_id: string | null; + }, +): Promise { + const { projectId, documentId, userId, userEmail, folder_id } = args; + + const access = await checkProjectAccess(projectId, userId, userEmail, db); + if (!access.ok) return { ok: false, kind: "forbidden" }; + + if (folder_id) { + const folder = await loadProjectFolder(db, projectId, folder_id); + if (!folder) return { ok: false, kind: "folder_not_found" }; + } + + const { data, error } = await db.from("documents") + .update({ folder_id: folder_id ?? null, updated_at: new Date().toISOString() }) + .eq("id", documentId).eq("project_id", projectId) + .select("*").single(); + if (error || !data) return { ok: false, kind: "doc_not_found" }; + return { ok: true, doc: data }; +} diff --git a/backend/src/modules/projects/projects.routes.ts b/backend/src/modules/projects/projects.routes.ts new file mode 100644 index 000000000..1749a764f --- /dev/null +++ b/backend/src/modules/projects/projects.routes.ts @@ -0,0 +1,449 @@ +// HTTP layer for the projects module. Handlers parse params/query/body, call +// the service functions in projects.service.ts, and map their typed results +// onto status codes and JSON bodies. Endpoint registration order matches the +// old src/routes/projects.ts monolith. + +import { Router } from "express"; +import { requireAuth, requireMfaIfEnrolled } from "../../middleware/auth"; +import { createServerSupabase } from "../../lib/supabase"; +import { singleFileUpload } from "../../lib/upload"; +import { + ALLOWED_DOCUMENT_TYPES, + ALLOWED_DOCUMENT_TYPES_LABEL, +} from "../../lib/documentTypes"; +import { + getProjectsOverview, + createProject, + getProjectDetail, + getProjectPeople, + updateProject, + deleteProject, + exportProjectManifest, + listProjectDocuments, + assignOrCopyDocument, + renameProjectDocument, + ensureProjectUploadAccess, + processProjectDocumentUpload, + listProjectChats, + createProjectFolder, + updateProjectFolder, + deleteProjectFolder, + moveProjectDocument, +} from "./projects.service"; + +export const projectsRouter = Router(); + +// GET /projects +// Pass ?include=documents to also receive each project's documents in the +// same response. The directory pickers (useDirectoryData) previously fanned +// out one GET /projects/:id per project to obtain those documents; with N +// projects that burst — auth check plus several DB queries per request — +// could overwhelm the Supabase gateway. Batching keeps it at one request +// and a fixed number of queries regardless of project count. +projectsRouter.get("/", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const includeDocuments = req.query.include === "documents"; + const db = createServerSupabase(); + + const result = await getProjectsOverview(db, { + userId, + userEmail, + includeDocuments, + }); + if (!result.ok) return void res.status(500).json({ detail: result.detail }); + res.json(result.data); +}); + +// POST /projects +projectsRouter.post("/", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { name, cm_number, practice, shared_with } = req.body as { + name: string; + cm_number?: string; + practice?: string; + shared_with?: string[]; + }; + const db = createServerSupabase(); + + const result = await createProject(db, { + userId, + userEmail, + name, + cm_number, + practice, + shared_with, + }); + if (!result.ok) { + if (result.kind === "db_error") + return void res.status(500).json({ detail: result.detail }); + return void res.status(400).json({ detail: result.detail }); + } + res.status(201).json(result.project); +}); + +// GET /projects/:projectId +projectsRouter.get("/:projectId", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string; + const { projectId } = req.params; + const db = createServerSupabase(); + + const result = await getProjectDetail(db, { projectId, userId, userEmail }); + if (!result.ok) + return void res.status(404).json({ detail: "Project not found" }); + res.json(result.body); +}); + +// GET /projects/:projectId/people +// Resolve the owner + every shared member to {email, display_name}. Used +// by the People modal so the UI can show display names where available +// and tag the current user as "You". +projectsRouter.get("/:projectId/people", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { projectId } = req.params; + const db = createServerSupabase(); + + const result = await getProjectPeople(db, { projectId, userId, userEmail }); + if (!result.ok) + return void res.status(404).json({ detail: "Project not found" }); + res.json(result.body); +}); + +// PATCH /projects/:projectId +projectsRouter.patch("/:projectId", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { projectId } = req.params; + const db = createServerSupabase(); + + const result = await updateProject(db, { + projectId, + userId, + userEmail, + body: req.body ?? {}, + }); + if (!result.ok) { + if (result.kind === "self_share" || result.kind === "missing_user") + return void res.status(400).json({ detail: result.detail }); + return void res.status(404).json({ detail: "Project not found" }); + } + res.json(result.body); +}); + +// DELETE /projects/:projectId +projectsRouter.delete("/:projectId", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const { projectId } = req.params; + const db = createServerSupabase(); + + const result = await deleteProject(db, userId, projectId); + if (!result.ok) { + if (result.kind === "not_found") + return void res.status(404).json({ detail: "Project not found" }); + return void res.status(500).json({ detail: result.detail }); + } + res.status(204).send(); +}); + +// GET /projects/:projectId/documents +projectsRouter.get("/:projectId/documents", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { projectId } = req.params; + const db = createServerSupabase(); + + const result = await listProjectDocuments(db, { + projectId, + userId, + userEmail, + }); + if (!result.ok) + return void res.status(404).json({ detail: "Project not found" }); + res.json(result.docs); +}); + +// GET /projects/:projectId/export — tamper-evident manifest of the project's +// documents: every version with its content_sha256 plus the accept/reject +// trail, under a SHA-256 digest that is Ed25519-signed when the deployment has +// MANIFEST_SIGNING_KEY set. To check an export, recompute a downloaded file's +// SHA-256 and compare, then check the manifest's signature against the key +// served at GET /manifest-signing-key. See the README. +projectsRouter.get( + "/:projectId/export", + requireAuth, + requireMfaIfEnrolled, + async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { projectId } = req.params; + const db = createServerSupabase(); + + const result = await exportProjectManifest(db, { + projectId, + userId, + userEmail, + }); + if (!result.ok) { + if (result.kind === "forbidden") + return void res.status(404).json({ detail: "Project not found" }); + return void res + .status(500) + .json({ detail: "Failed to build project export manifest" }); + } + res.setHeader("Content-Type", "application/json; charset=utf-8"); + res.setHeader( + "Content-Disposition", + `attachment; filename="${result.filename}"`, + ); + res.json(result.data); + }, +); + +// POST /projects/:projectId/documents/:documentId — assign or copy existing doc into project +projectsRouter.post( + "/:projectId/documents/:documentId", + requireAuth, + async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { projectId, documentId } = req.params; + const db = createServerSupabase(); + + const result = await assignOrCopyDocument(db, { + projectId, + documentId, + userId, + userEmail, + }); + if (!result.ok) { + switch (result.kind) { + case "forbidden": + return void res.status(404).json({ detail: "Project not found" }); + case "doc_not_found": + return void res.status(404).json({ detail: "Document not found" }); + case "no_active_version": + return void res + .status(404) + .json({ detail: "Source document has no active version" }); + case "update_failed": + return void res + .status(500) + .json({ detail: "Failed to update document" }); + case "read_failed": + return void res + .status(500) + .json({ detail: "Failed to read source document bytes" }); + case "copy_failed": + return void res + .status(500) + .json({ detail: "Failed to copy document" }); + } + } + res.status(result.status).json(result.doc); + }, +); + +// PATCH /projects/:projectId/documents/:documentId — rename a project document +projectsRouter.patch("/:projectId/documents/:documentId", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { projectId, documentId } = req.params; + const db = createServerSupabase(); + + const result = await renameProjectDocument(db, { + projectId, + documentId, + userId, + userEmail, + filename: req.body?.filename, + }); + if (!result.ok) { + if (result.kind === "forbidden") + return void res.status(404).json({ detail: "Project not found" }); + if (result.kind === "doc_not_found") + return void res.status(404).json({ detail: "Document not found" }); + return void res.status(400).json({ detail: result.detail }); + } + res.json(result.doc); +}); + +// POST /projects/:projectId/documents +projectsRouter.post( + "/:projectId/documents", + requireAuth, + singleFileUpload("file"), + async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { projectId } = req.params; + const db = createServerSupabase(); + + const access = await ensureProjectUploadAccess(db, { + projectId, + userId, + userEmail, + }); + if (!access.ok) + return void res.status(404).json({ detail: "Project not found" }); + + const file = req.file; + if (!file) return void res.status(400).json({ detail: "file is required" }); + + const filename = file.originalname; + const suffix = filename.includes(".") + ? filename.split(".").pop()!.toLowerCase() + : ""; + if (!ALLOWED_DOCUMENT_TYPES.has(suffix)) + return void res + .status(400) + .json({ + detail: `Unsupported file type: ${suffix}. Allowed: ${ALLOWED_DOCUMENT_TYPES_LABEL}`, + }); + + const result = await processProjectDocumentUpload(db, { + userId, + projectId, + filename, + suffix, + content: file.buffer, + }); + if (!result.ok) { + if (result.kind === "create_failed") + return void res + .status(500) + .json({ detail: "Failed to create document record" }); + return void res + .status(500) + .json({ detail: `Document processing failed: ${result.detail}` }); + } + res.status(201).json(result.doc); + }, +); + +// GET /projects/:projectId/chats — every assistant chat under this project +// (any author with project access). Used by the project page's chat tab so +// it doesn't have to filter the global GET /chat list — and so collaborators +// see each other's chats inside the project even though those don't appear +// in the global list. +projectsRouter.get("/:projectId/chats", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { projectId } = req.params; + const db = createServerSupabase(); + + const result = await listProjectChats(db, { projectId, userId, userEmail }); + if (!result.ok) { + if (result.kind === "forbidden") + return void res.status(404).json({ detail: "Project not found" }); + return void res.status(500).json({ detail: result.detail }); + } + res.json(result.chats); +}); + +// ── Folder routes ───────────────────────────────────────────────────────────── + +// POST /projects/:projectId/folders +projectsRouter.post("/:projectId/folders", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { projectId } = req.params; + const { name, parent_folder_id } = req.body as { name: string; parent_folder_id?: string | null }; + if (!name?.trim()) return void res.status(400).json({ detail: "name is required" }); + + const db = createServerSupabase(); + const result = await createProjectFolder(db, { + projectId, + userId, + userEmail, + name, + parent_folder_id, + }); + if (!result.ok) { + if (result.kind === "forbidden") + return void res.status(404).json({ detail: "Project not found" }); + if (result.kind === "parent_not_found") + return void res.status(404).json({ detail: "Parent folder not found" }); + return void res.status(500).json({ detail: result.detail }); + } + res.status(201).json(result.folder); +}); + +// PATCH /projects/:projectId/folders/:folderId +projectsRouter.patch("/:projectId/folders/:folderId", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { projectId, folderId } = req.params; + const body = req.body as { name?: string; parent_folder_id?: string | null }; + const db = createServerSupabase(); + + const result = await updateProjectFolder(db, { + projectId, + folderId, + userId, + userEmail, + body, + }); + if (!result.ok) { + if (result.kind === "forbidden") + return void res.status(404).json({ detail: "Project not found" }); + if (result.kind === "parent_not_found") + return void res.status(404).json({ detail: "Parent folder not found" }); + if (result.kind === "cycle") + return void res + .status(400) + .json({ detail: "Cannot move a folder into itself or a descendant" }); + return void res.status(404).json({ detail: "Folder not found" }); + } + res.json(result.folder); +}); + +// DELETE /projects/:projectId/folders/:folderId +projectsRouter.delete("/:projectId/folders/:folderId", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { projectId, folderId } = req.params; + const db = createServerSupabase(); + + const result = await deleteProjectFolder(db, { + projectId, + folderId, + userId, + userEmail, + }); + if (!result.ok) { + if (result.kind === "forbidden") + return void res.status(404).json({ detail: "Project not found" }); + if (result.kind === "not_found") + return void res.status(404).json({ detail: "Folder not found" }); + return void res.status(500).json({ detail: result.detail }); + } + res.status(204).send(); +}); + +// PATCH /projects/:projectId/documents/:documentId/folder — move doc to a folder +projectsRouter.patch("/:projectId/documents/:documentId/folder", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { projectId, documentId } = req.params; + const { folder_id } = req.body as { folder_id: string | null }; + const db = createServerSupabase(); + + const result = await moveProjectDocument(db, { + projectId, + documentId, + userId, + userEmail, + folder_id, + }); + if (!result.ok) { + if (result.kind === "forbidden") + return void res.status(404).json({ detail: "Project not found" }); + if (result.kind === "folder_not_found") + return void res.status(404).json({ detail: "Folder not found" }); + return void res.status(404).json({ detail: "Document not found" }); + } + res.json(result.doc); +}); diff --git a/backend/src/modules/projects/projects.service.ts b/backend/src/modules/projects/projects.service.ts new file mode 100644 index 000000000..9c9540135 --- /dev/null +++ b/backend/src/modules/projects/projects.service.ts @@ -0,0 +1,59 @@ +// Business logic + data-access for the projects module. +// +// These functions are the service layer behind projects.routes.ts. They take +// an explicit Supabase client (`db`) plus request-derived primitives, perform +// the project / document / folder orchestration, and RETURN values or typed +// error results. They never touch req/res — the thin route handlers map the +// results onto HTTP status codes, headers, and response bodies. +// +// The implementation is split by concern across sibling files; this module is +// the aggregate surface the routes (and tests) import from: +// +// projects.shared.ts — shared types + helpers (Db, normalisers, …) +// projects.crud.ts — overview, create, detail, people, update, +// delete, export manifest +// projects.documents.ts — list, assign/copy, rename, upload orchestration +// projects.folders.ts — subfolders + moving documents between them +// projects.chats.ts — list a project's chats + +export { + normalizeOptionalString, + normalizeDocumentFilename, +} from "./projects.shared"; + +export { + getProjectsOverview, + createProject, + getProjectDetail, + getProjectPeople, + updateProject, + deleteProject, + exportProjectManifest, + type CreateProjectResult, + type UpdateProjectResult, + type ExportProjectResult, +} from "./projects.crud"; + +export { + listProjectDocuments, + assignOrCopyDocument, + renameProjectDocument, + ensureProjectUploadAccess, + processProjectDocumentUpload, + type AssignOrCopyResult, + type RenameDocumentResult, + type UploadDocumentResult, +} from "./projects.documents"; + +export { + createProjectFolder, + updateProjectFolder, + deleteProjectFolder, + moveProjectDocument, + type CreateFolderResult, + type UpdateFolderResult, + type DeleteFolderResult, + type MoveDocumentResult, +} from "./projects.folders"; + +export { listProjectChats } from "./projects.chats"; diff --git a/backend/src/modules/projects/projects.shared.ts b/backend/src/modules/projects/projects.shared.ts new file mode 100644 index 000000000..589f9abed --- /dev/null +++ b/backend/src/modules/projects/projects.shared.ts @@ -0,0 +1,189 @@ +// Shared types + helpers for the projects module service layer. +// +// The projects service is split by concern across sibling files +// (projects.crud.ts, projects.documents.ts, projects.folders.ts, +// projects.chats.ts). Anything used by more than one of them lives here, and +// projects.service.ts re-exports the whole surface so route/test importers see +// a single module. + +import { createServerSupabase } from "../../lib/supabase"; +import { deleteFile } from "../../lib/storage"; + +export type Db = ReturnType; + +export function normalizeOptionalString(value: unknown) { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : null; +} + +export function normalizeDocumentFilename(nextName: unknown, currentName: string) { + if (typeof nextName !== "string") return null; + const trimmed = nextName.trim().slice(0, 200); + if (!trimmed) return null; + if (/\.[a-z0-9]{1,6}$/i.test(trimmed)) return trimmed; + const ext = currentName.match(/\.[a-z0-9]{1,6}$/i)?.[0] ?? ""; + return `${trimmed}${ext}`; +} + +// Normalise a `shared_with` email list: lowercase + dedupe + drop empties. +// Returns `{ ok: false, self: true }` when the caller's own email appears in +// the list — POST /projects and PATCH /projects/:projectId previously carried +// identical copies of this loop inline and both surface that case as a 400. +export function normalizeSharedWith( + raw: unknown, + normalizedUserEmail: string | undefined, +): { ok: true; cleaned: string[] } | { ok: false; self: true } { + const cleaned: string[] = []; + const seen = new Set(); + if (Array.isArray(raw)) { + for (const value of raw) { + if (typeof value !== "string") continue; + const e = value.trim().toLowerCase(); + if (!e || seen.has(e)) continue; + if (normalizedUserEmail && e === normalizedUserEmail) { + return { ok: false, self: true }; + } + seen.add(e); + cleaned.push(e); + } + } + return { ok: true, cleaned }; +} + +export async function deleteProjectDocumentsAndVersionFiles( + db: Db, + projectId: string, + documentIds: string[], +) { + if (documentIds.length === 0) return null; + const { data: versions, error: versionsError } = await db + .from("document_versions") + .select("storage_path, pdf_storage_path") + .in("document_id", documentIds); + if (versionsError) return versionsError; + + const paths = new Set(); + for (const v of versions ?? []) { + if (typeof v.storage_path === "string" && v.storage_path.length > 0) { + paths.add(v.storage_path); + } + if (typeof v.pdf_storage_path === "string" && v.pdf_storage_path.length > 0) { + paths.add(v.pdf_storage_path); + } + } + await Promise.all([...paths].map((p) => deleteFile(p).catch(() => {}))); + + const { error } = await db + .from("documents") + .delete() + .eq("project_id", projectId) + .in("id", documentIds); + return error ?? null; +} + +export async function attachDocumentOwnerLabels( + db: Db, + docs: { user_id?: string | null }[], +) { + const ownerIds = docs + .map((doc) => doc.user_id) + .filter((id): id is string => typeof id === "string" && id.length > 0) + .filter((id, index, arr) => arr.indexOf(id) === index); + if (ownerIds.length === 0) return; + + const displayNameByUserId = new Map(); + const { data: profiles, error: profilesError } = await db + .from("user_profiles") + .select("user_id, display_name") + .in("user_id", ownerIds); + if (profilesError) { + console.warn("[projects] failed to load document owner profiles", profilesError); + } + for (const profile of profiles ?? []) { + const displayName = + typeof profile.display_name === "string" + ? profile.display_name.trim() + : ""; + if (displayName) { + displayNameByUserId.set(profile.user_id as string, displayName); + } + } + + for (const doc of docs as ({ + user_id?: string | null; + owner_email?: string | null; + owner_display_name?: string | null; + })[]) { + if (!doc.user_id) continue; + doc.owner_email = null; + doc.owner_display_name = displayNameByUserId.get(doc.user_id) ?? null; + } +} + +export async function attachChatCreatorLabels( + db: Db, + chats: { user_id?: string | null }[], +) { + const creatorIds = chats + .map((chat) => chat.user_id) + .filter((id): id is string => typeof id === "string" && id.length > 0) + .filter((id, index, arr) => arr.indexOf(id) === index); + if (creatorIds.length === 0) return; + + const displayNameByUserId = new Map(); + const { data: profiles, error: profilesError } = await db + .from("user_profiles") + .select("user_id, display_name") + .in("user_id", creatorIds); + if (profilesError) { + console.warn("[projects] failed to load chat creator profiles", profilesError); + } + for (const profile of profiles ?? []) { + const displayName = + typeof profile.display_name === "string" + ? profile.display_name.trim() + : ""; + if (displayName) { + displayNameByUserId.set(profile.user_id as string, displayName); + } + } + + for (const chat of chats as ({ + user_id?: string | null; + creator_display_name?: string | null; + })[]) { + if (!chat.user_id) continue; + chat.creator_display_name = displayNameByUserId.get(chat.user_id) ?? null; + } +} + +export async function loadProjectFolder( + db: Db, + projectId: string, + folderId: string, +): Promise<{ id: string; parent_folder_id: string | null } | null> { + const { data } = await db + .from("project_subfolders") + .select("id, parent_folder_id") + .eq("id", folderId) + .eq("project_id", projectId) + .maybeSingle(); + return (data as { id: string; parent_folder_id: string | null } | null) ?? null; +} + +export async function countPdfPages(buf: ArrayBuffer): Promise { + try { + const pdfjsLib = await import("pdfjs-dist/legacy/build/pdf.mjs" as string); + const pdf = await ( + pdfjsLib as unknown as { + getDocument: (opts: unknown) => { + promise: Promise<{ numPages: number }>; + }; + } + ).getDocument({ data: new Uint8Array(buf) }).promise; + return pdf.numPages; + } catch { + return null; + } +} diff --git a/backend/src/lib/tabular/__tests__/tabular.extractRow.test.ts b/backend/src/modules/tabular/__tests__/tabular.extractRow.test.ts similarity index 100% rename from backend/src/lib/tabular/__tests__/tabular.extractRow.test.ts rename to backend/src/modules/tabular/__tests__/tabular.extractRow.test.ts diff --git a/backend/src/lib/tabular/__tests__/tabular.generateStream.test.ts b/backend/src/modules/tabular/__tests__/tabular.generateStream.test.ts similarity index 100% rename from backend/src/lib/tabular/__tests__/tabular.generateStream.test.ts rename to backend/src/modules/tabular/__tests__/tabular.generateStream.test.ts diff --git a/backend/src/modules/tabular/tabular.chats.ts b/backend/src/modules/tabular/tabular.chats.ts new file mode 100644 index 000000000..53a3865aa --- /dev/null +++ b/backend/src/modules/tabular/tabular.chats.ts @@ -0,0 +1,104 @@ +// Chat support for the tabular module: parsing the model's block +// into typed annotations, and building the system + history messages the +// agentic review chat streams over. Moved out of routes/tabular.ts; bodies +// unchanged. The streaming loop itself stays in tabular.routes.ts. + +import { + type ChatMessage, + type TabularCellStore, +} from "../../lib/chat"; + +// --------------------------------------------------------------------------- +// Tabular citation parsing +// --------------------------------------------------------------------------- + +export type TabularParsedCitation = { + ref: number; + col_index: number; + row_index: number; + quote: string; +}; + +const TABULAR_CITATIONS_BLOCK_RE = /\s*([\s\S]*?)\s*<\/CITATIONS>/; + +export function parseTabularCitations(text: string): TabularParsedCitation[] { + const match = text.match(TABULAR_CITATIONS_BLOCK_RE); + if (!match) return []; + try { + return JSON.parse(match[1]) as TabularParsedCitation[]; + } catch { + return []; + } +} + +export function extractTabularAnnotations( + fullText: string, + tabularStore: TabularCellStore, +) { + return parseTabularCitations(fullText).map((c) => ({ + type: "tabular_citation" as const, + ref: c.ref, + col_index: c.col_index, + row_index: c.row_index, + col_name: + tabularStore.columns[c.col_index]?.name ?? `Col ${c.col_index}`, + doc_name: + tabularStore.documents[c.row_index]?.filename ?? + `Row ${c.row_index}`, + quote: c.quote, + })); +} + +// --------------------------------------------------------------------------- +// Build messages for tabular chat +// --------------------------------------------------------------------------- + +export function buildTabularMessages( + messages: ChatMessage[], + tabularStore: TabularCellStore, + reviewTitle: string, +): unknown[] { + const docList = tabularStore.documents + .map((d, i) => `- ROW:${i} "${d.filename}"`) + .join("\n"); + const colList = tabularStore.columns + .map((c, i) => `- COL:${i} "${c.name}"`) + .join("\n"); + + const systemContent = `You are Mike, an AI legal assistant. You are helping with the tabular review titled "${reviewTitle}". + +The review extracts specific fields from multiple legal documents into a structured table. +You do NOT have the cell content yet — call read_table_cells to fetch the cells you need before answering. + +DOCUMENTS (rows): +${docList || "- (none)"} + +COLUMNS (fields): +${colList || "- (none)"} + +TABULAR CITATION INSTRUCTIONS: +When you reference specific cell content, place a numbered marker [1], [2], etc. inline in your prose at the point of reference. + +After your complete response, append a block containing a JSON array with one entry per marker: + + +[ + {"ref": 1, "col_index": 0, "row_index": 2, "quote": "verbatim text from the cell"}, + {"ref": 2, "col_index": 1, "row_index": 0, "quote": "another excerpt"} +] + + +Rules: +- col_index and row_index are 0-based (matching the COL/ROW numbers listed above) +- Only cite cells you have read via read_table_cells +- quote should be verbatim text from the cell's summary +- Omit if you make no citations +- Do not fabricate cell content +- Answer in clear, concise prose. You may use markdown formatting.`; + + const formatted: unknown[] = [{ role: "system", content: systemContent }]; + for (const msg of messages) { + formatted.push({ role: msg.role, content: msg.content ?? "" }); + } + return formatted; +} diff --git a/backend/src/lib/tabular/tabular.extract.ts b/backend/src/modules/tabular/tabular.extract.ts similarity index 97% rename from backend/src/lib/tabular/tabular.extract.ts rename to backend/src/modules/tabular/tabular.extract.ts index a36a40292..a9ca24316 100644 --- a/backend/src/lib/tabular/tabular.extract.ts +++ b/backend/src/modules/tabular/tabular.extract.ts @@ -1,21 +1,21 @@ // Extraction for the tabular-review module: the LLM cell-extraction helpers // and document (PDF/DOCX/Office) text extraction. -import { docxToPdf, normalizeDocxZipPaths } from "../convert"; +import { docxToPdf, normalizeDocxZipPaths } from "../../lib/convert"; import { isPresentationDocumentType, isSpreadsheetDocumentType, isWordDocumentType, -} from "../documentTypes"; -import { extractPresentationText } from "../officeText"; -import { spreadsheetToLLMText } from "../spreadsheet"; +} from "../../lib/documentTypes"; +import { extractPresentationText } from "../../lib/officeText"; +import { spreadsheetToLLMText } from "../../lib/spreadsheet"; import { completeText, streamChatWithTools, type UserApiKeys, -} from "../llm"; -import { safeErrorLog } from "../safeError"; -import { loadPdfjs } from "../pdfjs"; +} from "../../lib/llm"; +import { safeErrorLog } from "../../lib/safeError"; +import { loadPdfjs } from "../../lib/pdfjs"; import { formatPromptSuffix } from "./tabular.prompt"; import { type CellResult, type Column } from "./tabular.shared"; diff --git a/backend/src/lib/tabular/tabular.extractRow.ts b/backend/src/modules/tabular/tabular.extractRow.ts similarity index 98% rename from backend/src/lib/tabular/tabular.extractRow.ts rename to backend/src/modules/tabular/tabular.extractRow.ts index 3a8488b86..0a645aeda 100644 --- a/backend/src/lib/tabular/tabular.extractRow.ts +++ b/backend/src/modules/tabular/tabular.extractRow.ts @@ -13,7 +13,7 @@ // terminal policy for columns the model failed to return — it reports them via // `missing` and lets each caller apply its own policy. -import { type UserApiKeys } from "../llm"; +import { type UserApiKeys } from "../../lib/llm"; import { queryTabularAllColumns } from "./tabular.extract"; import { loadRowDocumentText, type ReviewRow } from "./tabular.rows"; import { type CellResult, type Column, type Db } from "./tabular.shared"; diff --git a/backend/src/lib/tabular/tabular.generate.ts b/backend/src/modules/tabular/tabular.generate.ts similarity index 96% rename from backend/src/lib/tabular/tabular.generate.ts rename to backend/src/modules/tabular/tabular.generate.ts index ed1a11ab7..7f672e46a 100644 --- a/backend/src/lib/tabular/tabular.generate.ts +++ b/backend/src/modules/tabular/tabular.generate.ts @@ -6,12 +6,12 @@ // checks, row loading, missing-API-key checks) that returns the data the // route then streams over. -import { type UserApiKeys } from "../llm"; -import { getUserModelSettings } from "../userSettings"; +import { type UserApiKeys } from "../../lib/llm"; +import { getUserModelSettings } from "../../lib/userSettings"; import { ensureReviewAccess, filterAccessibleDocumentIds, -} from "../access"; +} from "../../lib/access"; import { loadReviewRows, type ReviewRow } from "./tabular.rows"; import { missingModelApiKey, diff --git a/backend/src/lib/tabular/tabular.generateStream.ts b/backend/src/modules/tabular/tabular.generateStream.ts similarity index 98% rename from backend/src/lib/tabular/tabular.generateStream.ts rename to backend/src/modules/tabular/tabular.generateStream.ts index e030e1d58..baa574dcc 100644 --- a/backend/src/lib/tabular/tabular.generateStream.ts +++ b/backend/src/modules/tabular/tabular.generateStream.ts @@ -16,14 +16,14 @@ import IORedis from "ioredis"; import type { Response } from "express"; -import { REDIS_URL } from "../queue/connection"; -import { startSseHeartbeat } from "../sseHeartbeat"; -import { enqueueExtraction } from "../queue/extractionQueue"; +import { REDIS_URL } from "../../lib/queue/connection"; +import { startSseHeartbeat } from "../../lib/sseHeartbeat"; +import { enqueueExtraction } from "../../lib/queue/extractionQueue"; import { runProgressChannel, type CellUpdate, -} from "../queue/runProgress"; -import { safeErrorLog } from "../safeError"; +} from "../../lib/queue/runProgress"; +import { safeErrorLog } from "../../lib/safeError"; import { parseCellContent, type Column, type Db, type Log } from "./tabular.shared"; import type { PreparedGenerate } from "./tabular.generate"; diff --git a/backend/src/lib/tabular/tabular.prompt.ts b/backend/src/modules/tabular/tabular.prompt.ts similarity index 100% rename from backend/src/lib/tabular/tabular.prompt.ts rename to backend/src/modules/tabular/tabular.prompt.ts diff --git a/backend/src/modules/tabular/tabular.reviews.ts b/backend/src/modules/tabular/tabular.reviews.ts new file mode 100644 index 000000000..2fd9aa9ff --- /dev/null +++ b/backend/src/modules/tabular/tabular.reviews.ts @@ -0,0 +1,284 @@ +// Review-lifecycle services for the tabular module: building a review's rows +// from its selected documents (grouped per document or per folder), rebuilding +// them when the selection changes, and reconciling the cell grid to the active +// column set. Moved out of routes/tabular.ts; bodies unchanged. + +import { + fetchSourceDocuments, + type ReviewRow, + type SourceDocument, +} from "./tabular.rows"; +import { type Column, type Db } from "./tabular.shared"; + +export type DocumentGrouping = "document" | "folder"; + +export function normalizeGrouping(value: unknown): DocumentGrouping { + return value === "folder" ? "folder" : "document"; +} + +function buildFolderPathMap( + folders: { + id: string; + name: string; + parent_folder_id: string | null; + }[], +): Map { + const byId = new Map(folders.map((folder) => [folder.id, folder])); + const paths = new Map(); + const resolve = (id: string): string => { + const existing = paths.get(id); + if (existing) return existing; + const folder = byId.get(id); + if (!folder) return "Unknown folder"; + const path = folder.parent_folder_id + ? `${resolve(folder.parent_folder_id)} / ${folder.name}` + : folder.name; + paths.set(id, path); + return path; + }; + for (const folder of folders) resolve(folder.id); + return paths; +} + +async function getFolderPathMaps( + db: Db, + userId: string, + docs: SourceDocument[], +): Promise<{ + project: Map; + library: Map; +}> { + const projectIds = [ + ...new Set( + docs + .map((doc) => doc.project_id) + .filter((id): id is string => !!id), + ), + ]; + const [projectResult, libraryResult] = await Promise.all([ + projectIds.length + ? db + .from("project_subfolders") + .select("id, name, parent_folder_id") + .in("project_id", projectIds) + : Promise.resolve({ data: [] }), + db + .from("library_folders") + .select("id, name, parent_folder_id") + .eq("user_id", userId), + ]); + return { + project: buildFolderPathMap(projectResult.data ?? []), + library: buildFolderPathMap(libraryResult.data ?? []), + }; +} + +export async function createRowsForReview( + db: Db, + reviewId: string, + userId: string, + documentIds: string[], + columns: Column[], + grouping: DocumentGrouping, +): Promise { + const docs = await fetchSourceDocuments(db, documentIds); + const folderPaths = await getFolderPathMaps(db, userId, docs); + const inputs: { + label: string; + row_type: "document" | "folder"; + folder_id: string | null; + library_folder_id: string | null; + document_id: string | null; + sourceIds: string[]; + }[] = []; + + if (grouping === "folder") { + const byFolder = new Map< + string, + { + folder_id: string | null; + library_folder_id: string | null; + docs: SourceDocument[]; + } + >(); + for (const doc of docs) { + const folderKey = doc.folder_id + ? `project:${doc.folder_id}` + : doc.library_folder_id + ? `library:${doc.library_folder_id}` + : null; + if (!folderKey) { + inputs.push({ + label: doc.filename, + row_type: "document", + folder_id: null, + library_folder_id: null, + document_id: doc.id, + sourceIds: [doc.id], + }); + continue; + } + const existing = byFolder.get(folderKey); + if (existing) { + existing.docs.push(doc); + } else { + byFolder.set(folderKey, { + folder_id: doc.folder_id ?? null, + library_folder_id: doc.library_folder_id ?? null, + docs: [doc], + }); + } + } + for (const folder of byFolder.values()) { + const label = folder.folder_id + ? folderPaths.project.get(folder.folder_id) + : folder.library_folder_id + ? folderPaths.library.get(folder.library_folder_id) + : null; + inputs.push({ + label: label ?? "Unknown folder", + row_type: "folder", + folder_id: folder.folder_id, + library_folder_id: folder.library_folder_id, + document_id: null, + sourceIds: folder.docs.map((doc) => doc.id), + }); + } + } else { + for (const doc of docs) { + inputs.push({ + label: doc.filename, + row_type: "document", + folder_id: null, + library_folder_id: null, + document_id: doc.id, + sourceIds: [doc.id], + }); + } + } + + inputs.sort((a, b) => a.label.localeCompare(b.label)); + if (inputs.length === 0) return; + + const { data, error } = await db + .from("tabular_review_rows") + .insert( + inputs.map((input, sort_index) => ({ + review_id: reviewId, + label: input.label, + row_type: input.row_type, + folder_id: input.folder_id, + library_folder_id: input.library_folder_id, + document_id: input.document_id, + sort_index, + })), + ) + .select("*"); + if (error) throw new Error(error.message); + const rows = ((data ?? []) as ReviewRow[]).sort( + (a, b) => a.sort_index - b.sort_index, + ); + const sources = rows.flatMap((row) => + (inputs[row.sort_index]?.sourceIds ?? []).map( + (document_id, sort_index) => ({ + row_id: row.id, + document_id, + sort_index, + }), + ), + ); + if (sources.length) { + const { error: sourceError } = await db + .from("tabular_review_row_sources") + .insert(sources); + if (sourceError) throw new Error(sourceError.message); + } + const cells = rows.flatMap((row) => + columns.map((column) => ({ + review_id: reviewId, + row_id: row.id, + document_id: row.document_id, + column_index: column.index, + status: "pending", + })), + ); + if (cells.length) { + const { error: cellError } = await db + .from("tabular_cells") + .insert(cells); + if (cellError) throw new Error(cellError.message); + } +} + +export async function rebuildRowsForReview( + db: Db, + reviewId: string, + userId: string, + documentIds: string[], + columns: Column[], + grouping: DocumentGrouping, +): Promise { + const { error } = await db + .from("tabular_review_rows") + .delete() + .eq("review_id", reviewId); + if (error) throw new Error(error.message); + await createRowsForReview( + db, + reviewId, + userId, + documentIds, + columns, + grouping, + ); +} + +export async function syncCellsForReviewRows( + db: Db, + reviewId: string, + columns: Column[], +): Promise { + const { data: rows, error: rowsError } = await db + .from("tabular_review_rows") + .select("id,document_id") + .eq("review_id", reviewId); + if (rowsError) throw new Error(rowsError.message); + const { data: cells, error: cellsError } = await db + .from("tabular_cells") + .select("id,row_id,column_index") + .eq("review_id", reviewId); + if (cellsError) throw new Error(cellsError.message); + + const activeColumnIndexes = new Set(columns.map((column) => column.index)); + const staleCellIds = (cells ?? []) + .filter((cell) => !activeColumnIndexes.has(cell.column_index)) + .map((cell) => cell.id); + if (staleCellIds.length) { + const { error } = await db + .from("tabular_cells") + .delete() + .in("id", staleCellIds); + if (error) throw new Error(error.message); + } + + const existingKeys = new Set( + (cells ?? []) + .filter((cell) => activeColumnIndexes.has(cell.column_index)) + .map((cell) => `${cell.row_id}:${cell.column_index}`), + ); + const missingCells = (rows ?? []).flatMap((row) => + columns + .filter((column) => !existingKeys.has(`${row.id}:${column.index}`)) + .map((column) => ({ + review_id: reviewId, + row_id: row.id, + document_id: row.document_id, + column_index: column.index, + status: "pending", + })), + ); + if (missingCells.length) { + const { error } = await db.from("tabular_cells").insert(missingCells); + if (error) throw new Error(error.message); + } +} diff --git a/backend/src/routes/tabular.ts b/backend/src/modules/tabular/tabular.routes.ts similarity index 78% rename from backend/src/routes/tabular.ts rename to backend/src/modules/tabular/tabular.routes.ts index 8dcf360e5..5a3954741 100644 --- a/backend/src/routes/tabular.ts +++ b/backend/src/modules/tabular/tabular.routes.ts @@ -1,7 +1,11 @@ +// HTTP layer of the tabular-review module. Handlers parse and validate the +// request, delegate to the module's service files, and map typed results onto +// status codes. Streaming endpoints (generate, chat) keep their SSE loops here; +// their non-streaming prepare/persist logic lives in the service files. + import { Router } from "express"; -import { requireAuth } from "../middleware/auth"; -import { createServerSupabase } from "../lib/supabase"; -import { attachActiveVersionPaths } from "../lib/documentVersions"; +import { requireAuth } from "../../middleware/auth"; +import { createServerSupabase } from "../../lib/supabase"; import { AssistantStreamError, buildCancelledAssistantMessage, @@ -11,329 +15,61 @@ import { TABULAR_TOOLS, type ChatMessage, type TabularCellStore, -} from "../lib/chat"; -import { completeText } from "../lib/llm"; +} from "../../lib/chat"; +import { completeText } from "../../lib/llm"; import { extractDocumentMarkdown, generateChatTitle, queryTabularCell, -} from "../lib/tabular/tabular.extract"; +} from "./tabular.extract"; import { missingModelApiKey, parseCellContent, type Column, -} from "../lib/tabular/tabular.shared"; -import { extractRowColumns } from "../lib/tabular/tabular.extractRow"; -import { prepareTabularGenerate } from "../lib/tabular/tabular.generate"; +} from "./tabular.shared"; +import { extractRowColumns } from "./tabular.extractRow"; +import { prepareTabularGenerate } from "./tabular.generate"; import { awaitCellTerminal, streamTabularGenerateAsync, streamTabularRunView, -} from "../lib/tabular/tabular.generateStream"; -import { enqueueExtraction } from "../lib/queue/extractionQueue"; +} from "./tabular.generateStream"; +import { enqueueExtraction } from "../../lib/queue/extractionQueue"; +import { loadReviewRows, loadRowDocumentText } from "./tabular.rows"; +import { + createRowsForReview, + normalizeGrouping, + rebuildRowsForReview, + syncCellsForReviewRows, + type DocumentGrouping, +} from "./tabular.reviews"; import { - fetchSourceDocuments, - loadReviewRows, - loadRowDocumentText, - type ReviewRow, - type SourceDocument, -} from "../lib/tabular/tabular.rows"; -import { getUserModelSettings } from "../lib/userSettings"; + buildTabularMessages, + extractTabularAnnotations, +} from "./tabular.chats"; +import { getUserModelSettings } from "../../lib/userSettings"; import { checkProjectAccess, ensureReviewAccess, filterAccessibleDocumentIds, -} from "../lib/access"; -import { safeErrorLog, safeErrorMessage } from "../lib/safeError"; +} from "../../lib/access"; +import { safeErrorLog, safeErrorMessage } from "../../lib/safeError"; import { findMissingUserEmails, loadProfileUsersByEmail, -} from "../lib/userLookup"; -import { parsePaginationQuery } from "../lib/pagination"; -import { normalizeSearchTerm } from "../lib/search"; -import { parseTabularReviewSort } from "../lib/sort"; +} from "../../lib/userLookup"; +import { parsePaginationQuery } from "../../lib/pagination"; +import { normalizeSearchTerm } from "../../lib/search"; +import { parseTabularReviewSort } from "../../lib/sort"; import { buildTabularReviewIdsOverviewRpcArgs, buildTabularReviewsOverviewRpcArgs, parseTabularReviewScope, -} from "../lib/tabularReviewsOverview"; +} from "../../lib/tabularReviewsOverview"; +import { attachActiveVersionPaths } from "../../lib/documentVersions"; export const tabularRouter = Router(); -type DocumentGrouping = "document" | "folder"; -type SupabaseDb = ReturnType; - -function normalizeGrouping(value: unknown): DocumentGrouping { - return value === "folder" ? "folder" : "document"; -} - -function buildFolderPathMap( - folders: { - id: string; - name: string; - parent_folder_id: string | null; - }[], -): Map { - const byId = new Map(folders.map((folder) => [folder.id, folder])); - const paths = new Map(); - const resolve = (id: string): string => { - const existing = paths.get(id); - if (existing) return existing; - const folder = byId.get(id); - if (!folder) return "Unknown folder"; - const path = folder.parent_folder_id - ? `${resolve(folder.parent_folder_id)} / ${folder.name}` - : folder.name; - paths.set(id, path); - return path; - }; - for (const folder of folders) resolve(folder.id); - return paths; -} - -async function getFolderPathMaps( - db: SupabaseDb, - userId: string, - docs: SourceDocument[], -): Promise<{ - project: Map; - library: Map; -}> { - const projectIds = [ - ...new Set( - docs - .map((doc) => doc.project_id) - .filter((id): id is string => !!id), - ), - ]; - const [projectResult, libraryResult] = await Promise.all([ - projectIds.length - ? db - .from("project_subfolders") - .select("id, name, parent_folder_id") - .in("project_id", projectIds) - : Promise.resolve({ data: [] }), - db - .from("library_folders") - .select("id, name, parent_folder_id") - .eq("user_id", userId), - ]); - return { - project: buildFolderPathMap(projectResult.data ?? []), - library: buildFolderPathMap(libraryResult.data ?? []), - }; -} - -async function createRowsForReview( - db: SupabaseDb, - reviewId: string, - userId: string, - documentIds: string[], - columns: Column[], - grouping: DocumentGrouping, -): Promise { - const docs = await fetchSourceDocuments(db, documentIds); - const folderPaths = await getFolderPathMaps(db, userId, docs); - const inputs: { - label: string; - row_type: "document" | "folder"; - folder_id: string | null; - library_folder_id: string | null; - document_id: string | null; - sourceIds: string[]; - }[] = []; - - if (grouping === "folder") { - const byFolder = new Map< - string, - { - folder_id: string | null; - library_folder_id: string | null; - docs: SourceDocument[]; - } - >(); - for (const doc of docs) { - const folderKey = doc.folder_id - ? `project:${doc.folder_id}` - : doc.library_folder_id - ? `library:${doc.library_folder_id}` - : null; - if (!folderKey) { - inputs.push({ - label: doc.filename, - row_type: "document", - folder_id: null, - library_folder_id: null, - document_id: doc.id, - sourceIds: [doc.id], - }); - continue; - } - const existing = byFolder.get(folderKey); - if (existing) { - existing.docs.push(doc); - } else { - byFolder.set(folderKey, { - folder_id: doc.folder_id ?? null, - library_folder_id: doc.library_folder_id ?? null, - docs: [doc], - }); - } - } - for (const folder of byFolder.values()) { - const label = folder.folder_id - ? folderPaths.project.get(folder.folder_id) - : folder.library_folder_id - ? folderPaths.library.get(folder.library_folder_id) - : null; - inputs.push({ - label: label ?? "Unknown folder", - row_type: "folder", - folder_id: folder.folder_id, - library_folder_id: folder.library_folder_id, - document_id: null, - sourceIds: folder.docs.map((doc) => doc.id), - }); - } - } else { - for (const doc of docs) { - inputs.push({ - label: doc.filename, - row_type: "document", - folder_id: null, - library_folder_id: null, - document_id: doc.id, - sourceIds: [doc.id], - }); - } - } - - inputs.sort((a, b) => a.label.localeCompare(b.label)); - if (inputs.length === 0) return; - - const { data, error } = await db - .from("tabular_review_rows") - .insert( - inputs.map((input, sort_index) => ({ - review_id: reviewId, - label: input.label, - row_type: input.row_type, - folder_id: input.folder_id, - library_folder_id: input.library_folder_id, - document_id: input.document_id, - sort_index, - })), - ) - .select("*"); - if (error) throw new Error(error.message); - const rows = ((data ?? []) as ReviewRow[]).sort( - (a, b) => a.sort_index - b.sort_index, - ); - const sources = rows.flatMap((row) => - (inputs[row.sort_index]?.sourceIds ?? []).map( - (document_id, sort_index) => ({ - row_id: row.id, - document_id, - sort_index, - }), - ), - ); - if (sources.length) { - const { error: sourceError } = await db - .from("tabular_review_row_sources") - .insert(sources); - if (sourceError) throw new Error(sourceError.message); - } - const cells = rows.flatMap((row) => - columns.map((column) => ({ - review_id: reviewId, - row_id: row.id, - document_id: row.document_id, - column_index: column.index, - status: "pending", - })), - ); - if (cells.length) { - const { error: cellError } = await db - .from("tabular_cells") - .insert(cells); - if (cellError) throw new Error(cellError.message); - } -} - -async function rebuildRowsForReview( - db: SupabaseDb, - reviewId: string, - userId: string, - documentIds: string[], - columns: Column[], - grouping: DocumentGrouping, -): Promise { - const { error } = await db - .from("tabular_review_rows") - .delete() - .eq("review_id", reviewId); - if (error) throw new Error(error.message); - await createRowsForReview( - db, - reviewId, - userId, - documentIds, - columns, - grouping, - ); -} - -async function syncCellsForReviewRows( - db: SupabaseDb, - reviewId: string, - columns: Column[], -): Promise { - const { data: rows, error: rowsError } = await db - .from("tabular_review_rows") - .select("id,document_id") - .eq("review_id", reviewId); - if (rowsError) throw new Error(rowsError.message); - const { data: cells, error: cellsError } = await db - .from("tabular_cells") - .select("id,row_id,column_index") - .eq("review_id", reviewId); - if (cellsError) throw new Error(cellsError.message); - - const activeColumnIndexes = new Set(columns.map((column) => column.index)); - const staleCellIds = (cells ?? []) - .filter((cell) => !activeColumnIndexes.has(cell.column_index)) - .map((cell) => cell.id); - if (staleCellIds.length) { - const { error } = await db - .from("tabular_cells") - .delete() - .in("id", staleCellIds); - if (error) throw new Error(error.message); - } - - const existingKeys = new Set( - (cells ?? []) - .filter((cell) => activeColumnIndexes.has(cell.column_index)) - .map((cell) => `${cell.row_id}:${cell.column_index}`), - ); - const missingCells = (rows ?? []).flatMap((row) => - columns - .filter((column) => !existingKeys.has(`${row.id}:${column.index}`)) - .map((column) => ({ - review_id: reviewId, - row_id: row.id, - document_id: row.document_id, - column_index: column.index, - status: "pending", - })), - ); - if (missingCells.length) { - const { error } = await db.from("tabular_cells").insert(missingCells); - if (error) throw new Error(error.message); - } -} - // GET /tabular-review tabularRouter.get("/", requireAuth, async (req, res) => { const userId = res.locals.userId as string; @@ -1330,101 +1066,6 @@ tabularRouter.get( }, ); -// --------------------------------------------------------------------------- -// Tabular citation parsing -// --------------------------------------------------------------------------- - -type TabularParsedCitation = { - ref: number; - col_index: number; - row_index: number; - quote: string; -}; - -const TABULAR_CITATIONS_BLOCK_RE = /\s*([\s\S]*?)\s*<\/CITATIONS>/; - -function parseTabularCitations(text: string): TabularParsedCitation[] { - const match = text.match(TABULAR_CITATIONS_BLOCK_RE); - if (!match) return []; - try { - return JSON.parse(match[1]) as TabularParsedCitation[]; - } catch { - return []; - } -} - -function extractTabularAnnotations( - fullText: string, - tabularStore: TabularCellStore, -) { - return parseTabularCitations(fullText).map((c) => ({ - type: "tabular_citation" as const, - ref: c.ref, - col_index: c.col_index, - row_index: c.row_index, - col_name: - tabularStore.columns[c.col_index]?.name ?? `Col ${c.col_index}`, - doc_name: - tabularStore.documents[c.row_index]?.filename ?? - `Row ${c.row_index}`, - quote: c.quote, - })); -} - -// --------------------------------------------------------------------------- -// Build messages for tabular chat -// --------------------------------------------------------------------------- - -function buildTabularMessages( - messages: ChatMessage[], - tabularStore: TabularCellStore, - reviewTitle: string, -): unknown[] { - const docList = tabularStore.documents - .map((d, i) => `- ROW:${i} "${d.filename}"`) - .join("\n"); - const colList = tabularStore.columns - .map((c, i) => `- COL:${i} "${c.name}"`) - .join("\n"); - - const systemContent = `You are Mike, an AI legal assistant. You are helping with the tabular review titled "${reviewTitle}". - -The review extracts specific fields from multiple legal documents into a structured table. -You do NOT have the cell content yet — call read_table_cells to fetch the cells you need before answering. - -DOCUMENTS (rows): -${docList || "- (none)"} - -COLUMNS (fields): -${colList || "- (none)"} - -TABULAR CITATION INSTRUCTIONS: -When you reference specific cell content, place a numbered marker [1], [2], etc. inline in your prose at the point of reference. - -After your complete response, append a block containing a JSON array with one entry per marker: - - -[ - {"ref": 1, "col_index": 0, "row_index": 2, "quote": "verbatim text from the cell"}, - {"ref": 2, "col_index": 1, "row_index": 0, "quote": "another excerpt"} -] - - -Rules: -- col_index and row_index are 0-based (matching the COL/ROW numbers listed above) -- Only cite cells you have read via read_table_cells -- quote should be verbatim text from the cell's summary -- Omit if you make no citations -- Do not fabricate cell content -- Answer in clear, concise prose. You may use markdown formatting.`; - - const formatted: unknown[] = [{ role: "system", content: systemContent }]; - for (const msg of messages) { - formatted.push({ role: msg.role, content: msg.content ?? "" }); - } - return formatted; -} - // --------------------------------------------------------------------------- // POST /tabular-review/:reviewId/chat — agentic streaming // --------------------------------------------------------------------------- diff --git a/backend/src/lib/tabular/tabular.rows.ts b/backend/src/modules/tabular/tabular.rows.ts similarity index 96% rename from backend/src/lib/tabular/tabular.rows.ts rename to backend/src/modules/tabular/tabular.rows.ts index 608a69c20..c13bca2a6 100644 --- a/backend/src/lib/tabular/tabular.rows.ts +++ b/backend/src/modules/tabular/tabular.rows.ts @@ -6,9 +6,9 @@ // text a row's extraction runs over. Moved out of routes/tabular.ts so the // synchronous SSE route and the async extraction worker share one copy. -import { downloadFile } from "../storage"; -import { attachActiveVersionPaths } from "../documentVersions"; -import { safeErrorLog } from "../safeError"; +import { downloadFile } from "../../lib/storage"; +import { attachActiveVersionPaths } from "../../lib/documentVersions"; +import { safeErrorLog } from "../../lib/safeError"; import { extractDocumentMarkdown } from "./tabular.extract"; import { type Db } from "./tabular.shared"; diff --git a/backend/src/modules/tabular/tabular.service.ts b/backend/src/modules/tabular/tabular.service.ts new file mode 100644 index 000000000..24e2ad23a --- /dev/null +++ b/backend/src/modules/tabular/tabular.service.ts @@ -0,0 +1,50 @@ +// Service facade for the tabular-review module. Named re-exports only — the +// module's public service surface in one place, without leaking intra-module +// helpers. Routes (and the extraction worker) import from the topic files +// directly; this facade exists so cross-module consumers and tests have one +// stable import path. + +export { + createRowsForReview, + normalizeGrouping, + rebuildRowsForReview, + syncCellsForReviewRows, + type DocumentGrouping, +} from "./tabular.reviews"; +export { + fetchSourceDocuments, + loadReviewRow, + loadReviewRows, + loadRowDocumentText, + type ReviewRow, + type SourceDocument, +} from "./tabular.rows"; +export { + extractDocumentMarkdown, + extractDocxMarkdown, + extractPdfMarkdown, + generateChatTitle, + queryTabularAllColumns, + queryTabularCell, +} from "./tabular.extract"; +export { extractRowColumns, type CellSink } from "./tabular.extractRow"; +export { prepareTabularGenerate, type PreparedGenerate } from "./tabular.generate"; +export { + streamTabularGenerateAsync, + streamTabularRunView, + targetPendingCells, +} from "./tabular.generateStream"; +export { + buildTabularMessages, + extractTabularAnnotations, + parseTabularCitations, + type TabularParsedCitation, +} from "./tabular.chats"; +export { + missingModelApiKey, + parseCellContent, + type CellResult, + type Column, + type MissingApiKey, +} from "./tabular.shared"; +export { formatPromptSuffix } from "./tabular.prompt"; diff --git a/backend/src/lib/tabular/tabular.shared.ts b/backend/src/modules/tabular/tabular.shared.ts similarity index 97% rename from backend/src/lib/tabular/tabular.shared.ts rename to backend/src/modules/tabular/tabular.shared.ts index 0be370754..199721328 100644 --- a/backend/src/lib/tabular/tabular.shared.ts +++ b/backend/src/modules/tabular/tabular.shared.ts @@ -4,8 +4,8 @@ // (tabular.prompt.ts, tabular.extract.ts, …) and routes/tabular.ts can // import them. -import { createServerSupabase } from "../supabase"; -import { providerForModel, type Provider, type UserApiKeys } from "../llm"; +import { createServerSupabase } from "../../lib/supabase"; +import { providerForModel, type Provider, type UserApiKeys } from "../../lib/llm"; export type Db = ReturnType; diff --git a/backend/src/modules/user/user.account.ts b/backend/src/modules/user/user.account.ts new file mode 100644 index 000000000..730f2261e --- /dev/null +++ b/backend/src/modules/user/user.account.ts @@ -0,0 +1,84 @@ +// Account / data deletion (destructive — exact call args + ordering preserved). +// +// Service layer behind user.routes.ts — see user.shared.ts for the module's +// contract. The userDataCleanup helpers + auth-admin deleteUser call are +// invoked with identical args and ordering. + +import { + deleteAllUserChats, + deleteAllUserTabularReviews, + deleteUserAccountData, + deleteUserProjects, +} from "../../lib/userDataCleanup"; +import { type Db, errorMessage } from "./user.shared"; + +export async function deleteUserAccount( + db: Db, + userId: string, + userEmail: string | undefined, +): Promise<{ ok: true } | { ok: false; detail: string }> { + try { + await deleteUserAccountData(db, userId, userEmail); + const { error } = await db.auth.admin.deleteUser(userId); + if (error) return { ok: false, detail: error.message }; + return { ok: true }; + } catch (err) { + const detail = errorMessage(err); + console.error("[user/account] delete failed", { + userId, + error: detail, + }); + return { ok: false, detail }; + } +} + +export async function deleteUserChats( + db: Db, + userId: string, +): Promise<{ ok: true } | { ok: false; detail: string }> { + try { + await deleteAllUserChats(db, userId); + return { ok: true }; + } catch (err) { + const detail = errorMessage(err); + console.error("[user/chats] delete failed", { + userId, + error: detail, + }); + return { ok: false, detail }; + } +} + +export async function deleteUserProjectsData( + db: Db, + userId: string, +): Promise<{ ok: true } | { ok: false; detail: string }> { + try { + await deleteUserProjects(db, userId); + return { ok: true }; + } catch (err) { + const detail = errorMessage(err); + console.error("[user/projects] delete failed", { + userId, + error: detail, + }); + return { ok: false, detail }; + } +} + +export async function deleteUserTabularReviews( + db: Db, + userId: string, +): Promise<{ ok: true } | { ok: false; detail: string }> { + try { + await deleteAllUserTabularReviews(db, userId); + return { ok: true }; + } catch (err) { + const detail = errorMessage(err); + console.error("[user/tabular-reviews] delete failed", { + userId, + error: detail, + }); + return { ok: false, detail }; + } +} diff --git a/backend/src/modules/user/user.apiKeys.ts b/backend/src/modules/user/user.apiKeys.ts new file mode 100644 index 000000000..975c91659 --- /dev/null +++ b/backend/src/modules/user/user.apiKeys.ts @@ -0,0 +1,45 @@ +// User BYO API keys: status read + save. +// +// Service layer behind user.routes.ts — see user.shared.ts for the module's +// contract. Security boundary preserved verbatim: writes funnel through +// saveUserApiKey (the crypto is never reimplemented here). + +import { + type ApiKeyProvider, + type ApiKeyStatus, + getUserApiKeyStatus, + hasEnvApiKey, + saveUserApiKey, +} from "../../lib/userApiKeys"; +import { type Db, errorMessage } from "./user.shared"; + +export function getApiKeyStatus(db: Db, userId: string) { + return getUserApiKeyStatus(userId, db); +} + +export type SaveApiKeyResult = + | { ok: true; status: ApiKeyStatus } + | { ok: false; kind: "env_configured" } + | { ok: false; kind: "save_failed"; detail: string }; + +export async function saveApiKey( + db: Db, + params: { userId: string; provider: ApiKeyProvider; apiKey: string | null }, +): Promise { + const { userId, provider, apiKey } = params; + try { + if (hasEnvApiKey(provider)) { + return { ok: false, kind: "env_configured" }; + } + await saveUserApiKey(userId, provider, apiKey, db); + const status = await getUserApiKeyStatus(userId, db); + return { ok: true, status }; + } catch (err) { + const detail = errorMessage(err); + console.error("[user/api-keys] save failed", { + provider, + error: detail, + }); + return { ok: false, kind: "save_failed", detail }; + } +} diff --git a/backend/src/modules/user/user.export.ts b/backend/src/modules/user/user.export.ts new file mode 100644 index 000000000..c9ae9bed0 --- /dev/null +++ b/backend/src/modules/user/user.export.ts @@ -0,0 +1,63 @@ +// Data export (the route owns the Content-Type / Content-Disposition headers +// and filenames; these functions just build the payloads). +// +// Service layer behind user.routes.ts — see user.shared.ts for the module's +// contract. + +import { + buildUserAccountExport, + buildUserChatsExport, + buildUserTabularReviewsExport, +} from "../../lib/userDataExport"; +import { type Db, errorMessage } from "./user.shared"; + +export async function exportUserAccount( + db: Db, + userId: string, + userEmail: string | undefined, +): Promise<{ ok: true; data: unknown } | { ok: false; detail: string }> { + try { + const data = await buildUserAccountExport(db, userId, userEmail); + return { ok: true, data }; + } catch (err) { + const detail = errorMessage(err); + console.error("[user/export] failed", { userId, error: detail }); + return { ok: false, detail }; + } +} + +export async function exportUserChats( + db: Db, + userId: string, + userEmail: string | undefined, +): Promise<{ ok: true; data: unknown } | { ok: false; detail: string }> { + try { + const data = await buildUserChatsExport(db, userId, userEmail); + return { ok: true, data }; + } catch (err) { + const detail = errorMessage(err); + console.error("[user/chats/export] failed", { + userId, + error: detail, + }); + return { ok: false, detail }; + } +} + +export async function exportUserTabularReviews( + db: Db, + userId: string, + userEmail: string | undefined, +): Promise<{ ok: true; data: unknown } | { ok: false; detail: string }> { + try { + const data = await buildUserTabularReviewsExport(db, userId, userEmail); + return { ok: true, data }; + } catch (err) { + const detail = errorMessage(err); + console.error("[user/tabular-reviews/export] failed", { + userId, + error: detail, + }); + return { ok: false, detail }; + } +} diff --git a/backend/src/modules/user/user.mcp.ts b/backend/src/modules/user/user.mcp.ts new file mode 100644 index 000000000..a79c71f05 --- /dev/null +++ b/backend/src/modules/user/user.mcp.ts @@ -0,0 +1,208 @@ +// MCP connectors: thin {ok,...}|{ok:false,detail} wrappers over +// lib/mcpConnectors. +// +// Service layer behind user.routes.ts — see user.shared.ts for the module's +// contract. The OAuth callback exchange (completeUserMcpConnectorOAuth) stays +// in the route: it is inseparable from the popup-HTML/CSP response it renders. + +import { + createUserMcpConnector, + deleteUserMcpConnector, + getUserMcpConnector, + listUserMcpConnectors, + McpOAuthRequiredError, + refreshUserMcpConnectorTools, + setUserMcpToolEnabled, + startUserMcpConnectorOAuth, + updateUserMcpConnector, +} from "../../lib/mcpConnectors"; +import { type Db, errorMessage } from "./user.shared"; + +export async function listMcpConnectors( + db: Db, + userId: string, +): Promise<{ ok: true; connectors: unknown } | { ok: false; detail: string }> { + try { + const connectors = await listUserMcpConnectors(userId, db, { + includeTools: false, + }); + return { ok: true, connectors }; + } catch (err) { + const detail = errorMessage(err); + console.error("[user/mcp-connectors] list failed", { + userId, + error: detail, + }); + return { ok: false, detail }; + } +} + +export async function getMcpConnector( + db: Db, + userId: string, + connectorId: string, +): Promise<{ ok: true; connector: unknown } | { ok: false; detail: string }> { + try { + const connector = await getUserMcpConnector(userId, connectorId, db); + return { ok: true, connector }; + } catch (err) { + const detail = errorMessage(err); + console.error("[user/mcp-connectors] get failed", { + userId, + connectorId, + error: detail, + }); + return { ok: false, detail }; + } +} + +export async function createMcpConnector( + db: Db, + userId: string, + params: { + name: string; + serverUrl: string; + bearerToken: string | null; + headers: Record | undefined; + }, +): Promise<{ ok: true; connector: unknown } | { ok: false; detail: string }> { + try { + const connector = await createUserMcpConnector(userId, params, db); + return { ok: true, connector }; + } catch (err) { + const detail = errorMessage(err); + console.error("[user/mcp-connectors] create failed", { + userId, + error: detail, + }); + return { ok: false, detail }; + } +} + +export async function updateMcpConnector( + db: Db, + userId: string, + connectorId: string, + updates: Parameters[2], +): Promise<{ ok: true; connector: unknown } | { ok: false; detail: string }> { + try { + const connector = await updateUserMcpConnector( + userId, + connectorId, + updates, + db, + ); + return { ok: true, connector }; + } catch (err) { + const detail = errorMessage(err); + console.error("[user/mcp-connectors] update failed", { + userId, + connectorId, + error: detail, + }); + return { ok: false, detail }; + } +} + +export async function deleteMcpConnector( + db: Db, + userId: string, + connectorId: string, +): Promise<{ ok: true } | { ok: false; detail: string }> { + try { + await deleteUserMcpConnector(userId, connectorId, db); + return { ok: true }; + } catch (err) { + const detail = errorMessage(err); + console.error("[user/mcp-connectors] delete failed", { + userId, + connectorId, + error: detail, + }); + return { ok: false, detail }; + } +} + +export async function startMcpConnectorOAuth( + db: Db, + userId: string, + connectorId: string, + redirectUri: string, +): Promise<{ ok: true; result: unknown } | { ok: false; detail: string }> { + try { + const result = await startUserMcpConnectorOAuth( + userId, + connectorId, + redirectUri, + db, + ); + return { ok: true, result }; + } catch (err) { + const detail = errorMessage(err); + console.error("[user/mcp-connectors] oauth start failed", { + userId, + connectorId, + error: detail, + }); + return { ok: false, detail }; + } +} + +export type RefreshMcpToolsResult = + | { ok: true; connector: unknown } + | { ok: false; kind: "oauth_required"; code: string; detail: string } + | { ok: false; kind: "refresh_failed"; detail: string }; + +export async function refreshMcpConnectorTools( + db: Db, + userId: string, + connectorId: string, +): Promise { + try { + const connector = await refreshUserMcpConnectorTools( + userId, + connectorId, + db, + ); + return { ok: true, connector }; + } catch (err) { + const detail = errorMessage(err); + console.error("[user/mcp-connectors] refresh failed", { + userId, + connectorId, + error: detail, + }); + if (err instanceof McpOAuthRequiredError) { + return { ok: false, kind: "oauth_required", code: err.code, detail }; + } + return { ok: false, kind: "refresh_failed", detail }; + } +} + +export async function setMcpToolEnabled( + db: Db, + userId: string, + connectorId: string, + toolId: string, + enabled: boolean, +): Promise<{ ok: true; connector: unknown } | { ok: false; detail: string }> { + try { + const connector = await setUserMcpToolEnabled( + userId, + connectorId, + toolId, + enabled, + db, + ); + return { ok: true, connector }; + } catch (err) { + const detail = errorMessage(err); + console.error("[user/mcp-connectors] tool toggle failed", { + userId, + connectorId, + toolId, + error: detail, + }); + return { ok: false, detail }; + } +} diff --git a/backend/src/modules/user/user.mfa.ts b/backend/src/modules/user/user.mfa.ts new file mode 100644 index 000000000..9d9874011 --- /dev/null +++ b/backend/src/modules/user/user.mfa.ts @@ -0,0 +1,72 @@ +// MFA-on-login toggle. +// +// Service layer behind user.routes.ts — see user.shared.ts for the module's +// contract. The requireMfaIfEnrolled guard stays in the route (HTTP layer); +// only the verified-TOTP factor lookup lives here. Reuses the profile-row +// helpers (ensureProfileRow / loadProfile) from user.profile.ts. + +import { getUserApiKeyStatus } from "../../lib/userApiKeys"; +import { type Db } from "./user.shared"; +import { ensureProfileRow, loadProfile } from "./user.profile"; + +async function userHasVerifiedTotpFactor(db: Db, userId: string) { + const { data, error } = await db.auth.admin.getUserById(userId); + if (error) return { ok: false as const, error }; + + const factors = data.user?.factors ?? []; + return { + ok: true as const, + hasVerifiedTotp: factors.some( + (factor) => + factor.factor_type === "totp" && factor.status === "verified", + ), + }; +} + +export type SetMfaOnLoginResult = + | { ok: true; body: Record } + | { ok: false; kind: "no_factor"; detail: string } + | { ok: false; kind: "db_error"; detail: string }; + +export async function setMfaOnLogin( + db: Db, + userId: string, + enabled: boolean, +): Promise { + if (enabled) { + const factorCheck = await userHasVerifiedTotpFactor(db, userId); + if (!factorCheck.ok) { + return { + ok: false, + kind: "db_error", + detail: factorCheck.error.message, + }; + } + if (!factorCheck.hasVerifiedTotp) { + return { + ok: false, + kind: "no_factor", + detail: "Set up an authenticator app before requiring verification on login.", + }; + } + } + + const ensureError = await ensureProfileRow(db, userId); + if (ensureError) + return { ok: false, kind: "db_error", detail: ensureError.message }; + + const { error: updateError } = await db + .from("user_profiles") + .update({ + mfa_on_login: enabled, + updated_at: new Date().toISOString(), + }) + .eq("user_id", userId); + if (updateError) + return { ok: false, kind: "db_error", detail: updateError.message }; + + const apiKeyStatus = await getUserApiKeyStatus(userId, db); + const { data, error } = await loadProfile(db, userId, { apiKeyStatus }); + if (error) return { ok: false, kind: "db_error", detail: error.message }; + return { ok: true, body: { ...data, apiKeyStatus } }; +} diff --git a/backend/src/modules/user/user.profile.ts b/backend/src/modules/user/user.profile.ts new file mode 100644 index 000000000..a4bd25db2 --- /dev/null +++ b/backend/src/modules/user/user.profile.ts @@ -0,0 +1,414 @@ +// User profile: load, serialize, validate, bootstrap, read + update. +// +// Service layer behind user.routes.ts — see user.shared.ts for the module's +// contract (explicit `db`, request-derived primitives in, typed result objects +// out, no req/res). The profile-row loaders (ensureProfileRow / loadProfile) +// are exported for intra-module reuse by user.mfa.ts; the facade does NOT +// re-export them, so they stay off the module's public surface. + +import { + DEFAULT_TABULAR_MODEL, + DEFAULT_TITLE_MODEL, + CLAUDE_LOW_MODELS, + OPENAI_LOW_MODELS, + resolveModel, +} from "../../lib/llm"; +import { + type ApiKeyStatus, + getUserApiKeyStatus, +} from "../../lib/userApiKeys"; +import { findProfileUserByEmail } from "../../lib/userLookup"; +import { type Db } from "./user.shared"; + +const MONTHLY_CREDIT_LIMIT = 999999; + +type UserProfileRow = { + display_name: string | null; + organisation: string | null; + message_credits_used: number; + credits_reset_date: string; + tier: string; + title_model: string | null; + tabular_model: string; + mfa_on_login: boolean | null; + legal_research_us: boolean | null; +}; + +const PROFILE_SELECT = + "display_name, organisation, message_credits_used, credits_reset_date, tier, title_model, tabular_model, mfa_on_login, legal_research_us"; +const PROFILE_SELECT_NO_LEGAL = + "display_name, organisation, message_credits_used, credits_reset_date, tier, title_model, tabular_model, mfa_on_login"; +const LEGACY_PROFILE_SELECT = + "display_name, organisation, message_credits_used, credits_reset_date, tier, tabular_model"; +const LEGACY_PROFILE_MODEL_SELECT = + "display_name, organisation, message_credits_used, credits_reset_date, tier, title_model, tabular_model"; + +function isMissingProfileColumn(error: unknown, column: string): boolean { + const record = + error && typeof error === "object" + ? (error as { code?: unknown; message?: unknown }) + : {}; + const message = typeof record.message === "string" ? record.message : ""; + return record.code === "42703" && message.includes(column); +} + +// Loads a profile while tolerating older databases that lack the +// legal_research_us column. Tries the full select first, then falls back to +// the legacy cascade (which also handles missing title_model / mfa_on_login) +// and defaults the feature flag to enabled. +async function selectProfile(db: Db, userId: string, mode: "maybe" | "single") { + const fullQuery = db + .from("user_profiles") + .select(PROFILE_SELECT) + .eq("user_id", userId); + const full = + mode === "single" + ? await fullQuery.single() + : await fullQuery.maybeSingle(); + if (!full.error) return full; + + const legacy = await selectProfileLegacy(db, userId, mode); + if (legacy.data && typeof legacy.data === "object") { + const row = legacy.data as Record; + if (!("legal_research_us" in row)) { + Object.assign(row, { legal_research_us: true }); + } + } + return legacy; +} + +async function selectProfileLegacy( + db: Db, + userId: string, + mode: "maybe" | "single", +) { + const query = db + .from("user_profiles") + .select(PROFILE_SELECT_NO_LEGAL) + .eq("user_id", userId); + const result = + mode === "single" ? await query.single() : await query.maybeSingle(); + if (!result.error) { + return result; + } + + const missingMfaOnLogin = isMissingProfileColumn( + result.error, + "mfa_on_login", + ); + if (missingMfaOnLogin) { + const modelQuery = db + .from("user_profiles") + .select(LEGACY_PROFILE_MODEL_SELECT) + .eq("user_id", userId); + const modelLegacy = + mode === "single" + ? await modelQuery.single() + : await modelQuery.maybeSingle(); + if ( + !modelLegacy.error || + !isMissingProfileColumn(modelLegacy.error, "title_model") + ) { + if (modelLegacy.data && typeof modelLegacy.data === "object") { + const row = modelLegacy.data as Record; + Object.assign(row, { + mfa_on_login: false, + }); + } + return modelLegacy; + } + } + + if ( + !missingMfaOnLogin && + !isMissingProfileColumn(result.error, "title_model") + ) { + return result; + } + + const legacyQuery = db + .from("user_profiles") + .select(LEGACY_PROFILE_SELECT) + .eq("user_id", userId); + const legacy = + mode === "single" + ? await legacyQuery.single() + : await legacyQuery.maybeSingle(); + if (legacy.data && typeof legacy.data === "object") { + const row = legacy.data as Record; + Object.assign(row, { + title_model: null, + mfa_on_login: false, + }); + } + return legacy; +} + +function serializeProfile(row: UserProfileRow, apiKeyStatus?: ApiKeyStatus) { + const creditsUsed = row.message_credits_used ?? 0; + const titleFallback = apiKeyStatus?.gemini + ? DEFAULT_TITLE_MODEL + : apiKeyStatus?.openai + ? OPENAI_LOW_MODELS[0] + : apiKeyStatus?.claude + ? CLAUDE_LOW_MODELS[0] + : DEFAULT_TITLE_MODEL; + return { + displayName: row.display_name, + organisation: row.organisation, + messageCreditsUsed: creditsUsed, + creditsResetDate: row.credits_reset_date, + creditsRemaining: Math.max(MONTHLY_CREDIT_LIMIT - creditsUsed, 0), + tier: row.tier || "Free", + titleModel: resolveModel(row.title_model, titleFallback), + tabularModel: resolveModel(row.tabular_model, DEFAULT_TABULAR_MODEL), + mfaOnLogin: row.mfa_on_login === true, + legalResearchUs: row.legal_research_us !== false, + ...(apiKeyStatus ? { apiKeyStatus } : {}), + }; +} + +export function validateProfilePayload(body: unknown): + | { + ok: true; + update: { + display_name?: string | null; + organisation?: string | null; + title_model?: string; + tabular_model?: string; + legal_research_us?: boolean; + updated_at: string; + }; + } + | { ok: false; detail: string } { + if (!body || typeof body !== "object" || Array.isArray(body)) { + return { ok: false, detail: "Expected a JSON object" }; + } + + const raw = body as Record; + const allowedFields = new Set([ + "displayName", + "organisation", + "titleModel", + "tabularModel", + "legalResearchUs", + ]); + const invalidField = Object.keys(raw).find( + (key) => !allowedFields.has(key), + ); + if (invalidField) { + return { + ok: false, + detail: `Unsupported profile field: ${invalidField}`, + }; + } + + const update: { + display_name?: string | null; + organisation?: string | null; + title_model?: string; + tabular_model?: string; + legal_research_us?: boolean; + updated_at: string; + } = { updated_at: new Date().toISOString() }; + + if ("displayName" in raw) { + if (raw.displayName !== null && typeof raw.displayName !== "string") { + return { + ok: false, + detail: "displayName must be a string or null", + }; + } + update.display_name = raw.displayName?.trim() || null; + } + + if ("organisation" in raw) { + if (raw.organisation !== null && typeof raw.organisation !== "string") { + return { + ok: false, + detail: "organisation must be a string or null", + }; + } + update.organisation = raw.organisation?.trim() || null; + } + + if ("tabularModel" in raw) { + if (typeof raw.tabularModel !== "string") { + return { ok: false, detail: "tabularModel must be a string" }; + } + const resolved = resolveModel(raw.tabularModel, ""); + if (!resolved) { + return { ok: false, detail: "Unsupported tabularModel" }; + } + update.tabular_model = resolved; + } + + if ("titleModel" in raw) { + if (typeof raw.titleModel !== "string") { + return { ok: false, detail: "titleModel must be a string" }; + } + const resolved = resolveModel(raw.titleModel, ""); + if (!resolved) { + return { ok: false, detail: "Unsupported titleModel" }; + } + update.title_model = resolved; + } + + if ("legalResearchUs" in raw) { + if (typeof raw.legalResearchUs !== "boolean") { + return { + ok: false, + detail: "legalResearchUs must be a boolean", + }; + } + update.legal_research_us = raw.legalResearchUs; + } + + return { ok: true, update }; +} + +export function readBooleanBodyField( + body: unknown, + field: string, +): { ok: true; value: boolean } | { ok: false; detail: string } { + if (!body || typeof body !== "object" || Array.isArray(body)) { + return { ok: false, detail: "Expected a JSON object" }; + } + + const raw = body as Record; + const invalidField = Object.keys(raw).find((key) => key !== field); + if (invalidField) { + return { ok: false, detail: `Unsupported field: ${invalidField}` }; + } + if (typeof raw[field] !== "boolean") { + return { ok: false, detail: `${field} must be a boolean` }; + } + + return { ok: true, value: raw[field] }; +} + +export async function ensureProfileRow(db: Db, userId: string) { + const { error } = await db + .from("user_profiles") + .upsert( + { user_id: userId }, + { onConflict: "user_id", ignoreDuplicates: true }, + ); + return error; +} + +export async function loadProfile( + db: Db, + userId: string, + options: { repairMissing?: boolean; apiKeyStatus?: ApiKeyStatus } = {}, +) { + let { data, error } = await selectProfile(db, userId, "maybe"); + + if (error) return { data: null, error }; + if (!data) { + if (!options.repairMissing) { + return { data: null, error: new Error("Profile not found") }; + } + + const ensureError = await ensureProfileRow(db, userId); + if (ensureError) return { data: null, error: ensureError }; + + const created = await selectProfile(db, userId, "single"); + if (created.error) return { data: null, error: created.error }; + data = created.data; + } + + let row = data as UserProfileRow; + if ( + row.credits_reset_date && + new Date() > new Date(row.credits_reset_date) + ) { + const creditsResetDate = new Date(); + creditsResetDate.setDate(creditsResetDate.getDate() + 30); + const { error: resetError } = await db + .from("user_profiles") + .update({ + message_credits_used: 0, + credits_reset_date: creditsResetDate.toISOString(), + updated_at: new Date().toISOString(), + }) + .eq("user_id", userId); + + if (resetError) return { data: null, error: resetError }; + const { data: resetData, error: resetLoadError } = await selectProfile( + db, + userId, + "single", + ); + if (resetLoadError) return { data: null, error: resetLoadError }; + row = resetData as UserProfileRow; + } + + return { data: serializeProfile(row, options.apiKeyStatus), error: null }; +} + +// --------------------------------------------------------------------------- +// Profile +// --------------------------------------------------------------------------- + +export async function bootstrapUserProfile( + db: Db, + userId: string, +): Promise<{ ok: true } | { ok: false; detail: string }> { + const error = await ensureProfileRow(db, userId); + if (error) return { ok: false, detail: error.message }; + return { ok: true }; +} + +export async function getUserProfile( + db: Db, + userId: string, +): Promise< + { ok: true; body: Record } | { ok: false; detail: string } +> { + const apiKeyStatus = await getUserApiKeyStatus(userId, db); + const { data, error } = await loadProfile(db, userId, { + repairMissing: true, + apiKeyStatus, + }); + if (error) return { ok: false, detail: error.message }; + return { ok: true, body: { ...data, apiKeyStatus } }; +} + +export async function lookupUserByEmail( + db: Db, + email: string, +): Promise<{ + exists: boolean; + email: string; + display_name: string | null; +}> { + const user = await findProfileUserByEmail(db, email); + return { + exists: !!user, + email: user?.email ?? email.trim().toLowerCase(), + display_name: user?.display_name ?? null, + }; +} + +export async function updateUserProfile( + db: Db, + userId: string, + update: Record, +): Promise< + { ok: true; body: Record } | { ok: false; detail: string } +> { + const ensureError = await ensureProfileRow(db, userId); + if (ensureError) return { ok: false, detail: ensureError.message }; + + const { error: updateError } = await db + .from("user_profiles") + .update(update) + .eq("user_id", userId); + if (updateError) return { ok: false, detail: updateError.message }; + + const apiKeyStatus = await getUserApiKeyStatus(userId, db); + const { data, error } = await loadProfile(db, userId, { apiKeyStatus }); + if (error) return { ok: false, detail: error.message }; + return { ok: true, body: { ...data, apiKeyStatus } }; +} diff --git a/backend/src/modules/user/user.routes.ts b/backend/src/modules/user/user.routes.ts new file mode 100644 index 000000000..033eaac50 --- /dev/null +++ b/backend/src/modules/user/user.routes.ts @@ -0,0 +1,582 @@ +import crypto from "crypto"; +import { Router } from "express"; +import { requireAuth, requireMfaIfEnrolled } from "../../middleware/auth"; +import { createServerSupabase } from "../../lib/supabase"; +import { normalizeApiKeyProvider } from "../../lib/userApiKeys"; +import { completeUserMcpConnectorOAuth } from "../../lib/mcpConnectors"; +import { userExportFilename } from "../../lib/userDataExport"; +import { + bootstrapUserProfile, + createMcpConnector, + deleteMcpConnector, + deleteUserAccount, + deleteUserChats, + deleteUserProjectsData, + deleteUserTabularReviews, + errorMessage, + exportUserAccount, + exportUserChats, + exportUserTabularReviews, + getApiKeyStatus, + getMcpConnector, + getUserProfile, + lookupUserByEmail, + listMcpConnectors, + readBooleanBodyField, + refreshMcpConnectorTools, + saveApiKey, + setMcpToolEnabled, + setMfaOnLogin, + startMcpConnectorOAuth, + updateMcpConnector, + updateUserProfile, + validateProfilePayload, +} from "./user.service"; + +export const userRouter = Router(); + +function backendPublicUrl(req: { + protocol: string; + get(name: string): string | undefined; +}) { + return ( + process.env.API_PUBLIC_URL || + process.env.BACKEND_URL || + `${req.protocol}://${req.get("host")}` + ).replace(/\/+$/, ""); +} + +function frontendUrl(path = "/account/connectors") { + const base = (process.env.FRONTEND_URL ?? "http://localhost:3000").replace( + /\/+$/, + "", + ); + return `${base}${path}`; +} + +function shortHash(value: string) { + return value + ? crypto.createHash("sha256").update(value).digest("hex").slice(0, 12) + : null; +} + +function mcpOAuthPopupHtml(payload: { + success: boolean; + connectorId?: string; + detail?: string; +}, nonce: string) { + const targetOrigin = new URL(frontendUrl()).origin; + const targetUrl = frontendUrl(); + const message = JSON.stringify({ + type: "mcp_oauth_result", + ...payload, + }); + return ` + + + + + MCP authorization + + + +
+

${payload.success ? "Authorization complete" : "Authorization failed"}

+

${payload.success ? "You can return to Mike." : "Return to Mike and try connecting again."}

+
+ + +`; +} + +function mcpOAuthPopupCsp(nonce: string) { + return [ + "default-src 'none'", + `script-src 'nonce-${nonce}'`, + "style-src 'unsafe-inline'", + "base-uri 'none'", + "form-action 'none'", + "frame-ancestors 'none'", + ].join("; "); +} + +// POST /user/profile +userRouter.post("/profile", requireAuth, async (_req, res) => { + const userId = res.locals.userId as string; + const db = createServerSupabase(); + const result = await bootstrapUserProfile(db, userId); + if (!result.ok) return void res.status(500).json({ detail: result.detail }); + res.json({ ok: true }); +}); + +// GET /user/lookup?email=person@example.com +userRouter.get("/lookup", requireAuth, async (req, res) => { + const email = typeof req.query.email === "string" ? req.query.email : ""; + if (!email.trim()) { + return void res.status(400).json({ detail: "email is required" }); + } + + const db = createServerSupabase(); + res.json(await lookupUserByEmail(db, email)); +}); + +// GET /user/profile +userRouter.get("/profile", requireAuth, async (_req, res) => { + const userId = res.locals.userId as string; + const db = createServerSupabase(); + const result = await getUserProfile(db, userId); + if (!result.ok) return void res.status(500).json({ detail: result.detail }); + res.json(result.body); +}); + +// PATCH /user/profile +userRouter.patch("/profile", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const parsed = validateProfilePayload(req.body); + if (!parsed.ok) return void res.status(400).json({ detail: parsed.detail }); + + const db = createServerSupabase(); + const result = await updateUserProfile(db, userId, parsed.update); + if (!result.ok) return void res.status(500).json({ detail: result.detail }); + res.json(result.body); +}); + +// PATCH /user/security/mfa-login +userRouter.patch( + "/security/mfa-login", + requireAuth, + requireMfaIfEnrolled, + async (req, res) => { + const userId = res.locals.userId as string; + const parsed = readBooleanBodyField(req.body, "enabled"); + if (!parsed.ok) + return void res.status(400).json({ detail: parsed.detail }); + + const db = createServerSupabase(); + const result = await setMfaOnLogin(db, userId, parsed.value); + if (!result.ok) { + if (result.kind === "no_factor") + return void res.status(400).json({ detail: result.detail }); + return void res.status(500).json({ detail: result.detail }); + } + res.json(result.body); + }, +); + +// GET /user/api-keys +userRouter.get("/api-keys", requireAuth, async (_req, res) => { + const userId = res.locals.userId as string; + const db = createServerSupabase(); + const status = await getApiKeyStatus(db, userId); + res.json(status); +}); + +// PUT /user/api-keys/:provider +userRouter.put( + "/api-keys/:provider", + requireAuth, + requireMfaIfEnrolled, + async (req, res) => { + const userId = res.locals.userId as string; + const provider = normalizeApiKeyProvider(req.params.provider); + if (!provider) + return void res + .status(400) + .json({ detail: "Unsupported provider" }); + + const apiKey = + typeof req.body?.api_key === "string" ? req.body.api_key : null; + const db = createServerSupabase(); + const result = await saveApiKey(db, { userId, provider, apiKey }); + if (!result.ok) { + if (result.kind === "env_configured") + return void res.status(409).json({ + detail: "This provider is configured by the server environment and cannot be changed from the browser.", + }); + return void res.status(500).json({ detail: result.detail }); + } + res.json(result.status); + }, +); + +// GET /user/mcp-connectors +userRouter.get("/mcp-connectors", requireAuth, async (_req, res) => { + const userId = res.locals.userId as string; + const db = createServerSupabase(); + const result = await listMcpConnectors(db, userId); + if (!result.ok) return void res.status(500).json({ detail: result.detail }); + res.json(result.connectors); +}); + +// GET /user/mcp-connectors/:connectorId +userRouter.get( + "/mcp-connectors/:connectorId", + requireAuth, + async (req, res) => { + const userId = res.locals.userId as string; + const db = createServerSupabase(); + const result = await getMcpConnector( + db, + userId, + req.params.connectorId, + ); + if (!result.ok) + return void res.status(404).json({ detail: result.detail }); + res.json(result.connector); + }, +); + +// POST /user/mcp-connectors +userRouter.post( + "/mcp-connectors", + requireAuth, + requireMfaIfEnrolled, + async (req, res) => { + const userId = res.locals.userId as string; + const name = typeof req.body?.name === "string" ? req.body.name : ""; + const serverUrl = + typeof req.body?.serverUrl === "string" ? req.body.serverUrl : ""; + const bearerToken = + typeof req.body?.bearerToken === "string" + ? req.body.bearerToken + : null; + const headers = + req.body?.headers && + typeof req.body.headers === "object" && + !Array.isArray(req.body.headers) + ? (req.body.headers as Record) + : undefined; + const db = createServerSupabase(); + const result = await createMcpConnector(db, userId, { + name, + serverUrl, + bearerToken, + headers, + }); + if (!result.ok) + return void res.status(400).json({ detail: result.detail }); + res.status(201).json(result.connector); + }, +); + +// PATCH /user/mcp-connectors/:connectorId +userRouter.patch( + "/mcp-connectors/:connectorId", + requireAuth, + requireMfaIfEnrolled, + async (req, res) => { + const userId = res.locals.userId as string; + const db = createServerSupabase(); + const body = req.body ?? {}; + const result = await updateMcpConnector( + db, + userId, + req.params.connectorId, + { + ...(typeof body.name === "string" ? { name: body.name } : {}), + ...(typeof body.serverUrl === "string" + ? { serverUrl: body.serverUrl } + : {}), + ...(typeof body.enabled === "boolean" + ? { enabled: body.enabled } + : {}), + ...("bearerToken" in body + ? { + bearerToken: + typeof body.bearerToken === "string" + ? body.bearerToken + : null, + } + : {}), + ...("headers" in body + ? { + headers: + body.headers && + typeof body.headers === "object" && + !Array.isArray(body.headers) + ? (body.headers as Record) + : {}, + } + : {}), + }, + ); + if (!result.ok) + return void res.status(400).json({ detail: result.detail }); + res.json(result.connector); + }, +); + +// DELETE /user/mcp-connectors/:connectorId +userRouter.delete( + "/mcp-connectors/:connectorId", + requireAuth, + requireMfaIfEnrolled, + async (req, res) => { + const userId = res.locals.userId as string; + const db = createServerSupabase(); + const result = await deleteMcpConnector( + db, + userId, + req.params.connectorId, + ); + if (!result.ok) + return void res.status(500).json({ detail: result.detail }); + res.status(204).send(); + }, +); + +// POST /user/mcp-connectors/:connectorId/oauth/start +userRouter.post( + "/mcp-connectors/:connectorId/oauth/start", + requireAuth, + requireMfaIfEnrolled, + async (req, res) => { + const userId = res.locals.userId as string; + const db = createServerSupabase(); + const redirectUri = `${backendPublicUrl(req)}/user/mcp-connectors/oauth/callback`; + const result = await startMcpConnectorOAuth( + db, + userId, + req.params.connectorId, + redirectUri, + ); + if (!result.ok) + return void res.status(400).json({ detail: result.detail }); + res.json(result.result); + }, +); + +// GET /user/mcp-connectors/oauth/callback +userRouter.get("/mcp-connectors/oauth/callback", async (req, res) => { + const nonce = crypto.randomBytes(16).toString("base64"); + const state = typeof req.query.state === "string" ? req.query.state : ""; + const code = typeof req.query.code === "string" ? req.query.code : ""; + const error = + typeof req.query.error === "string" ? req.query.error : undefined; + const db = createServerSupabase(); + try { + if (error) throw new Error(error); + if (!state || !code) + throw new Error("OAuth callback is missing state or code."); + const result = await completeUserMcpConnectorOAuth(state, code, db); + res.set("Content-Security-Policy", mcpOAuthPopupCsp(nonce)) + .type("html") + .send( + mcpOAuthPopupHtml( + { + success: true, + connectorId: result.connectorId, + }, + nonce, + ), + ); + } catch (err) { + const detail = errorMessage(err); + console.error("[user/mcp-connectors] oauth callback failed", { + error: detail, + stateHash: shortHash(state), + hasCode: !!code, + hasError: !!error, + issuer: + typeof req.query.iss === "string" ? req.query.iss : undefined, + scope: + typeof req.query.scope === "string" + ? req.query.scope + : undefined, + }); + res.status(400) + .set("Content-Security-Policy", mcpOAuthPopupCsp(nonce)) + .type("html") + .send(mcpOAuthPopupHtml({ success: false, detail }, nonce)); + } +}); + +// POST /user/mcp-connectors/:connectorId/refresh-tools +userRouter.post( + "/mcp-connectors/:connectorId/refresh-tools", + requireAuth, + requireMfaIfEnrolled, + async (req, res) => { + const userId = res.locals.userId as string; + const db = createServerSupabase(); + const result = await refreshMcpConnectorTools( + db, + userId, + req.params.connectorId, + ); + if (!result.ok) { + if (result.kind === "oauth_required") + return void res.status(401).json({ + code: result.code, + detail: result.detail, + }); + return void res.status(400).json({ detail: result.detail }); + } + res.json(result.connector); + }, +); + +// PATCH /user/mcp-connectors/:connectorId/tools/:toolId +userRouter.patch( + "/mcp-connectors/:connectorId/tools/:toolId", + requireAuth, + requireMfaIfEnrolled, + async (req, res) => { + const userId = res.locals.userId as string; + const parsed = readBooleanBodyField(req.body, "enabled"); + if (!parsed.ok) + return void res.status(400).json({ detail: parsed.detail }); + + const db = createServerSupabase(); + const result = await setMcpToolEnabled( + db, + userId, + req.params.connectorId, + req.params.toolId, + parsed.value, + ); + if (!result.ok) + return void res.status(400).json({ detail: result.detail }); + res.json(result.connector); + }, +); + +// DELETE /user/account +userRouter.delete( + "/account", + requireAuth, + requireMfaIfEnrolled, + async (_req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const db = createServerSupabase(); + const result = await deleteUserAccount(db, userId, userEmail); + if (!result.ok) + return void res.status(500).json({ detail: result.detail }); + res.status(204).send(); + }, +); + +// DELETE /user/chats +userRouter.delete( + "/chats", + requireAuth, + requireMfaIfEnrolled, + async (_req, res) => { + const userId = res.locals.userId as string; + const db = createServerSupabase(); + const result = await deleteUserChats(db, userId); + if (!result.ok) + return void res.status(500).json({ detail: result.detail }); + res.status(204).send(); + }, +); + +// DELETE /user/projects +userRouter.delete( + "/projects", + requireAuth, + requireMfaIfEnrolled, + async (_req, res) => { + const userId = res.locals.userId as string; + const db = createServerSupabase(); + const result = await deleteUserProjectsData(db, userId); + if (!result.ok) + return void res.status(500).json({ detail: result.detail }); + res.status(204).send(); + }, +); + +// DELETE /user/tabular-reviews +userRouter.delete( + "/tabular-reviews", + requireAuth, + requireMfaIfEnrolled, + async (_req, res) => { + const userId = res.locals.userId as string; + const db = createServerSupabase(); + const result = await deleteUserTabularReviews(db, userId); + if (!result.ok) + return void res.status(500).json({ detail: result.detail }); + res.status(204).send(); + }, +); + +// GET /user/export +userRouter.get( + "/export", + requireAuth, + requireMfaIfEnrolled, + async (_req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const db = createServerSupabase(); + const result = await exportUserAccount(db, userId, userEmail); + if (!result.ok) + return void res.status(500).json({ detail: result.detail }); + res.setHeader("Content-Type", "application/json; charset=utf-8"); + res.setHeader( + "Content-Disposition", + `attachment; filename="${userExportFilename("account", userId)}"`, + ); + res.json(result.data); + }, +); + +// GET /user/chats/export +userRouter.get( + "/chats/export", + requireAuth, + requireMfaIfEnrolled, + async (_req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const db = createServerSupabase(); + const result = await exportUserChats(db, userId, userEmail); + if (!result.ok) + return void res.status(500).json({ detail: result.detail }); + res.setHeader("Content-Type", "application/json; charset=utf-8"); + res.setHeader( + "Content-Disposition", + `attachment; filename="${userExportFilename("chats", userId)}"`, + ); + res.json(result.data); + }, +); + +// GET /user/tabular-reviews/export +userRouter.get( + "/tabular-reviews/export", + requireAuth, + requireMfaIfEnrolled, + async (_req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const db = createServerSupabase(); + const result = await exportUserTabularReviews(db, userId, userEmail); + if (!result.ok) + return void res.status(500).json({ detail: result.detail }); + res.setHeader("Content-Type", "application/json; charset=utf-8"); + res.setHeader( + "Content-Disposition", + `attachment; filename="${userExportFilename("tabular-reviews", userId)}"`, + ); + res.json(result.data); + }, +); diff --git a/backend/src/modules/user/user.service.ts b/backend/src/modules/user/user.service.ts new file mode 100644 index 000000000..6bfdcd0c7 --- /dev/null +++ b/backend/src/modules/user/user.service.ts @@ -0,0 +1,76 @@ +// Business logic + data-access for the user module. +// +// These functions are the service layer behind user.routes.ts. They take an +// explicit Supabase client (`db`) plus request-derived primitives, perform the +// profile / MFA / API-key / MCP / export / deletion orchestration, and RETURN +// values or typed error results. They never touch req/res — the thin route +// handlers map the results onto HTTP status codes, headers, and response +// bodies. +// +// The implementation is split by concern across sibling files; this module is +// the aggregate surface the routes (and tests) import from: +// +// user.shared.ts — shared types + helpers (Db, errorMessage) +// user.profile.ts — load/serialize/validate + bootstrap/read/update profile +// user.mfa.ts — the MFA-on-login toggle (+ verified-TOTP factor lookup) +// user.apiKeys.ts — BYO API-key status + save (crypto stays in the lib) +// user.mcp.ts — MCP connector wrappers over lib/mcpConnectors +// user.account.ts — destructive account/data deletion (args + ordering kept) +// user.export.ts — data-export payload builders +// +// Security boundaries preserved across the split verbatim: +// - API-key crypto: writes funnel through saveUserApiKey (never reimplemented). +// - MFA: the requireMfaIfEnrolled guard stays in the route (HTTP layer); only +// the verified-TOTP factor lookup lives here. +// - Data deletion: the userDataCleanup helpers + auth-admin deleteUser call are +// invoked with identical args and ordering (destructive — exact preservation). +// - Exports: the payload builders are called here; the route owns the +// Content-Type / Content-Disposition headers and filenames. +// +// The re-exports below are NAMED so intra-module helpers (e.g. the profile-row +// loaders reused by user.mfa.ts) stay off this public surface — the routes and +// tests import exactly the same names they always did. + +export { errorMessage } from "./user.shared"; + +export { + validateProfilePayload, + readBooleanBodyField, + bootstrapUserProfile, + getUserProfile, + lookupUserByEmail, + updateUserProfile, +} from "./user.profile"; + +export { setMfaOnLogin, type SetMfaOnLoginResult } from "./user.mfa"; + +export { + getApiKeyStatus, + saveApiKey, + type SaveApiKeyResult, +} from "./user.apiKeys"; + +export { + listMcpConnectors, + getMcpConnector, + createMcpConnector, + updateMcpConnector, + deleteMcpConnector, + startMcpConnectorOAuth, + refreshMcpConnectorTools, + setMcpToolEnabled, + type RefreshMcpToolsResult, +} from "./user.mcp"; + +export { + deleteUserAccount, + deleteUserChats, + deleteUserProjectsData, + deleteUserTabularReviews, +} from "./user.account"; + +export { + exportUserAccount, + exportUserChats, + exportUserTabularReviews, +} from "./user.export"; diff --git a/backend/src/modules/user/user.shared.ts b/backend/src/modules/user/user.shared.ts new file mode 100644 index 000000000..072c1c142 --- /dev/null +++ b/backend/src/modules/user/user.shared.ts @@ -0,0 +1,32 @@ +// Shared types + helpers for the user module service layer. +// +// The user service is split by concern across sibling files +// (user.profile.ts, user.mfa.ts, user.apiKeys.ts, user.mcp.ts, +// user.account.ts, user.export.ts). Anything used by more than one of them +// lives here, and user.service.ts re-exports the whole public surface so +// route/test importers see a single module. + +import { createServerSupabase } from "../../lib/supabase"; + +export type Db = ReturnType; + +export function errorMessage(error: unknown): string { + if (error instanceof Error && error.message) return error.message; + if (error && typeof error === "object") { + const record = error as { + message?: unknown; + details?: unknown; + hint?: unknown; + code?: unknown; + }; + return ( + [record.message, record.details, record.hint, record.code] + .filter( + (value): value is string => + typeof value === "string" && !!value, + ) + .join(" ") || JSON.stringify(error) + ); + } + return String(error); +} diff --git a/backend/src/modules/workflows/workflows.routes.ts b/backend/src/modules/workflows/workflows.routes.ts new file mode 100644 index 000000000..19a99b327 --- /dev/null +++ b/backend/src/modules/workflows/workflows.routes.ts @@ -0,0 +1,279 @@ +// HTTP surface for the workflows module. Handlers parse params/query/body, +// call the service layer in workflows.service.ts, and map its typed results +// onto status codes and JSON responses. + +import { Router, type NextFunction, type Request, type Response } from "express"; +import { requireAuth } from "../../middleware/auth"; +import { createServerSupabase } from "../../lib/supabase"; +import { + listWorkflows, + createWorkflow, + updateWorkflow, + deleteWorkflow, + getWorkflowDetail, + findSystemWorkflow, + withSystemWorkflowAccess, + submitOpenSourceWorkflow, + WORKFLOW_CONTRIBUTIONS_ENABLED, + listHiddenWorkflows, + hideWorkflow, + unhideWorkflow, + listWorkflowShares, + deleteWorkflowShare, + shareWorkflow, + type WorkflowMetadata, +} from "./workflows.service"; + +export const workflowsRouter = Router(); + +type AsyncRoute = (req: Request, res: Response) => Promise; + +function asyncRoute(handler: AsyncRoute) { + return (req: Request, res: Response, next: NextFunction) => { + void handler(req, res).catch(next); + }; +} + +// GET /workflows +workflowsRouter.get("/", requireAuth, asyncRoute(async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { type } = req.query as { type?: string }; + const db = createServerSupabase(); + const workflowType = typeof type === "string" && type ? type : null; + + const result = await listWorkflows(db, { + userId, + userEmail, + type: workflowType, + }); + if (!result.ok) { + return void res.status(500).json({ detail: result.detail }); + } + + res.json(result.data); +})); + +// POST /workflows +workflowsRouter.post("/", requireAuth, asyncRoute(async (req, res) => { + const userId = res.locals.userId as string; + const { + metadata, + skill_md, + columns_config, + } = req.body as { + metadata?: Partial; + skill_md?: string; + columns_config?: unknown; + }; + const title = metadata?.title; + const type = metadata?.type; + if (!title?.trim()) + return void res.status(400).json({ detail: "metadata.title is required" }); + if (type !== "assistant" && type !== "tabular") + return void res + .status(400) + .json({ detail: "metadata.type must be 'assistant' or 'tabular'" }); + + const db = createServerSupabase(); + const result = await createWorkflow(db, { + userId, + title, + type, + skill_md, + columns_config, + metadata, + }); + if (!result.ok) { + return void res.status(500).json({ detail: result.detail }); + } + res.status(201).json(result.workflow); +})); + +async function handleWorkflowUpdate(req: Request, res: Response) { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { workflowId } = req.params; + const db = createServerSupabase(); + + const result = await updateWorkflow(db, { + workflowId, + userId, + userEmail, + body: req.body, + }); + if (!result.ok) { + return void res + .status(404) + .json({ detail: "Workflow not found or not editable" }); + } + res.json(result.body); +} + +// PUT /workflows/:workflowId +workflowsRouter.put("/:workflowId", requireAuth, asyncRoute(handleWorkflowUpdate)); + +// PATCH /workflows/:workflowId +workflowsRouter.patch("/:workflowId", requireAuth, asyncRoute(handleWorkflowUpdate)); + +// DELETE /workflows/:workflowId +workflowsRouter.delete("/:workflowId", requireAuth, asyncRoute(async (req, res) => { + const userId = res.locals.userId as string; + const { workflowId } = req.params; + const systemWorkflow = findSystemWorkflow(workflowId); + if (systemWorkflow) { + return void res.json(withSystemWorkflowAccess(systemWorkflow)); + } + + const db = createServerSupabase(); + const result = await deleteWorkflow(db, userId, workflowId); + if (!result.ok) return void res.status(500).json({ detail: result.detail }); + res.status(204).send(); +})); + +// GET /workflows/hidden +workflowsRouter.get("/hidden", requireAuth, asyncRoute(async (req, res) => { + const userId = res.locals.userId as string; + const db = createServerSupabase(); + const result = await listHiddenWorkflows(db, userId); + if (!result.ok) return void res.status(500).json({ detail: result.detail }); + res.json(result.ids); +})); + +// POST /workflows/hidden +workflowsRouter.post("/hidden", requireAuth, asyncRoute(async (req, res) => { + const userId = res.locals.userId as string; + const { workflow_id } = req.body as { workflow_id: string }; + if (!workflow_id?.trim()) + return void res.status(400).json({ detail: "workflow_id is required" }); + const db = createServerSupabase(); + const result = await hideWorkflow(db, userId, workflow_id); + if (!result.ok) return void res.status(500).json({ detail: result.detail }); + res.status(204).send(); +})); + +// DELETE /workflows/hidden/:workflowId +workflowsRouter.delete("/hidden/:workflowId", requireAuth, asyncRoute(async (req, res) => { + const userId = res.locals.userId as string; + const { workflowId } = req.params; + const db = createServerSupabase(); + const result = await unhideWorkflow(db, userId, workflowId); + if (!result.ok) return void res.status(500).json({ detail: result.detail }); + res.status(204).send(); +})); + +// POST /workflows/:workflowId/open-source +workflowsRouter.post("/:workflowId/open-source", requireAuth, asyncRoute(async (req, res) => { + if (!WORKFLOW_CONTRIBUTIONS_ENABLED) { + return void res.status(404).json({ detail: "Workflow contributions are disabled" }); + } + + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { workflowId } = req.params; + const openSourceBody = req.body as { + contributor_mode?: unknown; + contributor?: unknown; + }; + const db = createServerSupabase(); + + const result = await submitOpenSourceWorkflow(db, { + workflowId, + userId, + userEmail, + body: openSourceBody, + }); + if (!result.ok) { + if (result.kind === "not_found") { + return void res + .status(404) + .json({ detail: "Workflow not found or not open-sourceable" }); + } + if (result.kind === "validation") { + return void res.status(400).json({ detail: result.detail }); + } + return void res.status(500).json({ detail: result.detail }); + } + + res.status(result.status).json(result.body); +})); + +// GET /workflows/:workflowId +workflowsRouter.get("/:workflowId", requireAuth, asyncRoute(async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { workflowId } = req.params; + const systemWorkflow = findSystemWorkflow(workflowId); + if (systemWorkflow) { + return void res.json(withSystemWorkflowAccess(systemWorkflow)); + } + + const db = createServerSupabase(); + const result = await getWorkflowDetail(db, { workflowId, userId, userEmail }); + if (!result.ok) + return void res.status(404).json({ detail: "Workflow not found" }); + res.json(result.body); +})); + +// GET /workflows/:workflowId/shares +workflowsRouter.get("/:workflowId/shares", requireAuth, asyncRoute(async (req, res) => { + const userId = res.locals.userId as string; + const { workflowId } = req.params; + const db = createServerSupabase(); + + const result = await listWorkflowShares(db, { workflowId, userId }); + if (!result.ok) { + if (result.kind === "not_found") + return void res.status(404).json({ detail: "Workflow not found or not editable" }); + return void res.status(500).json({ detail: result.detail }); + } + + res.json(result.shares); +})); + +// DELETE /workflows/:workflowId/shares/:shareId +workflowsRouter.delete("/:workflowId/shares/:shareId", requireAuth, asyncRoute(async (req, res) => { + const userId = res.locals.userId as string; + const { workflowId, shareId } = req.params; + const db = createServerSupabase(); + + const result = await deleteWorkflowShare(db, { workflowId, shareId, userId }); + if (!result.ok) return void res.status(404).json({ detail: "Workflow not found" }); + res.status(204).send(); +})); + +// POST /workflows/:workflowId/share +workflowsRouter.post("/:workflowId/share", requireAuth, asyncRoute(async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { workflowId } = req.params; + const { emails, allow_edit } = req.body as { emails: string[]; allow_edit: boolean }; + + if (!emails?.length) return void res.status(400).json({ detail: "emails is required" }); + + const db = createServerSupabase(); + const result = await shareWorkflow(db, { + workflowId, + userId, + userEmail, + emails, + allow_edit, + }); + if (!result.ok) { + if (result.kind === "not_found") + return void res.status(404).json({ detail: "Workflow not found or not editable" }); + if (result.kind === "db_error") + return void res.status(500).json({ detail: result.detail }); + return void res.status(400).json({ detail: result.detail }); + } + + res.status(204).send(); +})); + +workflowsRouter.use( + (err: unknown, _req: Request, res: Response, next: NextFunction) => { + if (res.headersSent) return next(err); + console.error("[workflows] unhandled route error", err); + res.status(500).json({ detail: "Failed to process workflow request" }); + }, +); diff --git a/backend/src/routes/workflows.ts b/backend/src/modules/workflows/workflows.service.ts similarity index 63% rename from backend/src/routes/workflows.ts rename to backend/src/modules/workflows/workflows.service.ts index 6dbeda6ea..6be78228d 100644 --- a/backend/src/routes/workflows.ts +++ b/backend/src/modules/workflows/workflows.service.ts @@ -1,15 +1,19 @@ -import { Router, type NextFunction, type Request, type Response } from "express"; -import { requireAuth } from "../middleware/auth"; -import { createServerSupabase } from "../lib/supabase"; +// Business logic + data access for the workflows module. +// +// These functions take an explicit Supabase client (`db`) plus +// request-derived primitives, perform the workflow / share / hidden-list +// orchestration, and RETURN typed results. They never touch req/res — the +// thin route handlers in workflows.routes.ts map the results onto HTTP +// status codes and response bodies. + +import { createServerSupabase } from "../../lib/supabase"; import { SYSTEM_WORKFLOW_IDS, SYSTEM_WORKFLOWS, type SystemWorkflow, -} from "../lib/systemWorkflows"; -import { findMissingUserEmails } from "../lib/userLookup"; -import { workflowNameFromSkillMd } from "../lib/workflowName"; - -export const workflowsRouter = Router(); +} from "../../lib/systemWorkflows"; +import { findMissingUserEmails } from "../../lib/userLookup"; +import { workflowNameFromSkillMd } from "../../lib/workflowName"; type Db = ReturnType; const isDev = process.env.NODE_ENV !== "production"; @@ -17,7 +21,7 @@ const devLog = (...args: Parameters) => { if (isDev) console.log(...args); }; -type WorkflowRecord = { +export type WorkflowRecord = { id: string; user_id: string | null; is_system?: boolean; @@ -33,16 +37,16 @@ type WorkflowRecord = { [key: string]: unknown; }; -type WorkflowType = "assistant" | "tabular"; +export type WorkflowType = "assistant" | "tabular"; -type WorkflowContributor = { +export type WorkflowContributor = { name: string; organisation: string | null; role: string | null; linkedin: string | null; }; -type WorkflowMetadata = { +export type WorkflowMetadata = { name: string | null; title: string; description: string | null; @@ -53,9 +57,9 @@ type WorkflowMetadata = { practice: string | null; jurisdictions: string[] | null; }; -type OpenSourceSubmissionStatus = "pending" | "approved" | "rejected"; +export type OpenSourceSubmissionStatus = "pending" | "approved" | "rejected"; -type OpenSourceSubmissionRow = { +export type OpenSourceSubmissionRow = { id: string; workflow_id: string; submitted_by_user_id: string; @@ -70,7 +74,7 @@ type OpenSourceSubmissionRow = { review_notes?: string | null; }; -type OpenSourceSubmissionSummary = Pick< +export type OpenSourceSubmissionSummary = Pick< OpenSourceSubmissionRow, "id" | "status" | "submitted_at" | "updated_at" > & { @@ -86,10 +90,10 @@ const DEFAULT_WORKFLOW_CONTRIBUTOR: WorkflowContributor = { const DEFAULT_WORKFLOW_LANGUAGE = "English"; const DEFAULT_WORKFLOW_PRACTICE = "General Transactions"; const DEFAULT_WORKFLOW_JURISDICTIONS = ["General"]; -const WORKFLOW_CONTRIBUTIONS_ENABLED = +export const WORKFLOW_CONTRIBUTIONS_ENABLED = process.env.WORKFLOW_CONTRIBUTIONS_ENABLED === "true"; -type WorkflowAccess = +export type WorkflowAccess = | { workflow: WorkflowRecord; allowEdit: boolean; @@ -97,14 +101,6 @@ type WorkflowAccess = } | null; -type AsyncRoute = (req: Request, res: Response) => Promise; - -function asyncRoute(handler: AsyncRoute) { - return (req: Request, res: Response, next: NextFunction) => { - void handler(req, res).catch(next); - }; -} - function withWorkflowAccess( workflow: T, access: { allowEdit: boolean; isOwner: boolean; sharedByName?: string | null }, @@ -127,13 +123,19 @@ function withOpenSourceSubmission( }; } -function withSystemWorkflowAccess(workflow: SystemWorkflow) { +export function withSystemWorkflowAccess(workflow: SystemWorkflow) { return withWorkflowAccess(workflow, { allowEdit: false, isOwner: false, }); } +export function findSystemWorkflow( + workflowId: string, +): SystemWorkflow | undefined { + return SYSTEM_WORKFLOWS.find((workflow) => workflow.id === workflowId); +} + function workflowTypeFrom(value: unknown): WorkflowType { return value === "tabular" ? "tabular" : "assistant"; } @@ -217,10 +219,10 @@ function contributorFromName(name: unknown): WorkflowContributor { } async function resolveWorkflowAccess( + db: Db, workflowId: string, userId: string, userEmail: string | null | undefined, - db: Db, ): Promise { const { data: workflow } = await db .from("workflows") @@ -247,82 +249,18 @@ async function resolveWorkflowAccess( return { workflow: workflowRecord, allowEdit: !!share.allow_edit, isOwner: false }; } -function toOpenSourceSubmissionSummary( - row: OpenSourceSubmissionRow, -): OpenSourceSubmissionSummary { - return { - id: row.id, - status: row.status, - submitted_at: row.submitted_at, - updated_at: row.updated_at, - reviewed_at: row.reviewed_at ?? null, - }; -} - -async function getLatestOpenSourceSubmission( +export async function listWorkflows( db: Db, - workflowId: string, - userId: string, -): Promise { - const { data, error } = await db - .from("workflow_open_source_submissions") - .select("id, status, submitted_at, updated_at, reviewed_at") - .eq("workflow_id", workflowId) - .eq("submitted_by_user_id", userId) - .order("submitted_at", { ascending: false }) - .limit(1) - .maybeSingle(); - if (error) throw error; - return data ? toOpenSourceSubmissionSummary(data as OpenSourceSubmissionRow) : null; -} - -function buildOpenSourceSnapshot( - workflow: WorkflowRecord, - contributors: WorkflowContributor[], - contributorMode: "named" | "anonymous", -) { - return { - workflow_id: workflow.id, - metadata: { - ...metadataFromWorkflowRecord(workflow), - contributors, - }, - skill_md: workflow.prompt_md ?? null, - columns_config: workflow.columns_config ?? null, - contributor_mode: contributorMode, - created_at: workflow.created_at ?? null, - }; -} - -function validateOpenSourceWorkflow(workflow: WorkflowRecord): string | null { - if (workflow.type === "assistant") { - return typeof workflow.prompt_md === "string" && workflow.prompt_md.trim() - ? null - : "Assistant workflows need instructions before they can be opened source."; - } - if (workflow.type === "tabular") { - return Array.isArray(workflow.columns_config) && workflow.columns_config.length > 0 - ? null - : "Tabular workflows need at least one column before they can be opened source."; - } - return "Workflow type must be 'assistant' or 'tabular'."; -} - -// GET /workflows -workflowsRouter.get("/", requireAuth, asyncRoute(async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { type } = req.query as { type?: string }; - const db = createServerSupabase(); - const workflowType = typeof type === "string" && type ? type : null; - + params: { userId: string; userEmail: string | undefined; type: string | null }, +): Promise<{ ok: true; data: unknown } | { ok: false; detail: string }> { + const { userId, userEmail, type: workflowType } = params; const { data, error } = await db.rpc("get_workflows_overview", { p_user_id: userId, p_user_email: userEmail ?? null, p_type: workflowType, }); if (error) { - return void res.status(500).json({ detail: error.message }); + return { ok: false, detail: error.message }; } const systemWorkflows = SYSTEM_WORKFLOWS.filter( @@ -332,31 +270,24 @@ workflowsRouter.get("/", requireAuth, asyncRoute(async (req, res) => { (workflow) => !SYSTEM_WORKFLOW_IDS.has(workflow.id), ).map(withDatabaseWorkflow); - res.json([...systemWorkflows, ...databaseWorkflows]); -})); + return { ok: true, data: [...systemWorkflows, ...databaseWorkflows] }; +} -// POST /workflows -workflowsRouter.post("/", requireAuth, asyncRoute(async (req, res) => { - const userId = res.locals.userId as string; - const { - metadata, - skill_md, - columns_config, - } = req.body as { - metadata?: Partial; +export async function createWorkflow( + db: Db, + params: { + userId: string; + title: string; + type: WorkflowType; skill_md?: string; columns_config?: unknown; - }; - const title = metadata?.title; - const type = metadata?.type; - if (!title?.trim()) - return void res.status(400).json({ detail: "metadata.title is required" }); - if (type !== "assistant" && type !== "tabular") - return void res - .status(400) - .json({ detail: "metadata.type must be 'assistant' or 'tabular'" }); - - const db = createServerSupabase(); + metadata?: Partial; + }, +): Promise< + | { ok: true; workflow: Record } + | { ok: false; detail: string } +> { + const { userId, title, type, skill_md, columns_config, metadata } = params; devLog("[workflows/create] request", { userId, title: title.trim(), @@ -398,7 +329,7 @@ workflowsRouter.post("/", requireAuth, asyncRoute(async (req, res) => { details: error.details, hint: error.hint, }); - return void res.status(500).json({ detail: error.message }); + return { ok: false, detail: error.message }; } devLog("[workflows/create] inserted", { id: data?.id, @@ -406,19 +337,33 @@ workflowsRouter.post("/", requireAuth, asyncRoute(async (req, res) => { title: data?.title, type: data?.type, }); - res.status(201).json(withDatabaseWorkflow(data as WorkflowRecord)); -})); + return { ok: true, workflow: withDatabaseWorkflow(data as WorkflowRecord) }; +} + +export type UpdateWorkflowResult = + | { ok: true; body: Record } + | { ok: false; kind: "not_editable" }; -async function handleWorkflowUpdate(req: Request, res: Response) { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { workflowId } = req.params; +export async function updateWorkflow( + db: Db, + params: { + workflowId: string; + userId: string; + userEmail: string | undefined; + body: { + metadata?: Partial; + skill_md?: unknown; + columns_config?: unknown; + }; + }, +): Promise { + const { workflowId, userId, userEmail, body } = params; const updates: Record = {}; - const metadata = req.body.metadata as Partial | undefined; + const metadata = body.metadata; if (metadata?.title != null) updates.title = metadata.title; - if (req.body.skill_md != null) updates.prompt_md = req.body.skill_md; - if (req.body.columns_config != null) - updates.columns_config = req.body.columns_config; + if (body.skill_md != null) updates.prompt_md = body.skill_md; + if (body.columns_config != null) + updates.columns_config = body.columns_config; if (metadata && "language" in metadata) updates.language = normalizeOptionalString(metadata.language); if (metadata && "practice" in metadata) @@ -426,12 +371,9 @@ async function handleWorkflowUpdate(req: Request, res: Response) { if (metadata && "jurisdictions" in metadata) updates.jurisdictions = normalizeJurisdictions(metadata.jurisdictions); - const db = createServerSupabase(); - const access = await resolveWorkflowAccess(workflowId, userId, userEmail, db); + const access = await resolveWorkflowAccess(db, workflowId, userId, userEmail); if (!access || !access.allowEdit) { - return void res - .status(404) - .json({ detail: "Workflow not found or not editable" }); + return { ok: false, kind: "not_editable" }; } const { data, error } = await db .from("workflows") @@ -439,103 +381,137 @@ async function handleWorkflowUpdate(req: Request, res: Response) { .eq("id", workflowId) .select("*") .single(); - if (error || !data) - return void res - .status(404) - .json({ detail: "Workflow not found or not editable" }); - res.json( - withWorkflowAccess(withDatabaseWorkflow(data as WorkflowRecord), { + if (error || !data) return { ok: false, kind: "not_editable" }; + return { + ok: true, + body: withWorkflowAccess(withDatabaseWorkflow(data as WorkflowRecord), { allowEdit: access.allowEdit, isOwner: access.isOwner, }), - ); + }; } -// PUT /workflows/:workflowId -workflowsRouter.put("/:workflowId", requireAuth, asyncRoute(handleWorkflowUpdate)); - -// PATCH /workflows/:workflowId -workflowsRouter.patch("/:workflowId", requireAuth, asyncRoute(handleWorkflowUpdate)); - -// DELETE /workflows/:workflowId -workflowsRouter.delete("/:workflowId", requireAuth, asyncRoute(async (req, res) => { - const userId = res.locals.userId as string; - const { workflowId } = req.params; - const systemWorkflow = SYSTEM_WORKFLOWS.find( - (workflow) => workflow.id === workflowId, - ); - if (systemWorkflow) { - return void res.json(withSystemWorkflowAccess(systemWorkflow)); - } - - const db = createServerSupabase(); +export async function deleteWorkflow( + db: Db, + userId: string, + workflowId: string, +): Promise<{ ok: true } | { ok: false; detail: string }> { const { error } = await db .from("workflows") .delete() .eq("id", workflowId) .eq("user_id", userId); - if (error) return void res.status(500).json({ detail: error.message }); - res.status(204).send(); -})); - -// GET /workflows/hidden -workflowsRouter.get("/hidden", requireAuth, asyncRoute(async (req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); + if (error) return { ok: false, detail: error.message }; + return { ok: true }; +} + +export async function getWorkflowDetail( + db: Db, + params: { workflowId: string; userId: string; userEmail: string | undefined }, +): Promise<{ ok: true; body: Record } | { ok: false }> { + const { workflowId, userId, userEmail } = params; + const access = await resolveWorkflowAccess(db, workflowId, userId, userEmail); + if (!access) return { ok: false }; + const openSourceSubmission = access.isOwner + ? await getLatestOpenSourceSubmission(db, workflowId, userId) + : null; + return { + ok: true, + body: withOpenSourceSubmission( + withWorkflowAccess(withDatabaseWorkflow(access.workflow), { + allowEdit: access.allowEdit, + isOwner: access.isOwner, + }), + openSourceSubmission, + ), + }; +} + +function toOpenSourceSubmissionSummary( + row: OpenSourceSubmissionRow, +): OpenSourceSubmissionSummary { + return { + id: row.id, + status: row.status, + submitted_at: row.submitted_at, + updated_at: row.updated_at, + reviewed_at: row.reviewed_at ?? null, + }; +} + +async function getLatestOpenSourceSubmission( + db: Db, + workflowId: string, + userId: string, +): Promise { const { data, error } = await db - .from("hidden_workflows") - .select("workflow_id") - .eq("user_id", userId); - if (error) return void res.status(500).json({ detail: error.message }); - res.json((data ?? []).map((r) => r.workflow_id)); -})); - -// POST /workflows/hidden -workflowsRouter.post("/hidden", requireAuth, asyncRoute(async (req, res) => { - const userId = res.locals.userId as string; - const { workflow_id } = req.body as { workflow_id: string }; - if (!workflow_id?.trim()) - return void res.status(400).json({ detail: "workflow_id is required" }); - const db = createServerSupabase(); - const { error } = await db - .from("hidden_workflows") - .upsert({ user_id: userId, workflow_id }, { onConflict: "user_id,workflow_id" }); - if (error) return void res.status(500).json({ detail: error.message }); - res.status(204).send(); -})); - -// DELETE /workflows/hidden/:workflowId -workflowsRouter.delete("/hidden/:workflowId", requireAuth, asyncRoute(async (req, res) => { - const userId = res.locals.userId as string; - const { workflowId } = req.params; - const db = createServerSupabase(); - const { error } = await db - .from("hidden_workflows") - .delete() - .eq("user_id", userId) - .eq("workflow_id", workflowId); - if (error) return void res.status(500).json({ detail: error.message }); - res.status(204).send(); -})); - -// POST /workflows/:workflowId/open-source -workflowsRouter.post("/:workflowId/open-source", requireAuth, asyncRoute(async (req, res) => { - if (!WORKFLOW_CONTRIBUTIONS_ENABLED) { - return void res.status(404).json({ detail: "Workflow contributions are disabled" }); - } + .from("workflow_open_source_submissions") + .select("id, status, submitted_at, updated_at, reviewed_at") + .eq("workflow_id", workflowId) + .eq("submitted_by_user_id", userId) + .order("submitted_at", { ascending: false }) + .limit(1) + .maybeSingle(); + if (error) throw error; + return data ? toOpenSourceSubmissionSummary(data as OpenSourceSubmissionRow) : null; +} - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { workflowId } = req.params; - const openSourceBody = req.body as { - contributor_mode?: unknown; - contributor?: unknown; +function buildOpenSourceSnapshot( + workflow: WorkflowRecord, + contributors: WorkflowContributor[], + contributorMode: "named" | "anonymous", +) { + return { + workflow_id: workflow.id, + metadata: { + ...metadataFromWorkflowRecord(workflow), + contributors, + }, + skill_md: workflow.prompt_md ?? null, + columns_config: workflow.columns_config ?? null, + contributor_mode: contributorMode, + created_at: workflow.created_at ?? null, }; +} + +function validateOpenSourceWorkflow(workflow: WorkflowRecord): string | null { + if (workflow.type === "assistant") { + return typeof workflow.prompt_md === "string" && workflow.prompt_md.trim() + ? null + : "Assistant workflows need instructions before they can be opened source."; + } + if (workflow.type === "tabular") { + return Array.isArray(workflow.columns_config) && workflow.columns_config.length > 0 + ? null + : "Tabular workflows need at least one column before they can be opened source."; + } + return "Workflow type must be 'assistant' or 'tabular'."; +} + +export type SubmitOpenSourceWorkflowResult = + | { + ok: true; + status: number; + body: OpenSourceSubmissionSummary & { mode: "created" | "updated" }; + } + | { ok: false; kind: "not_found" } + | { ok: false; kind: "validation"; detail: string } + | { ok: false; kind: "db_error"; detail: string }; + +export async function submitOpenSourceWorkflow( + db: Db, + params: { + workflowId: string; + userId: string; + userEmail: string | undefined; + body: { contributor_mode?: unknown; contributor?: unknown }; + }, +): Promise { + const { workflowId, userId, userEmail, body: openSourceBody } = params; const requestedContributorMode = openSourceBody.contributor_mode === "named" ? "named" : "anonymous"; - const db = createServerSupabase(); const { data: workflow, error: workflowError } = await db .from("workflows") @@ -544,18 +520,16 @@ workflowsRouter.post("/:workflowId/open-source", requireAuth, asyncRoute(async ( .eq("user_id", userId) .maybeSingle(); if (workflowError) { - return void res.status(500).json({ detail: workflowError.message }); + return { ok: false, kind: "db_error", detail: workflowError.message }; } if (!workflow) { - return void res - .status(404) - .json({ detail: "Workflow not found or not open-sourceable" }); + return { ok: false, kind: "not_found" }; } const workflowRecord = workflow as WorkflowRecord; const validationError = validateOpenSourceWorkflow(workflowRecord); if (validationError) { - return void res.status(400).json({ detail: validationError }); + return { ok: false, kind: "validation", detail: validationError }; } const { data: profile } = await db @@ -589,7 +563,7 @@ workflowsRouter.post("/:workflowId/open-source", requireAuth, asyncRoute(async ( .eq("status", "pending") .maybeSingle(); if (pendingError) { - return void res.status(500).json({ detail: pendingError.message }); + return { ok: false, kind: "db_error", detail: pendingError.message }; } if (pendingSubmission) { @@ -607,14 +581,20 @@ workflowsRouter.post("/:workflowId/open-source", requireAuth, asyncRoute(async ( .select("id, status, submitted_at, updated_at, reviewed_at") .single(); if (updateError || !updated) { - return void res.status(500).json({ + return { + ok: false, + kind: "db_error", detail: updateError?.message ?? "Failed to update submission", - }); + }; } - return void res.json({ - ...toOpenSourceSubmissionSummary(updated as OpenSourceSubmissionRow), - mode: "updated", - }); + return { + ok: true, + status: 200, + body: { + ...toOpenSourceSubmissionSummary(updated as OpenSourceSubmissionRow), + mode: "updated", + }, + }; } const { data: created, error: createError } = await db @@ -634,52 +614,74 @@ workflowsRouter.post("/:workflowId/open-source", requireAuth, asyncRoute(async ( .select("id, status, submitted_at, updated_at, reviewed_at") .single(); if (createError || !created) { - return void res.status(500).json({ + return { + ok: false, + kind: "db_error", detail: createError?.message ?? "Failed to create submission", - }); + }; } - res.status(201).json({ - ...toOpenSourceSubmissionSummary(created as OpenSourceSubmissionRow), - mode: "created", - }); -})); - -// GET /workflows/:workflowId -workflowsRouter.get("/:workflowId", requireAuth, asyncRoute(async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { workflowId } = req.params; - const systemWorkflow = SYSTEM_WORKFLOWS.find( - (workflow) => workflow.id === workflowId, - ); - if (systemWorkflow) { - return void res.json(withSystemWorkflowAccess(systemWorkflow)); - } + return { + ok: true, + status: 201, + body: { + ...toOpenSourceSubmissionSummary(created as OpenSourceSubmissionRow), + mode: "created", + }, + }; +} - const db = createServerSupabase(); - const access = await resolveWorkflowAccess(workflowId, userId, userEmail, db); - if (!access) - return void res.status(404).json({ detail: "Workflow not found" }); - const openSourceSubmission = access.isOwner - ? await getLatestOpenSourceSubmission(db, workflowId, userId) - : null; - res.json( - withOpenSourceSubmission( - withWorkflowAccess(withDatabaseWorkflow(access.workflow), { - allowEdit: access.allowEdit, - isOwner: access.isOwner, - }), - openSourceSubmission, - ), - ); -})); +export async function listHiddenWorkflows( + db: Db, + userId: string, +): Promise<{ ok: true; ids: unknown[] } | { ok: false; detail: string }> { + const { data, error } = await db + .from("hidden_workflows") + .select("workflow_id") + .eq("user_id", userId); + if (error) return { ok: false, detail: error.message }; + return { ok: true, ids: (data ?? []).map((r) => r.workflow_id) }; +} + +export async function hideWorkflow( + db: Db, + userId: string, + workflowId: string, +): Promise<{ ok: true } | { ok: false; detail: string }> { + const { error } = await db + .from("hidden_workflows") + .upsert( + { user_id: userId, workflow_id: workflowId }, + { onConflict: "user_id,workflow_id" }, + ); + if (error) return { ok: false, detail: error.message }; + return { ok: true }; +} + +export async function unhideWorkflow( + db: Db, + userId: string, + workflowId: string, +): Promise<{ ok: true } | { ok: false; detail: string }> { + const { error } = await db + .from("hidden_workflows") + .delete() + .eq("user_id", userId) + .eq("workflow_id", workflowId); + if (error) return { ok: false, detail: error.message }; + return { ok: true }; +} + +export type ListSharesResult = + | { ok: true; shares: unknown[] } + | { ok: false; kind: "not_found" } + | { ok: false; kind: "db_error"; detail: string }; -// GET /workflows/:workflowId/shares -workflowsRouter.get("/:workflowId/shares", requireAuth, asyncRoute(async (req, res) => { - const userId = res.locals.userId as string; - const { workflowId } = req.params; - const db = createServerSupabase(); +export async function listWorkflowShares( + db: Db, + params: { workflowId: string; userId: string }, +): Promise { + const { workflowId, userId } = params; const { data: wf } = await db .from("workflows") @@ -687,23 +689,23 @@ workflowsRouter.get("/:workflowId/shares", requireAuth, asyncRoute(async (req, r .eq("id", workflowId) .eq("user_id", userId) .single(); - if (!wf) return void res.status(404).json({ detail: "Workflow not found or not editable" }); + if (!wf) return { ok: false, kind: "not_found" }; const { data: shares, error } = await db .from("workflow_shares") .select("id, shared_with_email, allow_edit, created_at") .eq("workflow_id", workflowId) .order("created_at", { ascending: true }); - if (error) return void res.status(500).json({ detail: error.message }); + if (error) return { ok: false, kind: "db_error", detail: error.message }; - res.json(shares ?? []); -})); + return { ok: true, shares: shares ?? [] }; +} -// DELETE /workflows/:workflowId/shares/:shareId -workflowsRouter.delete("/:workflowId/shares/:shareId", requireAuth, asyncRoute(async (req, res) => { - const userId = res.locals.userId as string; - const { workflowId, shareId } = req.params; - const db = createServerSupabase(); +export async function deleteWorkflowShare( + db: Db, + params: { workflowId: string; shareId: string; userId: string }, +): Promise<{ ok: true } | { ok: false; kind: "not_found" }> { + const { workflowId, shareId, userId } = params; const { data: wf } = await db .from("workflows") @@ -711,20 +713,34 @@ workflowsRouter.delete("/:workflowId/shares/:shareId", requireAuth, asyncRoute(a .eq("id", workflowId) .eq("user_id", userId) .single(); - if (!wf) return void res.status(404).json({ detail: "Workflow not found" }); + if (!wf) return { ok: false, kind: "not_found" }; await db.from("workflow_shares").delete().eq("id", shareId).eq("workflow_id", workflowId); - res.status(204).send(); -})); + return { ok: true }; +} -// POST /workflows/:workflowId/share -workflowsRouter.post("/:workflowId/share", requireAuth, asyncRoute(async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { workflowId } = req.params; - const { emails, allow_edit } = req.body as { emails: string[]; allow_edit: boolean }; +export type ShareWorkflowResult = + | { ok: true } + | { + ok: false; + kind: "validation" | "self_share" | "missing_user"; + detail: string; + } + | { ok: false; kind: "not_found" } + | { ok: false; kind: "db_error"; detail: string }; + +export async function shareWorkflow( + db: Db, + params: { + workflowId: string; + userId: string; + userEmail: string | undefined; + emails: string[]; + allow_edit: boolean | undefined; + }, +): Promise { + const { workflowId, userId, userEmail, emails, allow_edit } = params; - if (!emails?.length) return void res.status(400).json({ detail: "emails is required" }); const normalizedEmails = [ ...new Set( emails @@ -733,21 +749,24 @@ workflowsRouter.post("/:workflowId/share", requireAuth, asyncRoute(async (req, r ), ]; if (normalizedEmails.length === 0) { - return void res.status(400).json({ detail: "emails is required" }); + return { ok: false, kind: "validation", detail: "emails is required" }; } const normalizedUserEmail = userEmail?.trim().toLowerCase(); if (normalizedUserEmail && normalizedEmails.includes(normalizedUserEmail)) { - return void res - .status(400) - .json({ detail: "You cannot share a workflow with yourself." }); + return { + ok: false, + kind: "self_share", + detail: "You cannot share a workflow with yourself.", + }; } - const db = createServerSupabase(); const missingSharedUsers = await findMissingUserEmails(db, normalizedEmails); if (missingSharedUsers.length > 0) { - return void res.status(400).json({ + return { + ok: false, + kind: "missing_user", detail: `${missingSharedUsers[0]} does not belong to a Mike user.`, - }); + }; } // Verify ownership @@ -757,7 +776,7 @@ workflowsRouter.post("/:workflowId/share", requireAuth, asyncRoute(async (req, r .eq("id", workflowId) .eq("user_id", userId) .single(); - if (!wf) return void res.status(404).json({ detail: "Workflow not found or not editable" }); + if (!wf) return { ok: false, kind: "not_found" }; const rows = normalizedEmails.map((email: string) => ({ workflow_id: workflowId, @@ -770,15 +789,7 @@ workflowsRouter.post("/:workflowId/share", requireAuth, asyncRoute(async (req, r const { error } = await db .from("workflow_shares") .upsert(rows, { onConflict: "workflow_id,shared_with_email" }); - if (error) return void res.status(500).json({ detail: error.message }); + if (error) return { ok: false, kind: "db_error", detail: error.message }; - res.status(204).send(); -})); - -workflowsRouter.use( - (err: unknown, _req: Request, res: Response, next: NextFunction) => { - if (res.headersSent) return next(err); - console.error("[workflows] unhandled route error", err); - res.status(500).json({ detail: "Failed to process workflow request" }); - }, -); + return { ok: true }; +} diff --git a/backend/src/routes/chat.ts b/backend/src/routes/chat.ts deleted file mode 100644 index f4dd76de3..000000000 --- a/backend/src/routes/chat.ts +++ /dev/null @@ -1,669 +0,0 @@ -import { Router } from "express"; -import { requireAuth } from "../middleware/auth"; -import { createServerSupabase } from "../lib/supabase"; -import { - buildDocContext, - buildMessages, - enrichWithPriorEvents, - buildWorkflowStore, - appendAskInputsResponseToLastAssistantMessage, - appendAssistantEventsToLastAssistantMessage, - AssistantStreamError, - buildCancelledAssistantMessage, - extractCitations, - generateSpotlightNonce, - isAbortError, - runLLMStream, - stripTransientAssistantEvents, - parseChatMessages, - parseOptionalAskInputsResponse, - parseOptionalChatId, - parseOptionalModel, - parseOptionalProjectId, -} from "../lib/chat"; -import { completeText } from "../lib/llm"; -import { - getUserModelSettings, -} from "../lib/userSettings"; -import { checkProjectAccess } from "../lib/access"; -import { safeErrorLog, safeErrorMessage } from "../lib/safeError"; - -export const chatRouter = Router(); - -type Db = ReturnType; -const isDev = process.env.NODE_ENV !== "production"; -const devLog = (...args: Parameters) => { - if (isDev) console.log(...args); -}; - -const TITLE_FALLBACK = "Misc. Query"; - -function normalizeGeneratedTitle(raw: string): string { - const title = raw.trim().replace(/^["'`]+|["'`.,:;!?]+$/g, "").trim(); - if (!title) return TITLE_FALLBACK; - return title.slice(0, 80); -} - -type AccessibleChat = { - id: string; - title: string | null; - user_id: string; - project_id: string | null; -} & Record; - -async function validateAccessibleProjectId( - projectId: string | null, - userId: string, - userEmail: string | null | undefined, - db: Db, -): Promise<{ ok: true } | { ok: false; status: number; detail: string }> { - if (!projectId) return { ok: true }; - const access = await checkProjectAccess(projectId, userId, userEmail, db); - if (!access.ok) - return { ok: false, status: 404, detail: "Project not found" }; - return { ok: true }; -} - -async function getAccessibleChat( - chatId: string, - userId: string, - userEmail: string | null | undefined, - db: Db, -): Promise { - const { data: chat, error } = await db - .from("chats") - .select("*") - .eq("id", chatId) - .maybeSingle(); - if (error || !chat) return null; - - const row = chat as AccessibleChat; - if (row.user_id === userId) return row; - - if (row.project_id) { - const access = await checkProjectAccess( - row.project_id, - userId, - userEmail, - db, - ); - if (access.ok) return row; - } - - return null; -} - -// GET /chat -// Visible chats = the user's own chats + every chat under a project the -// user owns (so a project owner sees all collaborator chats in their -// own projects in the global recent-chats list). Chats in projects that -// are merely *shared with* the user are NOT included here — those are -// listed per-project via GET /projects/:projectId/chats. -chatRouter.get("/", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - const requestedLimit = Number.parseInt(String(req.query.limit ?? ""), 10); - const limit = Number.isFinite(requestedLimit) - ? Math.min(Math.max(requestedLimit, 1), 100) - : null; - - const { data, error } = await db.rpc("get_chats_overview", { - p_user_id: userId, - p_limit: limit, - }); - if (error) return void res.status(500).json({ detail: error.message }); - res.json(data ?? []); -}); - -// POST /chat/create -chatRouter.post("/create", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const parsedProjectId = parseOptionalProjectId(req.body?.project_id); - if (!parsedProjectId.ok) { - return void res.status(400).json({ detail: parsedProjectId.detail }); - } - const projectId = parsedProjectId.value.projectId; - const db = createServerSupabase(); - const projectAccess = await validateAccessibleProjectId( - projectId, - userId, - userEmail, - db, - ); - if (!projectAccess.ok) - return void res - .status(projectAccess.status) - .json({ detail: projectAccess.detail }); - - const { data, error } = await db - .from("chats") - .insert({ user_id: userId, project_id: projectId ?? null }) - .select("id") - .single(); - - if (error) return void res.status(500).json({ detail: error.message }); - res.json({ id: data.id }); -}); - -// GET /chat/:chatId -chatRouter.get("/:chatId", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { chatId } = req.params; - const db = createServerSupabase(); - - const chat = await getAccessibleChat(chatId, userId, userEmail, db); - if (!chat) - return void res.status(404).json({ detail: "Chat not found" }); - - const { data: messages } = await db - .from("chat_messages") - .select("*") - .eq("chat_id", chatId) - .order("created_at", { ascending: true }); - - const hydrated = await hydrateEditStatuses(messages ?? [], db); - res.json({ chat, messages: hydrated }); -}); - -// Stored doc_edited events capture the `status` at the time the assistant -// produced the edit (always "pending"). If the user later accepts or rejects, -// `document_edits.status` is updated but the stored event is not. On chat load -// we merge the current DB status in so EditCards render with the real state. -async function hydrateEditStatuses( - messages: Record[], - db: ReturnType, -): Promise[]> { - const editIds = new Set(); - const versionIds = new Set(); - const collectFromAnnList = (list: unknown) => { - if (!Array.isArray(list)) return; - for (const a of list as Record[]) { - if (typeof a?.edit_id === "string") editIds.add(a.edit_id); - if (typeof a?.version_id === "string") - versionIds.add(a.version_id); - } - }; - for (const m of messages) { - const content = m.content; - if (Array.isArray(content)) { - for (const ev of content as Record[]) { - if (ev?.type === "doc_edited") { - collectFromAnnList(ev.annotations); - if (typeof ev.version_id === "string") - versionIds.add(ev.version_id); - } - } - } - } - if (editIds.size === 0 && versionIds.size === 0) return messages; - - // Edit status patch. - const statusById = new Map(); - if (editIds.size > 0) { - const { data: rows } = await db - .from("document_edits") - .select("id, status") - .in("id", Array.from(editIds)); - for (const r of (rows ?? []) as { id: string; status: string }[]) { - if ( - r.status === "pending" || - r.status === "accepted" || - r.status === "rejected" - ) { - statusById.set(r.id, r.status); - } - } - } - - // Version-number patch — old stored events don't carry `version_number` - // because they predate the schema change. Look it up from - // document_versions so the UI can render "V3" chips + download filenames. - const versionNumberById = new Map(); - if (versionIds.size > 0) { - const { data: vrows } = await db - .from("document_versions") - .select("id, version_number") - .in("id", Array.from(versionIds)); - for (const r of (vrows ?? []) as { - id: string; - version_number: number | null; - }[]) { - versionNumberById.set(r.id, r.version_number ?? null); - } - } - - const patchAnnList = (list: unknown): unknown => { - if (!Array.isArray(list)) return list; - return (list as Record[]).map((a) => { - let next = a; - if (typeof a?.edit_id === "string" && statusById.has(a.edit_id)) { - next = { ...next, status: statusById.get(a.edit_id) }; - } - if ( - typeof a?.version_id === "string" && - versionNumberById.has(a.version_id) - ) { - next = { - ...next, - version_number: versionNumberById.get(a.version_id) ?? null, - }; - } - return next; - }); - }; - return messages.map((m) => { - const next: Record = { ...m }; - if (Array.isArray(m.content)) { - next.content = (m.content as Record[]).map( - (ev) => { - if (ev?.type !== "doc_edited") return ev; - let patched: Record = { - ...ev, - annotations: patchAnnList(ev.annotations), - }; - if ( - typeof ev.version_id === "string" && - versionNumberById.has(ev.version_id) - ) { - patched = { - ...patched, - version_number: - versionNumberById.get(ev.version_id) ?? null, - }; - } - return patched; - }, - ); - } - return next; - }); -} - -// PATCH /chat/:chatId -chatRouter.patch("/:chatId", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const { chatId } = req.params; - const title = (req.body.title ?? "").trim(); - if (!title) - return void res.status(400).json({ detail: "title is required" }); - - const db = createServerSupabase(); - const { data, error } = await db - .from("chats") - .update({ title }) - .eq("id", chatId) - .eq("user_id", userId) - .select("id, title") - .single(); - - if (error || !data) - return void res.status(404).json({ detail: "Chat not found" }); - res.json(data); -}); - -// DELETE /chat/:chatId -chatRouter.delete("/:chatId", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const { chatId } = req.params; - const db = createServerSupabase(); - const { error } = await db - .from("chats") - .delete() - .eq("id", chatId) - .eq("user_id", userId); - - if (error) return void res.status(500).json({ detail: error.message }); - res.status(204).send(); -}); - -// POST /chat/:chatId/generate-title -chatRouter.post("/:chatId/generate-title", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { chatId } = req.params; - const message = - typeof req.body?.message === "string" ? req.body.message.trim() : ""; - if (!message) - return void res.status(400).json({ detail: "message is required" }); - - const db = createServerSupabase(); - const chat = await getAccessibleChat(chatId, userId, userEmail, db); - if (!chat) - return void res.status(404).json({ detail: "Chat not found" }); - - try { - const { title_model, api_keys } = await getUserModelSettings( - userId, - db, - ); - const titleText = await completeText({ - model: title_model, - user: `Generate a concise title (3–6 words) for a chat in an AI Legal Platform that starts with this message. The title should describe the topic or document — do NOT include words like "Legal Assistant", "AI", "Chat", or any similar prefix. If there is not enough information to generate a title, return exactly "${TITLE_FALLBACK}". Return only the title, no quotes or punctuation.\n\nMessage: ${message.slice(0, 500)}`, - maxTokens: 64, - apiKeys: api_keys, - }); - const title = normalizeGeneratedTitle(titleText); - - await db - .from("chats") - .update({ title }) - .eq("id", chatId); - - res.json({ title }); - } catch (err) { - console.error("[generate-title]", safeErrorLog(err)); - res.status(500).json({ detail: "Failed to generate title" }); - } -}); - -// POST /chat — streaming -chatRouter.post("/", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const body = - req.body && typeof req.body === "object" && !Array.isArray(req.body) - ? (req.body as Record) - : {}; - const parsedMessages = parseChatMessages(body.messages); - if (!parsedMessages.ok) { - return void res.status(400).json({ detail: parsedMessages.detail }); - } - const parsedChatId = parseOptionalChatId(body.chat_id); - if (!parsedChatId.ok) { - return void res.status(400).json({ detail: parsedChatId.detail }); - } - const parsedProjectId = parseOptionalProjectId(body.project_id); - if (!parsedProjectId.ok) { - return void res.status(400).json({ detail: parsedProjectId.detail }); - } - const parsedModel = parseOptionalModel(body.model); - if (!parsedModel.ok) { - return void res.status(400).json({ detail: parsedModel.detail }); - } - const parsedAskInputsResponse = parseOptionalAskInputsResponse( - body.ask_inputs_response, - ); - if (!parsedAskInputsResponse.ok) { - return void res - .status(400) - .json({ detail: parsedAskInputsResponse.detail }); - } - - const messages = parsedMessages.value; - const chat_id = parsedChatId.value; - const project_id = parsedProjectId.value.projectId; - const model = parsedModel.value; - const askInputsResponse = parsedAskInputsResponse.value; - - devLog("[chat/stream] incoming request", { - userId, - chat_id, - project_id, - model, - messageCount: messages?.length, - }); - - const userEmail = res.locals.userEmail as string | undefined; - const db = createServerSupabase(); - let chatId = chat_id ?? null; - let chatTitle: string | null = null; - let resolvedProjectId: string | null = parsedProjectId.value.projectId; - - if (chatId) { - const existing = await getAccessibleChat(chatId, userId, userEmail, db); - if (!existing) - return void res.status(404).json({ detail: "Chat not found" }); - - const existingProjectId = existing.project_id ?? null; - if ( - parsedProjectId.value.provided && - parsedProjectId.value.projectId !== existingProjectId - ) { - return void res - .status(400) - .json({ detail: "project_id does not match chat" }); - } - resolvedProjectId = existingProjectId; - chatTitle = existing.title; - } - - if (!chatId) { - // If creating a chat tied to a project, the user must have access - // to the project (own or shared). - const projectAccess = await validateAccessibleProjectId( - resolvedProjectId, - userId, - userEmail, - db, - ); - if (!projectAccess.ok) - return void res - .status(projectAccess.status) - .json({ detail: projectAccess.detail }); - - const { data: newChat, error } = await db - .from("chats") - .insert({ user_id: userId, project_id: resolvedProjectId }) - .select("id, title") - .single(); - if (error || !newChat) { - console.error("[chat/stream] failed to create chat", error); - return void res - .status(500) - .json({ detail: "Failed to create chat" }); - } - chatId = newChat.id as string; - chatTitle = newChat.title; - } - - devLog("[chat/stream] resolved chatId", chatId); - - const lastUser = [...messages].reverse().find((m) => m.role === "user"); - if (askInputsResponse) { - await appendAskInputsResponseToLastAssistantMessage( - db, - chatId, - askInputsResponse, - ); - } else if (lastUser) { - await db.from("chat_messages").insert({ - chat_id: chatId, - role: "user", - content: lastUser.content, - files: lastUser.files ?? null, - workflow: lastUser.workflow ?? null, - }); - } - - const { docIndex, docStore } = await buildDocContext( - messages, - userId, - db, - chatId, - ); - const docAvailability = Object.entries(docIndex).map(([doc_id, info]) => ({ - doc_id, - filename: info.filename, - })); - // Generate the nonce before enriching prior events so document filenames - // and workflow titles replayed from earlier turns are fenced as well. - const nonce = generateSpotlightNonce(); - const enrichedMessages = await enrichWithPriorEvents( - messages, - chatId, - db, - docIndex, - nonce, - ); - const { - api_keys: apiKeys, - legal_research_us: legalResearchUs, - } = await getUserModelSettings(userId, db); - const apiMessages = buildMessages( - enrichedMessages, - docAvailability, - undefined, - undefined, - legalResearchUs, - nonce, - ); - - const workflowStore = await buildWorkflowStore(userId, userEmail, db); - - devLog("[chat/stream] starting LLM stream", { - apiMessageCount: apiMessages.length, - docCount: Object.keys(docIndex).length, - workflowCount: Object.keys(workflowStore).length, - }); - - res.setHeader("Content-Type", "text/event-stream"); - res.setHeader("Cache-Control", "no-cache"); - res.setHeader("Connection", "keep-alive"); - res.setHeader("X-Accel-Buffering", "no"); - res.flushHeaders(); - - const write = (line: string) => res.write(line); - const streamAbort = new AbortController(); - let streamFinished = false; - res.on("close", () => { - if (!streamFinished) streamAbort.abort(); - }); - - try { - write(`data: ${JSON.stringify({ type: "chat_id", chatId })}\n\n`); - - const { fullText, events, citations } = await runLLMStream({ - apiMessages, - docStore, - docIndex, - userId, - db, - write, - workflowStore, - includeResearchTools: legalResearchUs, - model, - apiKeys, - signal: streamAbort.signal, - projectId: resolvedProjectId, - nonce, - }); - - devLog("[chat/stream] LLM stream finished", { - fullTextLen: fullText?.length ?? 0, - eventCount: events?.length ?? 0, - }); - - const persistedEvents = stripTransientAssistantEvents(events); - if (askInputsResponse) { - await appendAssistantEventsToLastAssistantMessage( - db, - chatId, - persistedEvents, - citations, - ); - } else { - await db.from("chat_messages").insert({ - chat_id: chatId, - role: "assistant", - content: persistedEvents.length ? persistedEvents : null, - citations: citations.length ? citations : null, - }); - } - - if (!chatTitle && lastUser?.content) { - await db - .from("chats") - .update({ title: lastUser.content.slice(0, 120) }) - .eq("id", chatId); - } - } catch (err) { - if (isAbortError(err)) { - devLog("[chat/stream] client aborted stream", { chatId }); - if (err instanceof AssistantStreamError) { - const partial = buildCancelledAssistantMessage({ - fullText: err.fullText, - events: err.events, - buildCitations: (fullText, events) => - extractCitations(fullText, docIndex, events), - }); - const saveError = askInputsResponse - ? null - : ( - await db.from("chat_messages").insert({ - chat_id: chatId, - role: "assistant", - content: partial.events.length - ? partial.events - : null, - citations: partial.citations.length - ? partial.citations - : null, - }) - ).error; - if (askInputsResponse) { - await appendAssistantEventsToLastAssistantMessage( - db, - chatId, - partial.events, - partial.citations, - ); - } - if (saveError) { - console.error( - "[chat/stream] failed to save aborted stream", - saveError, - ); - } - } - return; - } - console.error("[chat/stream] error:", safeErrorLog(err)); - const message = safeErrorMessage(err, "Stream error"); - const errorEvents = err instanceof AssistantStreamError - ? stripTransientAssistantEvents(err.events) - : [{ type: "error" as const, message }]; - const errorFullText = - err instanceof AssistantStreamError ? err.fullText : ""; - try { - const citations = extractCitations( - errorFullText, - docIndex, - errorEvents, - ); - const saveError = askInputsResponse - ? null - : ( - await db.from("chat_messages").insert({ - chat_id: chatId, - role: "assistant", - content: errorEvents.length ? errorEvents : null, - citations: citations.length ? citations : null, - }) - ).error; - if (askInputsResponse) { - await appendAssistantEventsToLastAssistantMessage( - db, - chatId, - errorEvents, - citations, - ); - } - if (saveError) - console.error("[chat/stream] failed to save error", saveError); - } catch (saveErr) { - console.error("[chat/stream] failed to save error", saveErr); - } - try { - write( - `data: ${JSON.stringify({ type: "error", message })}\n\n`, - ); - write("data: [DONE]\n\n"); - } catch { - /* ignore */ - } - } finally { - streamFinished = true; - res.end(); - } -}); diff --git a/backend/src/routes/documents.ts b/backend/src/routes/documents.ts deleted file mode 100644 index 5ae607969..000000000 --- a/backend/src/routes/documents.ts +++ /dev/null @@ -1,1595 +0,0 @@ -import { Router } from "express"; -import { requireAuth } from "../middleware/auth"; -import { createServerSupabase } from "../lib/supabase"; -import { - buildContentDisposition, - downloadFile, - deleteFile, - getSignedUrl, - storageKey, - uploadFile, - versionStorageKey, -} from "../lib/storage"; -import { docxToPdf, convertedPdfKey } from "../lib/convert"; -import { enqueueConversion } from "../lib/queue/conversionQueue"; -import { - extractTrackedChangeIds, - resolveTrackedChange, -} from "../lib/docxTrackedChanges"; -import { buildDownloadUrl } from "../lib/downloadTokens"; -import { - attachActiveVersionPaths, - attachLatestVersionNumbers, - contentSha256, - loadActiveVersion, -} from "../lib/documentVersions"; -import { ensureDocAccess } from "../lib/access"; -import { singleFileUpload } from "../lib/upload"; -import { - ALLOWED_DOCUMENT_TYPES, - ALLOWED_DOCUMENT_TYPES_LABEL, - contentTypeForDocumentType, - shouldConvertToPdf, -} from "../lib/documentTypes"; - -export const documentsRouter = Router(); -const isDev = process.env.NODE_ENV !== "production"; -const devLog = (...args: Parameters) => { - if (isDev) console.log(...args); -}; - -async function deleteDocumentAndVersionFiles( - db: ReturnType, - documentId: string, -) { - // Storage lives on document_versions — fan out and delete each version's - // bytes (source + PDF rendition) before dropping the document row. - const { data: versions } = await db - .from("document_versions") - .select("storage_path, pdf_storage_path") - .eq("document_id", documentId); - await Promise.all( - (versions ?? []).flatMap((v) => - [v.storage_path, v.pdf_storage_path] - .filter((p): p is string => typeof p === "string" && p.length > 0) - .map((p) => deleteFile(p).catch(() => {})), - ), - ); - return db.from("documents").delete().eq("id", documentId); -} - -// GET /single-documents -documentsRouter.get("/", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - const { data, error } = await db - .from("documents") - .select("*") - .eq("user_id", userId) - .is("project_id", null) - .or("library_kind.eq.file,library_kind.is.null") - .order("created_at", { ascending: false }); - if (error) return void res.status(500).json({ detail: error.message }); - const docs = (data ?? []) as unknown as { - id: string; - current_version_id?: string | null; - }[]; - await attachLatestVersionNumbers(db, docs); - await attachActiveVersionPaths(db, docs); - res.json(docs); -}); - -// GET /single-documents/:documentId -// One document, same shape as a list entry. Exists so the client can poll a -// single document's status while a deferred conversion runs, instead of -// refetching the whole collection. -documentsRouter.get("/:documentId", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { documentId } = req.params; - const db = createServerSupabase(); - - const { data: doc } = await db - .from("documents") - .select("*") - .eq("id", documentId) - .single(); - if (!doc) return void res.status(404).json({ detail: "Document not found" }); - const access = await ensureDocAccess(doc, userId, userEmail, db); - if (!access.ok) - return void res.status(404).json({ detail: "Document not found" }); - - const docs = [doc] as unknown as { - id: string; - current_version_id?: string | null; - }[]; - await attachLatestVersionNumbers(db, docs); - await attachActiveVersionPaths(db, docs); - res.json(docs[0]); -}); - -// POST /single-documents -documentsRouter.post( - "/", - requireAuth, - singleFileUpload("file"), - async (req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - await handleDocumentUpload(req, res, userId, null, db, { - libraryKind: "file", - }); - }, -); - -// DELETE /single-documents/:documentId -documentsRouter.delete("/:documentId", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const { documentId } = req.params; - const db = createServerSupabase(); - - const { data: doc, error } = await db - .from("documents") - .select("id") - .eq("id", documentId) - .eq("user_id", userId) - .single(); - if (error || !doc) - return void res.status(404).json({ detail: "Document not found" }); - - await deleteDocumentAndVersionFiles(db, documentId); - res.status(204).send(); -}); - -// GET /single-documents/:documentId/display -// Optional ?version_id= renders a historical version. Defaults to the -// document's current_version_id. -documentsRouter.get("/:documentId/display", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string; - const { documentId } = req.params; - const versionIdParam = - typeof req.query.version_id === "string" ? req.query.version_id : null; - const db = createServerSupabase(); - - const { data: doc } = await db - .from("documents") - .select("id, user_id, project_id") - .eq("id", documentId) - .single(); - if (!doc) - return void res.status(404).json({ detail: "Document not found" }); - const access = await ensureDocAccess(doc, userId, userEmail, db); - if (!access.ok) - return void res.status(404).json({ detail: "Document not found" }); - - const active = await loadActiveVersion(documentId, db, versionIdParam); - if (!active) - return void res.status(404).json({ detail: "No file available" }); - - const fileType = active.file_type ?? ""; - const isConvertibleOffice = shouldConvertToPdf(fileType); - const displayFilename = downloadFilenameForVersion( - active.filename, - active.version_number, - active.source === "assistant_edit", - ); - - // For Office files, prefer the per-version PDF rendition if one exists. - const servePath = - isConvertibleOffice && active.pdf_storage_path - ? active.pdf_storage_path - : active.storage_path; - const raw = await downloadFile(servePath); - if (!raw) - return void res - .status(404) - .json({ detail: "Document not found in storage" }); - - if (fileType === "pdf" || (isConvertibleOffice && active.pdf_storage_path)) { - res.setHeader("Content-Type", "application/pdf"); - res.setHeader( - "Content-Disposition", - buildContentDisposition("inline", displayFilename), - ); - res.send(Buffer.from(raw)); - } else { - // Fallback: serve raw Office bytes when PDF conversion was unavailable. - res.setHeader("Content-Type", contentTypeForDocumentType(fileType)); - res.setHeader( - "Content-Disposition", - buildContentDisposition("inline", displayFilename), - ); - res.send(Buffer.from(raw)); - } -}); - -// POST /single-documents/download-zip -documentsRouter.post("/download-zip", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { document_ids } = req.body as { document_ids?: string[] }; - - if (!Array.isArray(document_ids) || document_ids.length === 0) - return void res.status(400).json({ detail: "document_ids is required" }); - - const db = createServerSupabase(); - const { data: rawDocs, error } = await db - .from("documents") - .select("id, current_version_id, user_id, project_id") - .in("id", document_ids); - - if (error) return void res.status(500).json({ detail: error.message }); - // Filter to docs the user actually has access to (own + shared-project). - const accessChecks = await Promise.all( - (rawDocs ?? []).map(async (d) => ({ - doc: d, - access: await ensureDocAccess( - d as { user_id: string; project_id: string | null }, - userId, - userEmail, - db, - ), - })), - ); - const docs = accessChecks - .filter((x) => x.access.ok) - .map((x) => x.doc as { id: string }); - if (!docs || docs.length === 0) - return void res.status(404).json({ detail: "No documents found" }); - - const JSZip = (await import("jszip")).default; - const zip = new JSZip(); - - await Promise.all( - docs.map(async (doc) => { - const active = await loadActiveVersion(doc.id, db); - if (!active) return; - const raw = await downloadFile(active.storage_path); - if (!raw) return; - zip.file( - downloadFilenameForVersion( - active.filename, - active.version_number, - active.source === "assistant_edit", - ), - Buffer.from(raw), - ); - }), - ); - - const content = await zip.generateAsync({ type: "nodebuffer", compression: "DEFLATE" }); - res.setHeader("Content-Type", "application/zip"); - res.setHeader("Content-Disposition", 'attachment; filename="documents.zip"'); - res.send(content); -}); - -// GET /single-documents/:documentId/url -// Optional ?version_id= selects a specific tracked-changes version. -// Otherwise falls back to documents.current_version_id, else the original upload. -documentsRouter.get("/:documentId/url", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { documentId } = req.params; - const versionIdParam = typeof req.query.version_id === "string" ? req.query.version_id : null; - const db = createServerSupabase(); - - const { data: doc, error } = await db - .from("documents") - .select("id, user_id, project_id") - .eq("id", documentId) - .single(); - if (error || !doc) - return void res.status(404).json({ detail: "Document not found" }); - const access = await ensureDocAccess(doc, userId, userEmail, db); - if (!access.ok) - return void res.status(404).json({ detail: "Document not found" }); - - const active = await loadActiveVersion(documentId, db, versionIdParam); - if (!active) - return void res.status(404).json({ detail: "No file available" }); - - const downloadFilename = downloadFilenameForVersion( - active.filename, - active.version_number, - active.source === "assistant_edit", - ); - const url = await getSignedUrl( - active.storage_path, - 3600, - downloadFilename, - ); - if (!url) - return void res.status(503).json({ detail: "Storage not configured" }); - - res.json({ - url, - document_id: documentId, - filename: downloadFilename, - version_id: active.id, - // Lets the frontend decide between DocView (PDF.js) and DocxView - // (docx-preview) without a follow-up round-trip. - has_pdf_rendition: !!active.pdf_storage_path, - }); -}); - -// GET /single-documents/:documentId/docx -// Streams the raw .docx bytes for the given document, optionally at a -// specific tracked-changes version. Unlike /url, this bypasses R2 (avoids -// the browser CORS problem on signed URLs) so the frontend docx-preview -// viewer can load tracked-change documents directly. -documentsRouter.get("/:documentId/docx", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { documentId } = req.params; - const versionIdParam = typeof req.query.version_id === "string" ? req.query.version_id : null; - const db = createServerSupabase(); - - const { data: doc, error } = await db - .from("documents") - .select("id, user_id, project_id") - .eq("id", documentId) - .single(); - if (error || !doc) - return void res.status(404).json({ detail: "Document not found" }); - const access = await ensureDocAccess(doc, userId, userEmail, db); - if (!access.ok) - return void res.status(404).json({ detail: "Document not found" }); - - const active = await loadActiveVersion(documentId, db, versionIdParam); - if (!active) - return void res.status(404).json({ detail: "No file available" }); - - const raw = await downloadFile(active.storage_path); - if (!raw) - return void res.status(404).json({ detail: "Document bytes not available" }); - - res.setHeader( - "Content-Type", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - ); - res.setHeader( - "Content-Disposition", - buildContentDisposition( - "inline", - downloadFilenameForVersion( - active.filename, - active.version_number, - active.source === "assistant_edit", - ), - ), - ); - res.send(Buffer.from(raw)); -}); - -// Produce the filename a download should present to the user. Version -// filenames are expected to include the real extension. -function downloadFilenameForVersion( - filename: string | null | undefined, - versionNumber: number | null, - edited = false, -): string { - const resolved = filename?.trim() || "Untitled document.docx"; - if (!edited || !versionNumber || versionNumber < 1) return resolved; - const dot = resolved.lastIndexOf("."); - const stem = dot > 0 ? resolved.slice(0, dot) : resolved; - const ext = dot > 0 ? resolved.slice(dot) : ""; - return `${stem} [Edited V${versionNumber}]${ext}`; -} - -// GET /single-documents/:documentId/versions -// Returns every version row for the document in document order, with -// the human-friendly version number when present. -documentsRouter.get("/:documentId/versions", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { documentId } = req.params; - const db = createServerSupabase(); - - const { data: doc } = await db - .from("documents") - .select("id, current_version_id, user_id, project_id") - .eq("id", documentId) - .single(); - if (!doc) - return void res.status(404).json({ detail: "Document not found" }); - const access = await ensureDocAccess(doc, userId, userEmail, db); - if (!access.ok) - return void res.status(404).json({ detail: "Document not found" }); - - const { data: rows } = await db - .from("document_versions") - .select( - "id, version_number, source, created_at, filename, file_type, size_bytes, page_count, deleted_at, deleted_by", - ) - .eq("document_id", documentId) - .order("created_at", { ascending: true }); - - res.json({ - current_version_id: doc.current_version_id, - versions: rows ?? [], - }); -}); - -// POST /single-documents/:documentId/versions/from-document -// Create a new version of documentId from another existing document's active -// bytes. This keeps signed storage URLs out of the browser fetch path. -documentsRouter.post( - "/:documentId/versions/from-document", - requireAuth, - async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { documentId } = req.params; - const sourceDocumentId = - typeof req.body?.source_document_id === "string" - ? req.body.source_document_id - : ""; - const db = createServerSupabase(); - - if (!sourceDocumentId) { - return void res - .status(400) - .json({ detail: "source_document_id is required" }); - } - if (sourceDocumentId === documentId) { - return void res - .status(400) - .json({ detail: "Source and target documents must be different." }); - } - - const { data: targetDoc } = await db - .from("documents") - .select("id, user_id, project_id") - .eq("id", documentId) - .single(); - if (!targetDoc) - return void res.status(404).json({ detail: "Document not found" }); - const targetAccess = await ensureDocAccess(targetDoc, userId, userEmail, db); - if (!targetAccess.ok) - return void res.status(404).json({ detail: "Document not found" }); - - const { data: sourceDoc } = await db - .from("documents") - .select("id, user_id, project_id") - .eq("id", sourceDocumentId) - .single(); - if (!sourceDoc) - return void res.status(404).json({ detail: "Source document not found" }); - const sourceAccess = await ensureDocAccess(sourceDoc, userId, userEmail, db); - if (!sourceAccess.ok) - return void res.status(404).json({ detail: "Source document not found" }); - const willDeleteSource = - (sourceDoc.project_id && - targetDoc.project_id && - sourceDoc.project_id === targetDoc.project_id) || - (!sourceDoc.project_id && - !targetDoc.project_id && - sourceDoc.user_id === userId && - targetDoc.user_id === userId); - if (willDeleteSource && !sourceAccess.isOwner) { - return void res.status(403).json({ - detail: "Only the source document owner can move it into a version.", - }); - } - - const active = await loadActiveVersion(sourceDocumentId, db); - if (!active) - return void res - .status(404) - .json({ detail: "Source document has no active version." }); - const sourceType = active.file_type ?? ""; - - const bytes = await downloadFile(active.storage_path); - if (!bytes) - return void res - .status(404) - .json({ detail: "Source document bytes not available." }); - - const filename = - typeof req.body?.filename === "string" && req.body.filename.trim() - ? req.body.filename.trim().slice(0, 200) - : active.filename?.trim() || "Untitled document"; - const suffix = - sourceType || - (filename.includes(".") ? filename.split(".").pop()!.toLowerCase() : ""); - const versionSlug = crypto.randomUUID().replace(/-/g, ""); - const key = versionStorageKey(userId, documentId, versionSlug, filename); - const contentType = contentTypeForDocumentType(suffix); - - try { - await uploadFile(key, bytes, contentType); - } catch (e) { - console.error("[versions/copy] storage write failed", e); - return void res - .status(500) - .json({ detail: "Failed to create new version." }); - } - - let pdfStoragePath: string | null = null; - let deferConversion = false; - if (suffix === "pdf") { - pdfStoragePath = key; - } else if (active.pdf_storage_path) { - if (active.pdf_storage_path === active.storage_path) { - pdfStoragePath = key; - } else { - const pdfBytes = await downloadFile(active.pdf_storage_path); - if (pdfBytes) { - const pdfKey = `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`; - await uploadFile(pdfKey, pdfBytes, "application/pdf"); - pdfStoragePath = pdfKey; - } - } - } else if (shouldConvertToPdf(suffix)) { - // Only reached when the source has no rendition to copy — this is the - // one branch of the copy flow that pays for LibreOffice, so it's the - // branch the conversion queue takes over when the flag is on. - if (process.env.ASYNC_DOCUMENT_CONVERSION === "true") { - deferConversion = true; - } else { - try { - const pdfBuf = await docxToPdf(Buffer.from(bytes)); - const pdfKey = `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`; - await uploadFile( - pdfKey, - pdfBuf.buffer.slice( - pdfBuf.byteOffset, - pdfBuf.byteOffset + pdfBuf.byteLength, - ) as ArrayBuffer, - "application/pdf", - ); - pdfStoragePath = pdfKey; - } catch (err) { - console.error( - `[versions/copy] Office→PDF conversion failed for ${filename}:`, - err, - ); - } - } - } - - const { data: maxRow } = await db - .from("document_versions") - .select("version_number") - .eq("document_id", documentId) - .in("source", ["upload", "user_upload", "assistant_edit"]) - .order("version_number", { ascending: false, nullsFirst: false }) - .limit(1) - .maybeSingle(); - const nextVersionNumber = - ((maxRow?.version_number as number | null) ?? 1) + 1; - - const { data: versionRow, error: verErr } = await db - .from("document_versions") - .insert({ - document_id: documentId, - storage_path: key, - pdf_storage_path: pdfStoragePath, - source: "user_upload", - version_number: nextVersionNumber, - filename: filename, - file_type: sourceType || null, - size_bytes: active.size_bytes ?? bytes.byteLength, - page_count: active.page_count, - content_sha256: contentSha256(bytes), - }) - .select("id, version_number, source, created_at, filename") - .single(); - if (verErr || !versionRow) { - console.error("[versions/copy] insert failed", verErr); - return void res - .status(500) - .json({ detail: "Failed to record new version." }); - } - - const { error: updateDocErr } = await db - .from("documents") - .update({ - current_version_id: versionRow.id, - }) - .eq("id", documentId); - if (updateDocErr) { - console.error("[versions/copy] current version update failed", updateDocErr); - return void res - .status(500) - .json({ detail: "Failed to update document current version." }); - } - - if (deferConversion) { - await enqueueConversion({ - documentId, - versionId: versionRow.id as string, - userId, - storagePath: key, - fileType: suffix, - pdfKey: `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`, - finalizeDocumentStatus: false, - }); - } - - if (willDeleteSource) { - const { error: deleteErr } = await deleteDocumentAndVersionFiles( - db, - sourceDocumentId, - ); - if (deleteErr) { - console.error("[versions/copy] source document delete failed", deleteErr); - return void res - .status(500) - .json({ detail: "Failed to delete source document." }); - } - } - - res.status(201).json(versionRow); - }, -); - -// POST /single-documents/:documentId/versions -// Upload a brand-new version of an existing document. The uploaded file -// becomes the new current_version_id. filename defaults to the -// uploaded filename; client may override via the `filename` form field. -documentsRouter.post( - "/:documentId/versions", - requireAuth, - singleFileUpload("file"), - async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { documentId } = req.params; - const db = createServerSupabase(); - - const file = req.file; - if (!file) - return void res.status(400).json({ detail: "file is required" }); - - const { data: doc } = await db - .from("documents") - .select("id, user_id, project_id, current_version_id") - .eq("id", documentId) - .single(); - if (!doc) - return void res.status(404).json({ detail: "Document not found" }); - const access = await ensureDocAccess(doc, userId, userEmail, db); - if (!access.ok) - return void res.status(404).json({ detail: "Document not found" }); - - const suffix = file.originalname.includes(".") - ? file.originalname.split(".").pop()!.toLowerCase() - : ""; - if (!ALLOWED_DOCUMENT_TYPES.has(suffix)) { - return void res.status(400).json({ - detail: `Unsupported file type: ${suffix}. Allowed: ${ALLOWED_DOCUMENT_TYPES_LABEL}`, - }); - } - - // Peg the new version into a predictable /versions/:id path under the - // existing document folder so ops can spot the history in storage. - const versionSlug = crypto.randomUUID().replace(/-/g, ""); - const key = versionStorageKey( - userId, - documentId, - versionSlug, - file.originalname, - ); - const contentType = contentTypeForDocumentType(suffix); - try { - await uploadFile( - key, - file.buffer.buffer.slice( - file.buffer.byteOffset, - file.buffer.byteOffset + file.buffer.byteLength, - ) as ArrayBuffer, - contentType, - ); - } catch (e) { - console.error("[versions/upload] storage write failed", e); - return void res - .status(500) - .json({ detail: "Failed to upload new version." }); - } - - // Render this version's bytes to PDF up front so /display can show - // historical versions without on-demand conversion. Same logic as the - // initial-upload pipeline; failures don't block the version row. - // With the job queue enabled the LibreOffice work is deferred to the - // conversion worker instead of blocking this request; the version row is - // created with pdf_storage_path null and the worker fills it in. - const deferConversion = - shouldConvertToPdf(suffix) && - process.env.ASYNC_DOCUMENT_CONVERSION === "true"; - let pdfStoragePath: string | null = null; - if (!deferConversion && shouldConvertToPdf(suffix)) { - try { - const pdfBuf = await docxToPdf(file.buffer); - const pdfKey = `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`; - await uploadFile( - pdfKey, - pdfBuf.buffer.slice( - pdfBuf.byteOffset, - pdfBuf.byteOffset + pdfBuf.byteLength, - ) as ArrayBuffer, - "application/pdf", - ); - pdfStoragePath = pdfKey; - } catch (err) { - console.error( - `[versions/upload] Office→PDF conversion failed for ${file.originalname}:`, - err, - ); - } - } else if (suffix === "pdf") { - // For PDF uploads, the uploaded bytes are themselves the PDF rendition. - pdfStoragePath = key; - } - - const rawBuf = file.buffer.buffer.slice( - file.buffer.byteOffset, - file.buffer.byteOffset + file.buffer.byteLength, - ) as ArrayBuffer; - const pageCount = suffix === "pdf" ? await countPdfPages(rawBuf) : null; - - // Per-document sequential version_number — the upload is V1 and - // user_upload + assistant_edit count forward from there. - const { data: maxRow } = await db - .from("document_versions") - .select("version_number") - .eq("document_id", documentId) - .in("source", ["upload", "user_upload", "assistant_edit"]) - .order("version_number", { ascending: false, nullsFirst: false }) - .limit(1) - .maybeSingle(); - const nextVersionNumber = - ((maxRow?.version_number as number | null) ?? 1) + 1; - - const requestedFilename = - typeof req.body?.filename === "string" && - req.body.filename.trim() - ? req.body.filename.trim().slice(0, 200) - : file.originalname; - - const { data: versionRow, error: verErr } = await db - .from("document_versions") - .insert({ - document_id: documentId, - storage_path: key, - pdf_storage_path: pdfStoragePath, - source: "user_upload", - version_number: nextVersionNumber, - filename: requestedFilename, - file_type: suffix, - size_bytes: file.buffer.byteLength, - page_count: pageCount, - content_sha256: contentSha256(file.buffer), - }) - .select("id, version_number, source, created_at, filename") - .single(); - if (verErr || !versionRow) { - console.error("[versions/upload] insert failed", verErr); - return void res - .status(500) - .json({ detail: "Failed to record new version." }); - } - - const { error: updateDocErr } = await db - .from("documents") - .update({ - current_version_id: versionRow.id, - }) - .eq("id", documentId); - if (updateDocErr) { - console.error( - "[versions/upload] current version update failed", - updateDocErr, - ); - return void res - .status(500) - .json({ detail: "Failed to update document current version." }); - } - - if (deferConversion) { - // The document itself stays "ready" — only this version's rendition is - // pending, so the worker must not touch documents.status. - await enqueueConversion({ - documentId, - versionId: versionRow.id as string, - userId, - storagePath: key, - fileType: suffix, - pdfKey: `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`, - finalizeDocumentStatus: false, - }); - } - - res.status(201).json(versionRow); - }, -); - -// PATCH /single-documents/:documentId/versions/:versionId -// Rename a version's filename. Pass `{ "filename": "…" }`. -documentsRouter.patch( - "/:documentId/versions/:versionId", - requireAuth, - async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { documentId, versionId } = req.params; - const db = createServerSupabase(); - - const { data: doc } = await db - .from("documents") - .select("id, user_id, project_id") - .eq("id", documentId) - .single(); - if (!doc) - return void res.status(404).json({ detail: "Document not found" }); - const access = await ensureDocAccess(doc, userId, userEmail, db); - if (!access.ok) - return void res.status(404).json({ detail: "Document not found" }); - - const raw = req.body?.filename; - const filename = - typeof raw === "string" && raw.trim() ? raw.trim().slice(0, 200) : null; - - const { data: updated, error } = await db - .from("document_versions") - .update({ filename }) - .eq("id", versionId) - .eq("document_id", documentId) - .is("deleted_at", null) - .select( - "id, version_number, source, created_at, filename, file_type, size_bytes, page_count", - ) - .single(); - if (error || !updated) { - return void res.status(404).json({ detail: "Version not found" }); - } - res.json(updated); - }, -); - -// PUT /single-documents/:documentId/versions/:versionId/file -// Replace the file bytes and metadata for an existing version while keeping -// its version number and id. This is destructive and owner-only. -documentsRouter.put( - "/:documentId/versions/:versionId/file", - requireAuth, - singleFileUpload("file"), - async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { documentId, versionId } = req.params; - const db = createServerSupabase(); - - const file = req.file; - if (!file) - return void res.status(400).json({ detail: "file is required" }); - - const { data: doc } = await db - .from("documents") - .select("id, user_id, project_id") - .eq("id", documentId) - .single(); - if (!doc) - return void res.status(404).json({ detail: "Document not found" }); - const access = await ensureDocAccess(doc, userId, userEmail, db); - if (!access.ok || !access.isOwner) - return void res.status(404).json({ detail: "Document not found" }); - - const { data: target, error: targetErr } = await db - .from("document_versions") - .select("id, storage_path, pdf_storage_path, file_type, deleted_at") - .eq("id", versionId) - .eq("document_id", documentId) - .single(); - if (targetErr || !target) - return void res.status(404).json({ detail: "Version not found" }); - if (target.deleted_at) - return void res.status(400).json({ detail: "Version is deleted." }); - - const suffix = file.originalname.includes(".") - ? file.originalname.split(".").pop()!.toLowerCase() - : ""; - if (!ALLOWED_DOCUMENT_TYPES.has(suffix)) { - return void res.status(400).json({ - detail: `Unsupported file type: ${suffix}. Allowed: ${ALLOWED_DOCUMENT_TYPES_LABEL}`, - }); - } - if (target.file_type && target.file_type !== suffix) { - return void res.status(400).json({ - detail: `Uploaded file type (${suffix}) does not match version type (${target.file_type}).`, - }); - } - - const versionSlug = crypto.randomUUID().replace(/-/g, ""); - const key = versionStorageKey( - userId, - documentId, - versionSlug, - file.originalname, - ); - const contentType = contentTypeForDocumentType(suffix); - - try { - await uploadFile( - key, - file.buffer.buffer.slice( - file.buffer.byteOffset, - file.buffer.byteOffset + file.buffer.byteLength, - ) as ArrayBuffer, - contentType, - ); - } catch (e) { - console.error("[versions/replace] storage write failed", e); - return void res - .status(500) - .json({ detail: "Failed to upload replacement version." }); - } - - // Same queue deferral as version uploads: the replacement's rendition is - // produced by the conversion worker when the flag is on. The old rendition - // is deleted below either way, so /display briefly falls back until the - // worker writes the new one. - const deferConversion = - shouldConvertToPdf(suffix) && - process.env.ASYNC_DOCUMENT_CONVERSION === "true"; - let pdfStoragePath: string | null = null; - if (!deferConversion && shouldConvertToPdf(suffix)) { - try { - const pdfBuf = await docxToPdf(file.buffer); - const pdfKey = `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`; - await uploadFile( - pdfKey, - pdfBuf.buffer.slice( - pdfBuf.byteOffset, - pdfBuf.byteOffset + pdfBuf.byteLength, - ) as ArrayBuffer, - "application/pdf", - ); - pdfStoragePath = pdfKey; - } catch (err) { - console.error( - `[versions/replace] Office→PDF conversion failed for ${file.originalname}:`, - err, - ); - } - } else if (suffix === "pdf") { - pdfStoragePath = key; - } - - const rawBuf = file.buffer.buffer.slice( - file.buffer.byteOffset, - file.buffer.byteOffset + file.buffer.byteLength, - ) as ArrayBuffer; - const pageCount = suffix === "pdf" ? await countPdfPages(rawBuf) : null; - const requestedFilename = - typeof req.body?.filename === "string" && req.body.filename.trim() - ? req.body.filename.trim().slice(0, 200) - : file.originalname; - const uploadedAt = new Date().toISOString(); - - const { data: updated, error: updateErr } = await db - .from("document_versions") - .update({ - storage_path: key, - pdf_storage_path: pdfStoragePath, - filename: requestedFilename, - file_type: suffix, - size_bytes: file.buffer.byteLength, - page_count: pageCount, - content_sha256: contentSha256(file.buffer), - created_at: uploadedAt, - }) - .eq("id", versionId) - .eq("document_id", documentId) - .select( - "id, version_number, source, created_at, filename, file_type, size_bytes, page_count", - ) - .single(); - if (updateErr || !updated) { - await Promise.all( - [key, pdfStoragePath] - .filter((path): path is string => !!path) - .map((path) => deleteFile(path).catch(() => {})), - ); - return void res.status(500).json({ - detail: updateErr?.message ?? "Failed to replace version.", - }); - } - - await Promise.all( - [target.storage_path, target.pdf_storage_path] - .filter((path): path is string => !!path) - .map((path) => deleteFile(path).catch(() => {})), - ); - - if (deferConversion) { - // Replace reuses the versionId, which is exactly why terminal jobs are - // removed from the queue immediately — this enqueue must not be deduped - // against a completed job for the same version. - await enqueueConversion({ - documentId, - versionId, - userId, - storagePath: key, - fileType: suffix, - pdfKey: `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`, - finalizeDocumentStatus: false, - }); - } - - res.json(updated); - }, -); - -// DELETE /single-documents/:documentId/versions/:versionId -// Delete one version. The last remaining version cannot be deleted; if the -// deleted version is current, the newest remaining version becomes current. -documentsRouter.delete( - "/:documentId/versions/:versionId", - requireAuth, - async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { documentId, versionId } = req.params; - const db = createServerSupabase(); - - const { data: doc } = await db - .from("documents") - .select("id, user_id, project_id, current_version_id") - .eq("id", documentId) - .single(); - if (!doc) - return void res.status(404).json({ detail: "Document not found" }); - const access = await ensureDocAccess(doc, userId, userEmail, db); - if (!access.ok || !access.isOwner) - return void res.status(404).json({ detail: "Document not found" }); - - const { data: versions, error: versionsErr } = await db - .from("document_versions") - .select( - "id, storage_path, pdf_storage_path, version_number, created_at, deleted_at", - ) - .eq("document_id", documentId) - .is("deleted_at", null); - if (versionsErr) { - return void res.status(500).json({ detail: versionsErr.message }); - } - - const rows = (versions ?? []) as { - id: string; - storage_path: string | null; - pdf_storage_path: string | null; - version_number: number | null; - created_at: string | null; - deleted_at?: string | null; - }[]; - const target = rows.find((row) => row.id === versionId); - if (!target) - return void res.status(404).json({ detail: "Version not found" }); - if (rows.length <= 1) { - return void res - .status(400) - .json({ detail: "Cannot delete the only document version." }); - } - - const remaining = rows - .filter((row) => row.id !== versionId) - .sort((a, b) => { - const versionDelta = - (b.version_number ?? -1) - (a.version_number ?? -1); - if (versionDelta !== 0) return versionDelta; - return ( - new Date(b.created_at ?? 0).getTime() - - new Date(a.created_at ?? 0).getTime() - ); - }); - const nextCurrentVersionId = - doc.current_version_id === versionId - ? (remaining[0]?.id ?? null) - : doc.current_version_id; - const deletedAt = new Date().toISOString(); - - if (doc.current_version_id === versionId) { - const { error: updateErr } = await db - .from("documents") - .update({ - current_version_id: nextCurrentVersionId, - updated_at: new Date().toISOString(), - }) - .eq("id", documentId); - if (updateErr) { - return void res.status(500).json({ detail: updateErr.message }); - } - } - - const { error: deleteErr } = await db - .from("document_versions") - .update({ - storage_path: null, - pdf_storage_path: null, - deleted_at: deletedAt, - deleted_by: userId, - }) - .eq("id", versionId) - .eq("document_id", documentId) - .is("deleted_at", null); - if (deleteErr) { - return void res.status(500).json({ detail: deleteErr.message }); - } - - await Promise.all( - [target.storage_path, target.pdf_storage_path] - .filter((path): path is string => !!path) - .map((path) => deleteFile(path).catch(() => {})), - ); - - res.json({ - deleted_version_id: versionId, - current_version_id: nextCurrentVersionId, - deleted_at: deletedAt, - }); - }, -); - -// GET /single-documents/:documentId/tracked-change-ids -// Returns the ordered list of { kind, w_id } for every w:ins / w:del in -// the current (or specified) version's document.xml. The frontend uses -// this to tag each rendered / with data-w-id, since -// docx-preview drops the w:id attribute during parsing. -documentsRouter.get( - "/:documentId/tracked-change-ids", - requireAuth, - async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { documentId } = req.params; - const versionIdParam = - typeof req.query.version_id === "string" ? req.query.version_id : null; - const db = createServerSupabase(); - - const { data: doc } = await db - .from("documents") - .select("id, user_id, project_id") - .eq("id", documentId) - .single(); - if (!doc) - return void res.status(404).json({ detail: "Document not found" }); - const access = await ensureDocAccess(doc, userId, userEmail, db); - if (!access.ok) - return void res.status(404).json({ detail: "Document not found" }); - - const active = await loadActiveVersion(documentId, db, versionIdParam); - if (!active) - return void res.status(404).json({ detail: "No file available" }); - - const raw = await downloadFile(active.storage_path); - if (!raw) - return void res - .status(404) - .json({ detail: "Document bytes not available" }); - - const ids = await extractTrackedChangeIds(Buffer.from(raw)); - res.json({ ids }); - }, -); - -// POST /single-documents/:documentId/edits/:editId/accept -// POST /single-documents/:documentId/edits/:editId/reject -async function handleEditResolution( - req: import("express").Request, - res: import("express").Response, - mode: "accept" | "reject", -) { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { documentId, editId } = req.params; - const db = createServerSupabase(); - - devLog(`[edit-resolution] incoming ${mode}`, { - userId, - documentId, - editId, - }); - - const { data: edit, error: editErr } = await db - .from("document_edits") - .select("id, document_id, change_id, del_w_id, ins_w_id, status") - .eq("id", editId) - .eq("document_id", documentId) - .single(); - devLog(`[edit-resolution] fetched edit row`, { edit, editErr }); - if (!edit) { - devLog(`[edit-resolution] edit not found, returning 404`); - return void res.status(404).json({ detail: "Edit not found" }); - } - // Idempotent: if the edit is already resolved, return the current doc - // state so stale UI (e.g. an old chat reloaded in a new session) can - // reconcile without throwing. - if (edit.status !== "pending") { - devLog(`[edit-resolution] edit already resolved`, { - editId, - status: edit.status, - }); - const { data: doc } = await db - .from("documents") - .select("current_version_id, user_id, project_id") - .eq("id", documentId) - .single(); - if (!doc) { - devLog(`[edit-resolution] doc not found for resolved edit`); - return void res.status(404).json({ detail: "Document not found" }); - } - const accessResolved = await ensureDocAccess(doc, userId, userEmail, db); - if (!accessResolved.ok) { - devLog(`[edit-resolution] doc access denied for resolved edit`); - return void res.status(404).json({ detail: "Document not found" }); - } - const activeForResolved = await loadActiveVersion(documentId, db); - const payload = { - ok: true, - already_resolved: true, - status: edit.status, - version_id: doc.current_version_id ?? null, - download_url: activeForResolved - ? buildDownloadUrl( - activeForResolved.storage_path, - downloadFilenameForVersion( - activeForResolved.filename, - activeForResolved.version_number, - activeForResolved.source === "assistant_edit", - ), - ) - : null, - remaining_pending: 0, - }; - devLog(`[edit-resolution] returning already-resolved payload`, payload); - return void res.status(200).json(payload); - } - - const { data: doc, error: docErr } = await db - .from("documents") - .select("id, current_version_id, user_id, project_id") - .eq("id", documentId) - .single(); - devLog(`[edit-resolution] fetched doc`, { doc, docErr }); - if (!doc) - return void res.status(404).json({ detail: "Document not found" }); - const access = await ensureDocAccess(doc, userId, userEmail, db); - if (!access.ok) - return void res.status(404).json({ detail: "Document not found" }); - - const active = await loadActiveVersion(documentId, db); - const latestPath = active?.storage_path ?? null; - devLog(`[edit-resolution] resolved latestPath`, { - latestPath, - current_version_id: doc.current_version_id, - }); - if (!latestPath) - return void res.status(404).json({ detail: "No file to edit" }); - - const raw = await downloadFile(latestPath); - devLog(`[edit-resolution] downloaded bytes`, { - byteLength: raw?.byteLength ?? 0, - }); - if (!raw) - return void res.status(404).json({ detail: "Document bytes not available" }); - - const wIds = [edit.del_w_id, edit.ins_w_id].filter( - (v): v is string => typeof v === "string" && v.length > 0, - ); - const { bytes: resolvedBytes, found } = await resolveTrackedChange( - Buffer.from(raw), - wIds, - mode, - ); - devLog(`[edit-resolution] resolveTrackedChange result`, { - mode, - change_id: edit.change_id, - wIds, - found, - resolvedByteLength: resolvedBytes?.byteLength ?? 0, - }); - if (!found) { - devLog( - `[edit-resolution] change_id not found in docx — updating status only`, - ); - // Still update DB status so the UI reflects the decision — the change - // may have been auto-consumed by a previous accept/reject pass. - const { error: updErr } = await db - .from("document_edits") - .update({ status: mode === "accept" ? "accepted" : "rejected", resolved_at: new Date().toISOString() }) - .eq("id", editId); - devLog(`[edit-resolution] status-only update`, { updErr }); - const payload = { - ok: true, - version_id: doc.current_version_id, - download_url: buildDownloadUrl( - latestPath, - downloadFilenameForVersion( - active?.filename, - active?.version_number ?? null, - active?.source === "assistant_edit", - ), - ), - remaining_pending: 0, - }; - devLog(`[edit-resolution] returning not-found payload`, payload); - return void res.status(200).json(payload); - } - - // Overwrite bytes in place at the current version's storage path — - // accept/reject mutates the existing version rather than spawning a - // new row. This keeps document_versions lean (one row per assistant - // edit, not one per accept/reject click) and avoids the N-versions- - // per-doc churn as users resolve pending changes. - const ab = resolvedBytes.buffer.slice( - resolvedBytes.byteOffset, - resolvedBytes.byteOffset + resolvedBytes.byteLength, - ) as ArrayBuffer; - - // Clear the hash before the bytes change, and set it again after. The stored - // object and the hash live in different systems, so they cannot be written - // atomically; ordering it this way means a failure in between leaves the - // version unhashed, which the manifest reports as unverifiable. The - // alternative ordering can leave a hash attesting to content the version no - // longer holds, which is the one thing the manifest must never do. - await db - .from("document_versions") - .update({ content_sha256: null }) - .eq("id", doc.current_version_id); - - devLog(`[edit-resolution] overwriting bytes in place`, { - latestPath, - byteLength: ab.byteLength, - }); - await uploadFile( - latestPath, - ab, - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - ); - - await db - .from("document_versions") - .update({ content_sha256: contentSha256(ab) }) - .eq("id", doc.current_version_id); - - const { error: statusErr } = await db - .from("document_edits") - .update({ - status: mode === "accept" ? "accepted" : "rejected", - resolved_at: new Date().toISOString(), - }) - .eq("id", editId); - devLog(`[edit-resolution] updated document_edits status`, { - editId, - newStatus: mode === "accept" ? "accepted" : "rejected", - statusErr, - }); - const { count: remainingPending } = await db - .from("document_edits") - .select("id", { count: "exact", head: true }) - .eq("document_id", documentId) - .eq("status", "pending"); - devLog(`[edit-resolution] remaining pending count`, { remainingPending }); - - const payload = { - ok: true, - version_id: doc.current_version_id, - download_url: buildDownloadUrl( - latestPath, - downloadFilenameForVersion( - active?.filename, - active?.version_number ?? null, - active?.source === "assistant_edit", - ), - ), - remaining_pending: remainingPending ?? 0, - }; - devLog(`[edit-resolution] returning success payload`, payload); - res.json(payload); -} - -documentsRouter.post( - "/:documentId/edits/:editId/accept", - requireAuth, - (req, res) => void handleEditResolution(req, res, "accept"), -); - -documentsRouter.post( - "/:documentId/edits/:editId/reject", - requireAuth, - (req, res) => void handleEditResolution(req, res, "reject"), -); - -export async function handleDocumentUpload( - req: import("express").Request, - res: import("express").Response, - userId: string, - projectId: string | null, - db: ReturnType, - options: { - libraryKind?: "file" | "template"; - libraryFolderId?: string | null; - } = {}, -) { - const file = req.file; - if (!file) return void res.status(400).json({ detail: "file is required" }); - - const filename = file.originalname; - const suffix = filename.includes(".") - ? filename.split(".").pop()!.toLowerCase() - : ""; - if (!ALLOWED_DOCUMENT_TYPES.has(suffix)) - return void res - .status(400) - .json({ - detail: `Unsupported file type: ${suffix}. Allowed: ${ALLOWED_DOCUMENT_TYPES_LABEL}`, - }); - - const content = file.buffer; - const { data: doc, error: insertErr } = await db - .from("documents") - .insert({ - project_id: projectId, - user_id: userId, - status: "processing", - library_kind: options.libraryKind ?? "file", - library_folder_id: options.libraryFolderId ?? null, - }) - .select("*") - .single(); - - if (insertErr || !doc) - console.error("[single-documents/upload] failed to create document row", { - userId, - projectId, - filename, - suffix, - error: insertErr, - }); - if (insertErr || !doc) - return void res - .status(500) - .json({ detail: "Failed to create document record" }); - - try { - const docId = doc.id as string; - const key = storageKey(userId, docId, filename); - const contentType = contentTypeForDocumentType(suffix); - await uploadFile( - key, - content.buffer.slice( - content.byteOffset, - content.byteOffset + content.byteLength, - ) as ArrayBuffer, - contentType, - ); - - const rawBuf = content.buffer.slice( - content.byteOffset, - content.byteOffset + content.byteLength, - ) as ArrayBuffer; - const pageCount = suffix === "pdf" ? await countPdfPages(rawBuf) : null; - - // When the job queue is enabled, defer Office → PDF conversion to the - // BullMQ worker instead of blocking the upload request on LibreOffice. - const deferConversion = - shouldConvertToPdf(suffix) && - process.env.ASYNC_DOCUMENT_CONVERSION === "true"; - - // Convert Office files → PDF for display. PDFs are their own rendition. - let pdfStoragePath: string | null = null; - if (!deferConversion && shouldConvertToPdf(suffix)) { - try { - const pdfBuf = await docxToPdf(content); - const pdfKey = convertedPdfKey(userId, docId); - await uploadFile( - pdfKey, - pdfBuf.buffer.slice( - pdfBuf.byteOffset, - pdfBuf.byteOffset + pdfBuf.byteLength, - ) as ArrayBuffer, - "application/pdf", - ); - pdfStoragePath = pdfKey; - } catch (err) { - console.error( - `[upload] Office→PDF conversion failed for ${filename}:`, - err, - ); - } - } else if (suffix === "pdf") { - pdfStoragePath = key; - } - - // storage_path / pdf_storage_path live on document_versions now — - // create the V1 "upload" row and point documents.current_version_id - // at it. - const { data: versionRow, error: verErr } = await db - .from("document_versions") - .insert({ - document_id: docId, - storage_path: key, - pdf_storage_path: pdfStoragePath, - source: "upload", - version_number: 1, - filename: filename, - file_type: suffix, - size_bytes: content.byteLength, - page_count: pageCount, - content_sha256: contentSha256(content), - }) - .select("id") - .single(); - if (verErr || !versionRow) { - throw new Error( - `Failed to record upload version: ${verErr?.message ?? "unknown"}`, - ); - } - - await db - .from("documents") - .update({ - current_version_id: versionRow.id, - // Deferred conversion leaves the doc "processing" until the worker - // produces the PDF and flips it to "ready". - status: deferConversion ? "processing" : "ready", - updated_at: new Date().toISOString(), - }) - .eq("id", docId); - - if (deferConversion) { - await enqueueConversion({ - documentId: docId, - versionId: versionRow.id, - userId, - storagePath: key, - fileType: suffix, - }); - } - - const { data: updated } = await db - .from("documents") - .select("*") - .eq("id", docId) - .single(); - // Surface storage paths to the caller for backward compatibility. - const responseDoc = updated - ? { - ...updated, - filename, - storage_path: key, - pdf_storage_path: pdfStoragePath, - folder_id: - (updated.library_folder_id as string | null | undefined) ?? null, - file_type: suffix, - size_bytes: content.byteLength, - page_count: pageCount, - active_version_number: 1, - } - : updated; - return void res.status(201).json(responseDoc); - } catch (e) { - await db.from("documents").update({ status: "error" }).eq("id", doc.id); - return void res - .status(500) - .json({ detail: `Document processing failed: ${String(e)}` }); - } -} - -async function countPdfPages(buf: ArrayBuffer): Promise { - try { - const pdfjsLib = await import("pdfjs-dist/legacy/build/pdf.mjs" as string); - const pdf = await ( - pdfjsLib as unknown as { - getDocument: (opts: unknown) => { - promise: Promise<{ numPages: number }>; - }; - } - ).getDocument({ data: new Uint8Array(buf) }).promise; - return pdf.numPages; - } catch { - return null; - } -} diff --git a/backend/src/routes/library.ts b/backend/src/routes/library.ts deleted file mode 100644 index d46bf7174..000000000 --- a/backend/src/routes/library.ts +++ /dev/null @@ -1,414 +0,0 @@ -import { Router } from "express"; -import { requireAuth } from "../middleware/auth"; -import { createServerSupabase } from "../lib/supabase"; -import { deleteFile } from "../lib/storage"; -import { - attachActiveVersionPaths, - attachLatestVersionNumbers, -} from "../lib/documentVersions"; -import { singleFileUpload } from "../lib/upload"; -import { handleDocumentUpload } from "./documents"; - -export const libraryRouter = Router(); - -type LibraryKind = "file" | "template"; - -function normalizeLibraryKind(value: unknown): LibraryKind | null { - if (value === "file" || value === "files") return "file"; - if (value === "template" || value === "templates") return "template"; - return null; -} - -function normalizeDocumentFilename(nextName: unknown, currentName: string) { - if (typeof nextName !== "string") return null; - const trimmed = nextName.trim().slice(0, 200); - if (!trimmed) return null; - if (/\.[a-z0-9]{1,6}$/i.test(trimmed)) return trimmed; - const ext = currentName.match(/\.[a-z0-9]{1,6}$/i)?.[0] ?? ""; - return `${trimmed}${ext}`; -} - -function mapLibraryDocument>(doc: T) { - return { - ...doc, - folder_id: (doc.library_folder_id as string | null | undefined) ?? null, - }; -} - -async function loadLibraryFolder( - db: ReturnType, - userId: string, - kind: LibraryKind, - folderId: string, -): Promise<{ id: string; parent_folder_id: string | null } | null> { - const { data } = await db - .from("library_folders") - .select("id, parent_folder_id") - .eq("id", folderId) - .eq("user_id", userId) - .eq("library_kind", kind) - .maybeSingle(); - return (data as { id: string; parent_folder_id: string | null } | null) ?? null; -} - -async function deleteLibraryDocumentsAndVersionFiles( - db: ReturnType, - userId: string, - kind: LibraryKind, - documentIds: string[], -) { - if (documentIds.length === 0) return null; - const { data: versions, error: versionsError } = await db - .from("document_versions") - .select("storage_path, pdf_storage_path") - .in("document_id", documentIds); - if (versionsError) return versionsError; - - const paths = new Set(); - for (const version of versions ?? []) { - if (typeof version.storage_path === "string" && version.storage_path) { - paths.add(version.storage_path); - } - if ( - typeof version.pdf_storage_path === "string" && - version.pdf_storage_path - ) { - paths.add(version.pdf_storage_path); - } - } - await Promise.all([...paths].map((path) => deleteFile(path).catch(() => {}))); - - let deleteQuery = db - .from("documents") - .delete() - .eq("user_id", userId) - .is("project_id", null); - deleteQuery = - kind === "file" - ? deleteQuery.or("library_kind.eq.file,library_kind.is.null") - : deleteQuery.eq("library_kind", kind); - const { error } = await deleteQuery.in("id", documentIds); - return error ?? null; -} - -// GET /library/:kind -libraryRouter.get("/:kind", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const kind = normalizeLibraryKind(req.params.kind); - if (!kind) return void res.status(404).json({ detail: "Library not found" }); - - const db = createServerSupabase(); - let documentsQuery = db - .from("documents") - .select("*") - .eq("user_id", userId) - .is("project_id", null); - documentsQuery = - kind === "file" - ? documentsQuery.or("library_kind.eq.file,library_kind.is.null") - : documentsQuery.eq("library_kind", kind); - const [{ data: docs, error: docsError }, { data: folders, error: foldersError }] = - await Promise.all([ - documentsQuery.order("created_at", { ascending: true }), - db - .from("library_folders") - .select("*") - .eq("user_id", userId) - .eq("library_kind", kind) - .order("created_at", { ascending: true }), - ]); - if (docsError) return void res.status(500).json({ detail: docsError.message }); - if (foldersError) - return void res.status(500).json({ detail: foldersError.message }); - - const docsTyped = (docs ?? []).map(mapLibraryDocument) as { - id: string; - current_version_id?: string | null; - }[]; - await attachLatestVersionNumbers(db, docsTyped); - await attachActiveVersionPaths(db, docsTyped); - res.json({ documents: docsTyped, folders: folders ?? [] }); -}); - -// POST /library/:kind/documents -libraryRouter.post( - "/:kind/documents", - requireAuth, - singleFileUpload("file"), - async (req, res) => { - const userId = res.locals.userId as string; - const kind = normalizeLibraryKind(req.params.kind); - if (!kind) return void res.status(404).json({ detail: "Library not found" }); - const db = createServerSupabase(); - await handleDocumentUpload(req, res, userId, null, db, { - libraryKind: kind, - }); - }, -); - -// POST /library/:kind/folders -libraryRouter.post("/:kind/folders", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const kind = normalizeLibraryKind(req.params.kind); - if (!kind) return void res.status(404).json({ detail: "Library not found" }); - - const { name, parent_folder_id } = req.body as { - name?: string; - parent_folder_id?: string | null; - }; - if (!name?.trim()) - return void res.status(400).json({ detail: "name is required" }); - - const db = createServerSupabase(); - if (parent_folder_id) { - const parent = await loadLibraryFolder(db, userId, kind, parent_folder_id); - if (!parent) - return void res.status(404).json({ detail: "Parent folder not found" }); - } - - const { data, error } = await db - .from("library_folders") - .insert({ - user_id: userId, - library_kind: kind, - name: name.trim(), - parent_folder_id: parent_folder_id ?? null, - }) - .select("*") - .single(); - if (error) return void res.status(500).json({ detail: error.message }); - res.status(201).json(data); -}); - -// PATCH /library/:kind/folders/:folderId -libraryRouter.patch("/:kind/folders/:folderId", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const kind = normalizeLibraryKind(req.params.kind); - if (!kind) return void res.status(404).json({ detail: "Library not found" }); - - const { folderId } = req.params; - const body = req.body as { name?: string; parent_folder_id?: string | null }; - const db = createServerSupabase(); - const folder = await loadLibraryFolder(db, userId, kind, folderId); - if (!folder) return void res.status(404).json({ detail: "Folder not found" }); - - const updates: Record = { - updated_at: new Date().toISOString(), - }; - if (body.name != null) { - const trimmed = body.name.trim(); - if (!trimmed) - return void res.status(400).json({ detail: "name is required" }); - updates.name = trimmed; - } - if ("parent_folder_id" in body) { - if (body.parent_folder_id) { - let cur: string | null = body.parent_folder_id; - while (cur) { - if (cur === folderId) { - return void res.status(400).json({ - detail: "Cannot move a folder into itself or a descendant", - }); - } - const parent = await loadLibraryFolder(db, userId, kind, cur); - if (!parent) - return void res.status(404).json({ detail: "Parent folder not found" }); - cur = parent.parent_folder_id ?? null; - } - } - updates.parent_folder_id = body.parent_folder_id ?? null; - } - - const { data, error } = await db - .from("library_folders") - .update(updates) - .eq("id", folderId) - .eq("user_id", userId) - .eq("library_kind", kind) - .select("*") - .single(); - if (error || !data) - return void res.status(404).json({ detail: "Folder not found" }); - res.json(data); -}); - -// DELETE /library/:kind/folders/:folderId -libraryRouter.delete("/:kind/folders/:folderId", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const kind = normalizeLibraryKind(req.params.kind); - if (!kind) return void res.status(404).json({ detail: "Library not found" }); - - const { folderId } = req.params; - const db = createServerSupabase(); - const { data: allFolders, error: foldersError } = await db - .from("library_folders") - .select("id, parent_folder_id") - .eq("user_id", userId) - .eq("library_kind", kind); - if (foldersError) - return void res.status(500).json({ detail: foldersError.message }); - if (!(allFolders ?? []).some((folder) => folder.id === folderId)) { - return void res.status(404).json({ detail: "Folder not found" }); - } - - const childrenByParent = new Map(); - for (const folder of allFolders ?? []) { - const parentId = folder.parent_folder_id as string | null; - if (!parentId) continue; - const children = childrenByParent.get(parentId) ?? []; - children.push(folder.id as string); - childrenByParent.set(parentId, children); - } - - const folderIds = new Set(); - const stack = [folderId]; - while (stack.length > 0) { - const id = stack.pop()!; - if (folderIds.has(id)) continue; - folderIds.add(id); - stack.push(...(childrenByParent.get(id) ?? [])); - } - - let documentsInFolderQuery = db - .from("documents") - .select("id") - .eq("user_id", userId) - .is("project_id", null); - documentsInFolderQuery = - kind === "file" - ? documentsInFolderQuery.or("library_kind.eq.file,library_kind.is.null") - : documentsInFolderQuery.eq("library_kind", kind); - const { data: docs, error: docsError } = await documentsInFolderQuery.in( - "library_folder_id", - [...folderIds], - ); - if (docsError) return void res.status(500).json({ detail: docsError.message }); - - const docIds = (docs ?? []).map((doc) => doc.id as string); - const deleteDocsError = await deleteLibraryDocumentsAndVersionFiles( - db, - userId, - kind, - docIds, - ); - if (deleteDocsError) - return void res.status(500).json({ detail: deleteDocsError.message }); - - const { error } = await db - .from("library_folders") - .delete() - .eq("id", folderId) - .eq("user_id", userId) - .eq("library_kind", kind); - if (error) return void res.status(500).json({ detail: error.message }); - res.status(204).send(); -}); - -// PATCH /library/:kind/documents/:documentId/folder -libraryRouter.patch( - "/:kind/documents/:documentId/folder", - requireAuth, - async (req, res) => { - const userId = res.locals.userId as string; - const kind = normalizeLibraryKind(req.params.kind); - if (!kind) return void res.status(404).json({ detail: "Library not found" }); - - const { documentId } = req.params; - const { folder_id } = req.body as { folder_id: string | null }; - const db = createServerSupabase(); - - if (folder_id) { - const folder = await loadLibraryFolder(db, userId, kind, folder_id); - if (!folder) - return void res.status(404).json({ detail: "Folder not found" }); - } - - let moveQuery = db - .from("documents") - .update({ - library_folder_id: folder_id ?? null, - updated_at: new Date().toISOString(), - }) - .eq("id", documentId) - .eq("user_id", userId) - .is("project_id", null); - moveQuery = - kind === "file" - ? moveQuery.or("library_kind.eq.file,library_kind.is.null") - : moveQuery.eq("library_kind", kind); - const { data, error } = await moveQuery - .select("*") - .single(); - if (error || !data) - return void res.status(404).json({ detail: "Document not found" }); - res.json(mapLibraryDocument(data)); - }, -); - -// PATCH /library/:kind/documents/:documentId -libraryRouter.patch( - "/:kind/documents/:documentId", - requireAuth, - async (req, res) => { - const userId = res.locals.userId as string; - const kind = normalizeLibraryKind(req.params.kind); - if (!kind) return void res.status(404).json({ detail: "Library not found" }); - - const { documentId } = req.params; - const db = createServerSupabase(); - let docQuery = db - .from("documents") - .select("id, current_version_id") - .eq("id", documentId) - .eq("user_id", userId) - .is("project_id", null); - docQuery = - kind === "file" - ? docQuery.or("library_kind.eq.file,library_kind.is.null") - : docQuery.eq("library_kind", kind); - const { data: doc } = await docQuery.single(); - if (!doc) return void res.status(404).json({ detail: "Document not found" }); - - const active = doc.current_version_id - ? await db - .from("document_versions") - .select("filename") - .eq("id", doc.current_version_id) - .eq("document_id", documentId) - .single() - : null; - const currentName = - typeof active?.data?.filename === "string" && active.data.filename.trim() - ? active.data.filename.trim() - : "Untitled document"; - const filename = normalizeDocumentFilename(req.body?.filename, currentName); - if (!filename) - return void res.status(400).json({ detail: "filename is required" }); - - let updateQuery = db - .from("documents") - .update({ updated_at: new Date().toISOString() }) - .eq("id", documentId) - .eq("user_id", userId) - .is("project_id", null); - updateQuery = - kind === "file" - ? updateQuery.or("library_kind.eq.file,library_kind.is.null") - : updateQuery.eq("library_kind", kind); - const { data: updated, error } = await updateQuery - .select("*") - .single(); - if (error || !updated) - return void res.status(404).json({ detail: "Document not found" }); - - if (doc.current_version_id) { - await db - .from("document_versions") - .update({ filename }) - .eq("id", doc.current_version_id) - .eq("document_id", documentId); - } - - res.json(mapLibraryDocument({ ...updated, filename })); - }, -); diff --git a/backend/src/routes/projects.ts b/backend/src/routes/projects.ts deleted file mode 100644 index 2fd1eeae1..000000000 --- a/backend/src/routes/projects.ts +++ /dev/null @@ -1,1159 +0,0 @@ -import { Router } from "express"; -import { requireAuth, requireMfaIfEnrolled } from "../middleware/auth"; -import { createServerSupabase } from "../lib/supabase"; -import { enqueueConversion } from "../lib/queue/conversionQueue"; -import { createClient } from "@supabase/supabase-js"; -import { - attachActiveVersionPaths, - attachLatestVersionNumbers, - contentSha256, -} from "../lib/documentVersions"; -import { safeErrorLog } from "../lib/safeError"; -import { - buildProjectExportManifest, - projectManifestFilename, -} from "../lib/userDataExport"; -import { - deleteFile, - downloadFile, - uploadFile, - storageKey, -} from "../lib/storage"; -import { docxToPdf, convertedPdfKey } from "../lib/convert"; -import { checkProjectAccess } from "../lib/access"; -import { singleFileUpload } from "../lib/upload"; -import { deleteUserProjects } from "../lib/userDataCleanup"; -import { - ALLOWED_DOCUMENT_TYPES, - ALLOWED_DOCUMENT_TYPES_LABEL, - contentTypeForDocumentType, - shouldConvertToPdf, -} from "../lib/documentTypes"; -import { - findMissingUserEmails, - loadProfileUsersByEmail, -} from "../lib/userLookup"; - -export const projectsRouter = Router(); - -function normalizeOptionalString(value: unknown) { - if (typeof value !== "string") return null; - const trimmed = value.trim(); - return trimmed.length > 0 ? trimmed : null; -} - -function normalizeDocumentFilename(nextName: unknown, currentName: string) { - if (typeof nextName !== "string") return null; - const trimmed = nextName.trim().slice(0, 200); - if (!trimmed) return null; - if (/\.[a-z0-9]{1,6}$/i.test(trimmed)) return trimmed; - const ext = currentName.match(/\.[a-z0-9]{1,6}$/i)?.[0] ?? ""; - return `${trimmed}${ext}`; -} - -async function deleteProjectDocumentsAndVersionFiles( - db: ReturnType, - projectId: string, - documentIds: string[], -) { - if (documentIds.length === 0) return null; - const { data: versions, error: versionsError } = await db - .from("document_versions") - .select("storage_path, pdf_storage_path") - .in("document_id", documentIds); - if (versionsError) return versionsError; - - const paths = new Set(); - for (const v of versions ?? []) { - if (typeof v.storage_path === "string" && v.storage_path.length > 0) { - paths.add(v.storage_path); - } - if (typeof v.pdf_storage_path === "string" && v.pdf_storage_path.length > 0) { - paths.add(v.pdf_storage_path); - } - } - await Promise.all([...paths].map((p) => deleteFile(p).catch(() => {}))); - - const { error } = await db - .from("documents") - .delete() - .eq("project_id", projectId) - .in("id", documentIds); - return error ?? null; -} - -async function attachDocumentOwnerLabels( - db: ReturnType, - docs: { user_id?: string | null }[], -) { - const ownerIds = docs - .map((doc) => doc.user_id) - .filter((id): id is string => typeof id === "string" && id.length > 0) - .filter((id, index, arr) => arr.indexOf(id) === index); - if (ownerIds.length === 0) return; - - const displayNameByUserId = new Map(); - const { data: profiles, error: profilesError } = await db - .from("user_profiles") - .select("user_id, display_name") - .in("user_id", ownerIds); - if (profilesError) { - console.warn("[projects] failed to load document owner profiles", profilesError); - } - for (const profile of profiles ?? []) { - const displayName = - typeof profile.display_name === "string" - ? profile.display_name.trim() - : ""; - if (displayName) { - displayNameByUserId.set(profile.user_id as string, displayName); - } - } - - for (const doc of docs as ({ - user_id?: string | null; - owner_email?: string | null; - owner_display_name?: string | null; - })[]) { - if (!doc.user_id) continue; - doc.owner_email = null; - doc.owner_display_name = displayNameByUserId.get(doc.user_id) ?? null; - } -} - -async function attachChatCreatorLabels( - db: ReturnType, - chats: { user_id?: string | null }[], -) { - const creatorIds = chats - .map((chat) => chat.user_id) - .filter((id): id is string => typeof id === "string" && id.length > 0) - .filter((id, index, arr) => arr.indexOf(id) === index); - if (creatorIds.length === 0) return; - - const displayNameByUserId = new Map(); - const { data: profiles, error: profilesError } = await db - .from("user_profiles") - .select("user_id, display_name") - .in("user_id", creatorIds); - if (profilesError) { - console.warn("[projects] failed to load chat creator profiles", profilesError); - } - for (const profile of profiles ?? []) { - const displayName = - typeof profile.display_name === "string" - ? profile.display_name.trim() - : ""; - if (displayName) { - displayNameByUserId.set(profile.user_id as string, displayName); - } - } - - for (const chat of chats as ({ - user_id?: string | null; - creator_display_name?: string | null; - })[]) { - if (!chat.user_id) continue; - chat.creator_display_name = displayNameByUserId.get(chat.user_id) ?? null; - } -} - -// GET /projects -// Pass ?include=documents to also receive each project's documents in the -// same response. The directory pickers (useDirectoryData) previously fanned -// out one GET /projects/:id per project to obtain those documents; with N -// projects that burst — auth check plus several DB queries per request — -// could overwhelm the Supabase gateway. Batching keeps it at one request -// and a fixed number of queries regardless of project count. -projectsRouter.get("/", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const includeDocuments = req.query.include === "documents"; - const db = createServerSupabase(); - - const { data, error } = await db.rpc("get_projects_overview", { - p_user_id: userId, - p_user_email: userEmail ?? null, - }); - if (error) return void res.status(500).json({ detail: error.message }); - - const projects = (data ?? []) as { id: string }[]; - if (!includeDocuments || projects.length === 0) { - return void res.json(projects); - } - - const projectIds = projects.map((p) => p.id); - const [ - { data: docs, error: docsError }, - { data: folders, error: foldersError }, - ] = await Promise.all([ - db - .from("documents") - .select("*") - .in("project_id", projectIds) - .order("created_at", { ascending: true }), - db - .from("project_subfolders") - .select("*") - .in("project_id", projectIds) - .order("created_at", { ascending: true }), - ]); - if (docsError) - return void res.status(500).json({ detail: docsError.message }); - if (foldersError) - return void res.status(500).json({ detail: foldersError.message }); - - const docsTyped = (docs ?? []) as unknown as { - id: string; - project_id?: string | null; - user_id?: string | null; - current_version_id?: string | null; - }[]; - await attachLatestVersionNumbers(db, docsTyped); - await attachActiveVersionPaths(db, docsTyped); - await attachDocumentOwnerLabels(db, docsTyped); - - const docsByProject = new Map(); - for (const doc of docsTyped) { - if (!doc.project_id) continue; - const bucket = docsByProject.get(doc.project_id); - if (bucket) bucket.push(doc); - else docsByProject.set(doc.project_id, [doc]); - } - const foldersByProject = new Map>(); - for (const folder of folders ?? []) { - const projectId = folder.project_id as string; - const bucket = foldersByProject.get(projectId); - if (bucket) bucket.push(folder); - else foldersByProject.set(projectId, [folder]); - } - res.json( - projects.map((p) => ({ - ...p, - documents: docsByProject.get(p.id) ?? [], - folders: foldersByProject.get(p.id) ?? [], - })), - ); -}); - -// POST /projects -projectsRouter.post("/", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { name, cm_number, practice, shared_with } = req.body as { - name: string; - cm_number?: string; - practice?: string; - shared_with?: string[]; - }; - if (!name?.trim()) - return void res.status(400).json({ detail: "name is required" }); - const normalizedUserEmail = userEmail?.trim().toLowerCase(); - const cleanedSharedWith: string[] = []; - const seenSharedEmails = new Set(); - if (Array.isArray(shared_with)) { - for (const raw of shared_with) { - if (typeof raw !== "string") continue; - const e = raw.trim().toLowerCase(); - if (!e || seenSharedEmails.has(e)) continue; - if (normalizedUserEmail && e === normalizedUserEmail) { - return void res - .status(400) - .json({ detail: "You cannot share a project with yourself." }); - } - seenSharedEmails.add(e); - cleanedSharedWith.push(e); - } - } - - const db = createServerSupabase(); - const missingSharedUsers = await findMissingUserEmails(db, cleanedSharedWith); - if (missingSharedUsers.length > 0) { - return void res.status(400).json({ - detail: `${missingSharedUsers[0]} does not belong to a Mike user.`, - }); - } - - const { data, error } = await db - .from("projects") - .insert({ - user_id: userId, - name: name.trim(), - cm_number: normalizeOptionalString(cm_number), - practice: normalizeOptionalString(practice), - shared_with: cleanedSharedWith, - }) - .select("*") - .single(); - if (error) return void res.status(500).json({ detail: error.message }); - res.status(201).json({ ...data, documents: [] }); -}); - -// GET /projects/:projectId -projectsRouter.get("/:projectId", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string; - const { projectId } = req.params; - const db = createServerSupabase(); - - const { data: project, error } = await db - .from("projects") - .select("*") - .eq("id", projectId) - .single(); - if (error || !project) - return void res.status(404).json({ detail: "Project not found" }); - - const canAccess = - project.user_id === userId || - (userEmail && - Array.isArray(project.shared_with) && - project.shared_with.includes(userEmail)); - if (!canAccess) - return void res.status(404).json({ detail: "Project not found" }); - - const [{ data: docs }, { data: folderData }] = await Promise.all([ - db.from("documents").select("*").eq("project_id", projectId).order("created_at", { ascending: true }), - db.from("project_subfolders").select("*").eq("project_id", projectId).order("created_at", { ascending: true }), - ]); - const docsTyped = (docs ?? []) as unknown as { - id: string; - user_id?: string | null; - current_version_id?: string | null; - }[]; - await attachLatestVersionNumbers(db, docsTyped); - await attachActiveVersionPaths(db, docsTyped); - await attachDocumentOwnerLabels(db, docsTyped); - res.json({ - ...project, - is_owner: project.user_id === userId, - documents: docsTyped, - folders: folderData ?? [], - }); -}); - -// GET /projects/:projectId/people -// Resolve the owner + every shared member to {email, display_name}. Used -// by the People modal so the UI can show display names where available -// and tag the current user as "You". -projectsRouter.get("/:projectId/people", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { projectId } = req.params; - const db = createServerSupabase(); - - const { data: project } = await db - .from("projects") - .select("id, user_id, shared_with") - .eq("id", projectId) - .single(); - if (!project) - return void res.status(404).json({ detail: "Project not found" }); - - const isOwner = project.user_id === userId; - const sharedWith = (Array.isArray(project.shared_with) - ? (project.shared_with as string[]) - : [] - ).map((e) => e.toLowerCase()); - const isShared = - !!userEmail && sharedWith.includes(userEmail.toLowerCase()); - if (!isOwner && !isShared) - return void res.status(404).json({ detail: "Project not found" }); - - // Use the mirrored profile email so sharing checks do not scan auth.users. - const { userByEmail, userById } = await loadProfileUsersByEmail(db); - - const ownerInfo = userById.get(project.user_id as string); - const owner = { - user_id: project.user_id, - email: ownerInfo?.email ?? null, - display_name: ownerInfo?.display_name ?? null, - }; - const members = sharedWith.map((email) => { - const u = userByEmail.get(email); - const display_name = u?.display_name ?? null; - return { email, display_name }; - }); - - res.json({ owner, members }); -}); - -// PATCH /projects/:projectId -projectsRouter.patch("/:projectId", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { projectId } = req.params; - const updates: Record = {}; - if (req.body.name != null) updates.name = req.body.name; - if (req.body.cm_number != null) updates.cm_number = req.body.cm_number; - if ("practice" in req.body) { - updates.practice = normalizeOptionalString(req.body.practice); - } - if (Array.isArray(req.body.shared_with)) { - // Normalise: lowercase + dedupe + drop empties. - const normalizedUserEmail = userEmail?.trim().toLowerCase(); - const seen = new Set(); - const cleaned: string[] = []; - for (const raw of req.body.shared_with) { - if (typeof raw !== "string") continue; - const e = raw.trim().toLowerCase(); - if (!e || seen.has(e)) continue; - if (normalizedUserEmail && e === normalizedUserEmail) { - return void res - .status(400) - .json({ detail: "You cannot share a project with yourself." }); - } - seen.add(e); - cleaned.push(e); - } - updates.shared_with = cleaned; - } - - const db = createServerSupabase(); - if (Array.isArray(updates.shared_with)) { - const missingSharedUsers = await findMissingUserEmails( - db, - updates.shared_with as string[], - ); - if (missingSharedUsers.length > 0) { - return void res.status(400).json({ - detail: `${missingSharedUsers[0]} does not belong to a Mike user.`, - }); - } - } - - const { data, error } = await db - .from("projects") - .update({ ...updates, updated_at: new Date().toISOString() }) - .eq("id", projectId) - .eq("user_id", userId) - .select("*") - .single(); - if (error || !data) - return void res.status(404).json({ detail: "Project not found" }); - - const [{ data: docs }, { data: folderData }] = await Promise.all([ - db.from("documents").select("*").eq("project_id", projectId).order("created_at", { ascending: true }), - db.from("project_subfolders").select("*").eq("project_id", projectId).order("created_at", { ascending: true }), - ]); - const docsTyped = (docs ?? []) as unknown as { - id: string; - user_id?: string | null; - current_version_id?: string | null; - }[]; - await attachActiveVersionPaths(db, docsTyped); - await attachDocumentOwnerLabels(db, docsTyped); - res.json({ ...data, documents: docsTyped, folders: folderData ?? [] }); -}); - -// DELETE /projects/:projectId -projectsRouter.delete("/:projectId", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const { projectId } = req.params; - const db = createServerSupabase(); - try { - const deletedCount = await deleteUserProjects(db, userId, [projectId]); - if (deletedCount === 0) - return void res.status(404).json({ detail: "Project not found" }); - res.status(204).send(); - } catch (err) { - const detail = err instanceof Error ? err.message : String(err); - res.status(500).json({ detail }); - } -}); - -// GET /projects/:projectId/documents -projectsRouter.get("/:projectId/documents", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { projectId } = req.params; - const db = createServerSupabase(); - - const access = await checkProjectAccess(projectId, userId, userEmail, db); - if (!access.ok) - return void res.status(404).json({ detail: "Project not found" }); - - const { data: docs } = await db - .from("documents") - .select("*") - .eq("project_id", projectId) - .order("created_at", { ascending: true }); - const docsTyped = (docs ?? []) as unknown as { - id: string; - current_version_id?: string | null; - }[]; - await attachActiveVersionPaths(db, docsTyped); - res.json(docsTyped); -}); - -// GET /projects/:projectId/export — tamper-evident manifest of the project's -// documents: every version with its content_sha256 plus the accept/reject -// trail, under a SHA-256 digest that is Ed25519-signed when the deployment has -// MANIFEST_SIGNING_KEY set. To check an export, recompute a downloaded file's -// SHA-256 and compare, then check the manifest's signature against the key -// served at GET /manifest-signing-key. See the README. -projectsRouter.get( - "/:projectId/export", - requireAuth, - requireMfaIfEnrolled, - async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { projectId } = req.params; - const db = createServerSupabase(); - - const access = await checkProjectAccess(projectId, userId, userEmail, db); - if (!access.ok) - return void res.status(404).json({ detail: "Project not found" }); - - try { - const data = await buildProjectExportManifest(db, projectId); - res.setHeader("Content-Type", "application/json; charset=utf-8"); - res.setHeader( - "Content-Disposition", - `attachment; filename="${projectManifestFilename(projectId)}"`, - ); - res.json(data); - } catch (err) { - console.error("[projects/export] failed", { - projectId, - error: safeErrorLog(err), - }); - res - .status(500) - .json({ detail: "Failed to build project export manifest" }); - } - }, -); - -// POST /projects/:projectId/documents/:documentId — assign or copy existing doc into project -projectsRouter.post( - "/:projectId/documents/:documentId", - requireAuth, - async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { projectId, documentId } = req.params; - const db = createServerSupabase(); - - const access = await checkProjectAccess(projectId, userId, userEmail, db); - if (!access.ok) - return void res.status(404).json({ detail: "Project not found" }); - - // Adding-by-id pulls a doc into the project — only the doc's owner - // is allowed to do that, so other people's standalone docs can't be - // siphoned into a project the requester happens to share. - const { data: doc } = await db - .from("documents") - .select("*") - .eq("id", documentId) - .eq("user_id", userId) - .single(); - if (!doc) - return void res.status(404).json({ detail: "Document not found" }); - await attachActiveVersionPaths( - db, - [doc as { id: string; current_version_id?: string | null }], - ); - - // Already in this project — idempotent - if (doc.project_id === projectId) return void res.json(doc); - - if (doc.project_id === null) { - // Standalone → assign project_id - const { data: updated, error } = await db - .from("documents") - .update({ - project_id: projectId, - library_folder_id: null, - updated_at: new Date().toISOString(), - }) - .eq("id", documentId) - .select("*") - .single(); - if (error || !updated) - return void res.status(500).json({ detail: "Failed to update document" }); - await attachActiveVersionPaths( - db, - [updated as { id: string; current_version_id?: string | null }], - ); - return void res.json(updated); - } else { - // Belongs to another project → duplicate record AND copy the - // underlying storage objects so each project's copy is fully - // independent (edits/version bumps on one don't leak into the - // other). - if (!doc.current_version_id) { - return void res - .status(404) - .json({ detail: "Source document has no active version" }); - } - - const { data: srcV } = await db - .from("document_versions") - .select( - "storage_path, pdf_storage_path, version_number, filename, source, file_type, size_bytes, page_count", - ) - .eq("id", doc.current_version_id) - .single(); - if (!srcV?.storage_path) { - return void res - .status(404) - .json({ detail: "Source document has no active version" }); - } - - const activeVersionFilename = - (srcV.filename as string | null)?.trim() || "Untitled document"; - const srcBytes = await downloadFile(srcV.storage_path); - if (!srcBytes) { - return void res - .status(500) - .json({ detail: "Failed to read source document bytes" }); - } - - const { data: copy, error } = await db - .from("documents") - .insert({ - project_id: projectId, - user_id: userId, - status: doc.status, - }) - .select("*") - .single(); - if (error || !copy) - return void res.status(500).json({ detail: "Failed to copy document" }); - - const newKey = storageKey( - userId, - copy.id as string, - activeVersionFilename, - ); - let newPdfPath: string | null = null; - try { - const contentType = contentTypeForDocumentType( - (srcV.file_type as string | null) ?? doc.file_type, - ); - await uploadFile(newKey, srcBytes, contentType); - - // PDFs share one object for source + display rendition. DOCX - // store the converted PDF at a separate `converted-pdfs/` key — - // copy that too if it exists so the copy renders without going - // back through libreoffice. - if (srcV.pdf_storage_path) { - if (srcV.pdf_storage_path === srcV.storage_path) { - newPdfPath = newKey; - } else { - const pdfBytes = await downloadFile(srcV.pdf_storage_path); - if (pdfBytes) { - const newPdfKey = convertedPdfKey(userId, copy.id as string); - await uploadFile(newPdfKey, pdfBytes, "application/pdf"); - newPdfPath = newPdfKey; - } - } - } - - const { data: newV, error: newVError } = await db - .from("document_versions") - .insert({ - document_id: copy.id, - storage_path: newKey, - pdf_storage_path: newPdfPath, - source: (srcV.source as string | null) ?? "upload", - version_number: srcV.version_number ?? 1, - filename: activeVersionFilename, - file_type: (srcV.file_type as string | null) ?? doc.file_type, - size_bytes: - (srcV.size_bytes as number | null) ?? doc.size_bytes ?? null, - page_count: - (srcV.page_count as number | null) ?? doc.page_count ?? null, - content_sha256: contentSha256(srcBytes), - }) - .select("id") - .single(); - const copyVersionRowId = (newV?.id as string | null) ?? null; - if (newVError || !copyVersionRowId) { - throw new Error( - `Failed to create copied document version: ${newVError?.message ?? "unknown"}`, - ); - } - - const { data: updatedCopy, error: updateCopyError } = await db - .from("documents") - .update({ - current_version_id: copyVersionRowId, - }) - .eq("id", copy.id) - .select("*") - .single(); - if (updateCopyError || !updatedCopy) { - throw new Error( - `Failed to activate copied document version: ${updateCopyError?.message ?? "unknown"}`, - ); - } - - await attachActiveVersionPaths( - db, - [updatedCopy as { id: string; current_version_id?: string | null }], - ); - return void res.status(201).json(updatedCopy); - } catch (err) { - console.error("[projects/documents/copy] failed", err); - await Promise.all([ - deleteFile(newKey).catch(() => {}), - newPdfPath && newPdfPath !== newKey - ? deleteFile(newPdfPath).catch(() => {}) - : Promise.resolve(), - db.from("documents").delete().eq("id", copy.id), - ]); - return void res.status(500).json({ detail: "Failed to copy document" }); - } - } - }, -); - -// PATCH /projects/:projectId/documents/:documentId — rename a project document -projectsRouter.patch("/:projectId/documents/:documentId", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { projectId, documentId } = req.params; - const db = createServerSupabase(); - - const access = await checkProjectAccess(projectId, userId, userEmail, db); - if (!access.ok) - return void res.status(404).json({ detail: "Project not found" }); - - const { data: doc } = await db - .from("documents") - .select("id, current_version_id") - .eq("id", documentId) - .eq("project_id", projectId) - .single(); - if (!doc) - return void res.status(404).json({ detail: "Document not found" }); - - const active = doc.current_version_id - ? await db - .from("document_versions") - .select("filename") - .eq("id", doc.current_version_id) - .eq("document_id", documentId) - .single() - : null; - const currentName = - typeof active?.data?.filename === "string" && - active.data.filename.trim() - ? active.data.filename.trim() - : "Untitled document"; - const filename = normalizeDocumentFilename(req.body?.filename, currentName); - if (!filename) - return void res.status(400).json({ detail: "filename is required" }); - - const { data: updated, error } = await db - .from("documents") - .update({ updated_at: new Date().toISOString() }) - .eq("id", documentId) - .eq("project_id", projectId) - .select("*") - .single(); - if (error || !updated) - return void res.status(404).json({ detail: "Document not found" }); - - if (doc.current_version_id) { - await db - .from("document_versions") - .update({ filename }) - .eq("id", doc.current_version_id) - .eq("document_id", documentId); - } - - res.json({ - ...updated, - filename, - }); -}); - -// POST /projects/:projectId/documents -projectsRouter.post( - "/:projectId/documents", - requireAuth, - singleFileUpload("file"), - async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { projectId } = req.params; - const db = createServerSupabase(); - - const access = await checkProjectAccess(projectId, userId, userEmail, db); - if (!access.ok) - return void res.status(404).json({ detail: "Project not found" }); - - await handleDocumentUpload(req, res, userId, projectId, db); - }, -); - -// GET /projects/:projectId/chats — every assistant chat under this project -// (any author with project access). Used by the project page's chat tab so -// it doesn't have to filter the global GET /chat list — and so collaborators -// see each other's chats inside the project even though those don't appear -// in the global list. -projectsRouter.get("/:projectId/chats", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { projectId } = req.params; - const db = createServerSupabase(); - - const access = await checkProjectAccess(projectId, userId, userEmail, db); - if (!access.ok) - return void res.status(404).json({ detail: "Project not found" }); - - const { data, error } = await db - .from("chats") - .select("*") - .eq("project_id", projectId) - .order("created_at", { ascending: false }); - if (error) return void res.status(500).json({ detail: error.message }); - const chats = data ?? []; - await attachChatCreatorLabels(db, chats); - res.json(chats); -}); - -// ── Folder routes ───────────────────────────────────────────────────────────── - -// POST /projects/:projectId/folders -projectsRouter.post("/:projectId/folders", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { projectId } = req.params; - const { name, parent_folder_id } = req.body as { name: string; parent_folder_id?: string | null }; - if (!name?.trim()) return void res.status(400).json({ detail: "name is required" }); - - const db = createServerSupabase(); - const access = await checkProjectAccess(projectId, userId, userEmail, db); - if (!access.ok) return void res.status(404).json({ detail: "Project not found" }); - - // Verify parent folder belongs to this project - if (parent_folder_id) { - const { data: parent } = await db.from("project_subfolders").select("id").eq("id", parent_folder_id).eq("project_id", projectId).single(); - if (!parent) return void res.status(404).json({ detail: "Parent folder not found" }); - } - - const { data, error } = await db.from("project_subfolders").insert({ - project_id: projectId, - user_id: userId, - name: name.trim(), - parent_folder_id: parent_folder_id ?? null, - }).select("*").single(); - if (error) return void res.status(500).json({ detail: error.message }); - res.status(201).json(data); -}); - -// PATCH /projects/:projectId/folders/:folderId -projectsRouter.patch("/:projectId/folders/:folderId", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { projectId, folderId } = req.params; - const body = req.body as { name?: string; parent_folder_id?: string | null }; - - const db = createServerSupabase(); - const access = await checkProjectAccess(projectId, userId, userEmail, db); - if (!access.ok) return void res.status(404).json({ detail: "Project not found" }); - - const updates: Record = { updated_at: new Date().toISOString() }; - if (body.name != null) updates.name = body.name.trim(); - if ("parent_folder_id" in body) { - // Cycle check: walk up the tree from the proposed parent to ensure folderId is not an ancestor - if (body.parent_folder_id) { - const parent = await loadProjectFolder(db, projectId, body.parent_folder_id); - if (!parent) return void res.status(404).json({ detail: "Parent folder not found" }); - - let cur: string | null = body.parent_folder_id; - while (cur) { - if (cur === folderId) return void res.status(400).json({ detail: "Cannot move a folder into itself or a descendant" }); - const p = await loadProjectFolder(db, projectId, cur); - if (!p) return void res.status(404).json({ detail: "Parent folder not found" }); - cur = p?.parent_folder_id ?? null; - } - } - updates.parent_folder_id = body.parent_folder_id ?? null; - } - - const { data, error } = await db.from("project_subfolders") - .update(updates) - .eq("id", folderId).eq("project_id", projectId) - .select("*").single(); - if (error || !data) return void res.status(404).json({ detail: "Folder not found" }); - res.json(data); -}); - -// DELETE /projects/:projectId/folders/:folderId -projectsRouter.delete("/:projectId/folders/:folderId", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { projectId, folderId } = req.params; - const db = createServerSupabase(); - - const access = await checkProjectAccess(projectId, userId, userEmail, db); - if (!access.ok) return void res.status(404).json({ detail: "Project not found" }); - if (!access.isOwner) return void res.status(404).json({ detail: "Project not found" }); - - const { data: allFolders, error: foldersError } = await db - .from("project_subfolders") - .select("id, parent_folder_id") - .eq("project_id", projectId); - if (foldersError) - return void res.status(500).json({ detail: foldersError.message }); - if (!(allFolders ?? []).some((f) => f.id === folderId)) - return void res.status(404).json({ detail: "Folder not found" }); - - const childrenByParent = new Map(); - for (const f of allFolders ?? []) { - const parentId = f.parent_folder_id as string | null; - if (!parentId) continue; - const children = childrenByParent.get(parentId) ?? []; - children.push(f.id as string); - childrenByParent.set(parentId, children); - } - - const folderIds = new Set(); - const stack = [folderId]; - while (stack.length > 0) { - const id = stack.pop()!; - if (folderIds.has(id)) continue; - folderIds.add(id); - stack.push(...(childrenByParent.get(id) ?? [])); - } - - const { data: docs, error: docsError } = await db - .from("documents") - .select("id") - .eq("project_id", projectId) - .in("folder_id", [...folderIds]); - if (docsError) return void res.status(500).json({ detail: docsError.message }); - - const docIds = (docs ?? []).map((d) => d.id as string); - const deleteDocsError = await deleteProjectDocumentsAndVersionFiles( - db, - projectId, - docIds, - ); - if (deleteDocsError) - return void res.status(500).json({ detail: deleteDocsError.message }); - - const { error } = await db.from("project_subfolders") - .delete().eq("id", folderId).eq("project_id", projectId); - if (error) return void res.status(500).json({ detail: error.message }); - res.status(204).send(); -}); - -// PATCH /projects/:projectId/documents/:documentId/folder — move doc to a folder -projectsRouter.patch("/:projectId/documents/:documentId/folder", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { projectId, documentId } = req.params; - const { folder_id } = req.body as { folder_id: string | null }; - - const db = createServerSupabase(); - const access = await checkProjectAccess(projectId, userId, userEmail, db); - if (!access.ok) return void res.status(404).json({ detail: "Project not found" }); - - if (folder_id) { - const folder = await loadProjectFolder(db, projectId, folder_id); - if (!folder) return void res.status(404).json({ detail: "Folder not found" }); - } - - const { data, error } = await db.from("documents") - .update({ folder_id: folder_id ?? null, updated_at: new Date().toISOString() }) - .eq("id", documentId).eq("project_id", projectId) - .select("*").single(); - if (error || !data) return void res.status(404).json({ detail: "Document not found" }); - res.json(data); -}); - -async function loadProjectFolder( - db: ReturnType, - projectId: string, - folderId: string, -): Promise<{ id: string; parent_folder_id: string | null } | null> { - const { data } = await db - .from("project_subfolders") - .select("id, parent_folder_id") - .eq("id", folderId) - .eq("project_id", projectId) - .maybeSingle(); - return (data as { id: string; parent_folder_id: string | null } | null) ?? null; -} - -export async function handleDocumentUpload( - req: import("express").Request, - res: import("express").Response, - userId: string, - projectId: string | null, - db: ReturnType, -) { - const file = req.file; - if (!file) return void res.status(400).json({ detail: "file is required" }); - - const filename = file.originalname; - const suffix = filename.includes(".") - ? filename.split(".").pop()!.toLowerCase() - : ""; - if (!ALLOWED_DOCUMENT_TYPES.has(suffix)) - return void res - .status(400) - .json({ - detail: `Unsupported file type: ${suffix}. Allowed: ${ALLOWED_DOCUMENT_TYPES_LABEL}`, - }); - - const content = file.buffer; - const { data: doc, error: insertErr } = await db - .from("documents") - .insert({ - project_id: projectId, - user_id: userId, - status: "processing", - }) - .select("*") - .single(); - - if (insertErr || !doc) - return void res - .status(500) - .json({ detail: "Failed to create document record" }); - - try { - const docId = doc.id as string; - const key = storageKey(userId, docId, filename); - const contentType = contentTypeForDocumentType(suffix); - await uploadFile( - key, - content.buffer.slice( - content.byteOffset, - content.byteOffset + content.byteLength, - ) as ArrayBuffer, - contentType, - ); - - const rawBuf = content.buffer.slice( - content.byteOffset, - content.byteOffset + content.byteLength, - ) as ArrayBuffer; - const pageCount = suffix === "pdf" ? await countPdfPages(rawBuf) : null; - - // When the job queue is enabled, defer Office → PDF conversion to the - // BullMQ worker instead of blocking the upload request on LibreOffice — - // the same deferral the single-document upload path makes. - const deferConversion = - shouldConvertToPdf(suffix) && - process.env.ASYNC_DOCUMENT_CONVERSION === "true"; - - // Convert Office files → PDF for display. PDFs are their own rendition. - let pdfStoragePath: string | null = null; - if (!deferConversion && shouldConvertToPdf(suffix)) { - try { - const pdfBuf = await docxToPdf(content); - const pdfKey = convertedPdfKey(userId, docId); - await uploadFile( - pdfKey, - pdfBuf.buffer.slice( - pdfBuf.byteOffset, - pdfBuf.byteOffset + pdfBuf.byteLength, - ) as ArrayBuffer, - "application/pdf", - ); - pdfStoragePath = pdfKey; - } catch (err) { - console.error( - `[upload] Office→PDF conversion failed for ${filename}:`, - err, - ); - } - } else if (suffix === "pdf") { - pdfStoragePath = key; - } - - // Storage paths live on document_versions — create the V1 row and - // point documents.current_version_id at it. - const { data: versionRow, error: verErr } = await db - .from("document_versions") - .insert({ - document_id: docId, - storage_path: key, - pdf_storage_path: pdfStoragePath, - source: "upload", - version_number: 1, - filename, - file_type: suffix, - size_bytes: content.byteLength, - page_count: pageCount, - content_sha256: contentSha256(content), - }) - .select("id") - .single(); - if (verErr || !versionRow) { - throw new Error( - `Failed to record upload version: ${verErr?.message ?? "unknown"}`, - ); - } - - await db - .from("documents") - .update({ - current_version_id: versionRow.id, - // Deferred conversion leaves the doc "processing" until the worker - // produces the PDF and flips it to "ready". - status: deferConversion ? "processing" : "ready", - updated_at: new Date().toISOString(), - }) - .eq("id", docId); - - if (deferConversion) { - await enqueueConversion({ - documentId: docId, - versionId: versionRow.id as string, - userId, - storagePath: key, - fileType: suffix, - }); - } - - const { data: updated } = await db - .from("documents") - .select("*") - .eq("id", docId) - .single(); - const responseDoc = updated - ? { - ...updated, - filename, - storage_path: key, - pdf_storage_path: pdfStoragePath, - file_type: suffix, - size_bytes: content.byteLength, - page_count: pageCount, - active_version_number: 1, - } - : updated; - return void res.status(201).json(responseDoc); - } catch (e) { - await db.from("documents").update({ status: "error" }).eq("id", doc.id); - return void res - .status(500) - .json({ detail: `Document processing failed: ${String(e)}` }); - } -} - -async function countPdfPages(buf: ArrayBuffer): Promise { - try { - const pdfjsLib = await import("pdfjs-dist/legacy/build/pdf.mjs" as string); - const pdf = await ( - pdfjsLib as unknown as { - getDocument: (opts: unknown) => { - promise: Promise<{ numPages: number }>; - }; - } - ).getDocument({ data: new Uint8Array(buf) }).promise; - return pdf.numPages; - } catch { - return null; - } -} diff --git a/backend/src/routes/user.ts b/backend/src/routes/user.ts deleted file mode 100644 index ca77f5952..000000000 --- a/backend/src/routes/user.ts +++ /dev/null @@ -1,1132 +0,0 @@ -import crypto from "crypto"; -import { Router } from "express"; -import { requireAuth, requireMfaIfEnrolled } from "../middleware/auth"; -import { createServerSupabase } from "../lib/supabase"; -import { - DEFAULT_TABULAR_MODEL, - DEFAULT_TITLE_MODEL, - CLAUDE_LOW_MODELS, - OPENAI_LOW_MODELS, - resolveModel, -} from "../lib/llm"; -import { - type ApiKeyStatus, - getUserApiKeyStatus, - hasEnvApiKey, - normalizeApiKeyProvider, - saveUserApiKey, -} from "../lib/userApiKeys"; -import { - completeUserMcpConnectorOAuth, - createUserMcpConnector, - deleteUserMcpConnector, - getUserMcpConnector, - listUserMcpConnectors, - McpOAuthRequiredError, - refreshUserMcpConnectorTools, - setUserMcpToolEnabled, - startUserMcpConnectorOAuth, - updateUserMcpConnector, -} from "../lib/mcpConnectors"; -import { - deleteAllUserChats, - deleteAllUserTabularReviews, - deleteUserAccountData, - deleteUserProjects, -} from "../lib/userDataCleanup"; -import { - buildUserAccountExport, - buildUserChatsExport, - buildUserTabularReviewsExport, - userExportFilename, -} from "../lib/userDataExport"; -import { findProfileUserByEmail } from "../lib/userLookup"; - -export const userRouter = Router(); - -const MONTHLY_CREDIT_LIMIT = 999999; - -type UserProfileRow = { - display_name: string | null; - organisation: string | null; - message_credits_used: number; - credits_reset_date: string; - tier: string; - title_model: string | null; - tabular_model: string; - mfa_on_login: boolean | null; - legal_research_us: boolean | null; -}; - -function errorMessage(error: unknown): string { - if (error instanceof Error && error.message) return error.message; - if (error && typeof error === "object") { - const record = error as { - message?: unknown; - details?: unknown; - hint?: unknown; - code?: unknown; - }; - return ( - [record.message, record.details, record.hint, record.code] - .filter( - (value): value is string => - typeof value === "string" && !!value, - ) - .join(" ") || JSON.stringify(error) - ); - } - return String(error); -} - -function backendPublicUrl(req: { - protocol: string; - get(name: string): string | undefined; -}) { - return ( - process.env.API_PUBLIC_URL || - process.env.BACKEND_URL || - `${req.protocol}://${req.get("host")}` - ).replace(/\/+$/, ""); -} - -function frontendUrl(path = "/account/connectors") { - const base = (process.env.FRONTEND_URL ?? "http://localhost:3000").replace( - /\/+$/, - "", - ); - return `${base}${path}`; -} - -function shortHash(value: string) { - return value - ? crypto.createHash("sha256").update(value).digest("hex").slice(0, 12) - : null; -} - -function mcpOAuthPopupHtml(payload: { - success: boolean; - connectorId?: string; - detail?: string; -}, nonce: string) { - const targetOrigin = new URL(frontendUrl()).origin; - const targetUrl = frontendUrl(); - const message = JSON.stringify({ - type: "mcp_oauth_result", - ...payload, - }); - return ` - - - - - MCP authorization - - - -
-

${payload.success ? "Authorization complete" : "Authorization failed"}

-

${payload.success ? "You can return to Mike." : "Return to Mike and try connecting again."}

-
- - -`; -} - -function mcpOAuthPopupCsp(nonce: string) { - return [ - "default-src 'none'", - `script-src 'nonce-${nonce}'`, - "style-src 'unsafe-inline'", - "base-uri 'none'", - "form-action 'none'", - "frame-ancestors 'none'", - ].join("; "); -} - -const PROFILE_SELECT = - "display_name, organisation, message_credits_used, credits_reset_date, tier, title_model, tabular_model, mfa_on_login, legal_research_us"; -const PROFILE_SELECT_NO_LEGAL = - "display_name, organisation, message_credits_used, credits_reset_date, tier, title_model, tabular_model, mfa_on_login"; -const LEGACY_PROFILE_SELECT = - "display_name, organisation, message_credits_used, credits_reset_date, tier, tabular_model"; -const LEGACY_PROFILE_MODEL_SELECT = - "display_name, organisation, message_credits_used, credits_reset_date, tier, title_model, tabular_model"; - -function isMissingProfileColumn(error: unknown, column: string): boolean { - const record = - error && typeof error === "object" - ? (error as { code?: unknown; message?: unknown }) - : {}; - const message = typeof record.message === "string" ? record.message : ""; - return record.code === "42703" && message.includes(column); -} - -// Loads a profile while tolerating older databases that lack the -// legal_research_us column. Tries the full select first, then falls back to -// the legacy cascade (which also handles missing title_model / mfa_on_login) -// and defaults the feature flag to enabled. -async function selectProfile( - db: ReturnType, - userId: string, - mode: "maybe" | "single", -) { - const fullQuery = db - .from("user_profiles") - .select(PROFILE_SELECT) - .eq("user_id", userId); - const full = - mode === "single" - ? await fullQuery.single() - : await fullQuery.maybeSingle(); - if (!full.error) return full; - - const legacy = await selectProfileLegacy(db, userId, mode); - if (legacy.data && typeof legacy.data === "object") { - const row = legacy.data as Record; - if (!("legal_research_us" in row)) { - Object.assign(row, { legal_research_us: true }); - } - } - return legacy; -} - -async function selectProfileLegacy( - db: ReturnType, - userId: string, - mode: "maybe" | "single", -) { - const query = db - .from("user_profiles") - .select(PROFILE_SELECT_NO_LEGAL) - .eq("user_id", userId); - const result = - mode === "single" ? await query.single() : await query.maybeSingle(); - if (!result.error) { - return result; - } - - const missingMfaOnLogin = isMissingProfileColumn( - result.error, - "mfa_on_login", - ); - if (missingMfaOnLogin) { - const modelQuery = db - .from("user_profiles") - .select(LEGACY_PROFILE_MODEL_SELECT) - .eq("user_id", userId); - const modelLegacy = - mode === "single" - ? await modelQuery.single() - : await modelQuery.maybeSingle(); - if ( - !modelLegacy.error || - !isMissingProfileColumn(modelLegacy.error, "title_model") - ) { - if (modelLegacy.data && typeof modelLegacy.data === "object") { - const row = modelLegacy.data as Record; - Object.assign(row, { - mfa_on_login: false, - }); - } - return modelLegacy; - } - } - - if ( - !missingMfaOnLogin && - !isMissingProfileColumn(result.error, "title_model") - ) { - return result; - } - - const legacyQuery = db - .from("user_profiles") - .select(LEGACY_PROFILE_SELECT) - .eq("user_id", userId); - const legacy = - mode === "single" - ? await legacyQuery.single() - : await legacyQuery.maybeSingle(); - if (legacy.data && typeof legacy.data === "object") { - const row = legacy.data as Record; - Object.assign(row, { - title_model: null, - mfa_on_login: false, - }); - } - return legacy; -} - -function serializeProfile(row: UserProfileRow, apiKeyStatus?: ApiKeyStatus) { - const creditsUsed = row.message_credits_used ?? 0; - const titleFallback = apiKeyStatus?.gemini - ? DEFAULT_TITLE_MODEL - : apiKeyStatus?.openai - ? OPENAI_LOW_MODELS[0] - : apiKeyStatus?.claude - ? CLAUDE_LOW_MODELS[0] - : DEFAULT_TITLE_MODEL; - return { - displayName: row.display_name, - organisation: row.organisation, - messageCreditsUsed: creditsUsed, - creditsResetDate: row.credits_reset_date, - creditsRemaining: Math.max(MONTHLY_CREDIT_LIMIT - creditsUsed, 0), - tier: row.tier || "Free", - titleModel: resolveModel(row.title_model, titleFallback), - tabularModel: resolveModel(row.tabular_model, DEFAULT_TABULAR_MODEL), - mfaOnLogin: row.mfa_on_login === true, - legalResearchUs: row.legal_research_us !== false, - ...(apiKeyStatus ? { apiKeyStatus } : {}), - }; -} - -function validateProfilePayload(body: unknown): - | { - ok: true; - update: { - display_name?: string | null; - organisation?: string | null; - title_model?: string; - tabular_model?: string; - legal_research_us?: boolean; - updated_at: string; - }; - } - | { ok: false; detail: string } { - if (!body || typeof body !== "object" || Array.isArray(body)) { - return { ok: false, detail: "Expected a JSON object" }; - } - - const raw = body as Record; - const allowedFields = new Set([ - "displayName", - "organisation", - "titleModel", - "tabularModel", - "legalResearchUs", - ]); - const invalidField = Object.keys(raw).find( - (key) => !allowedFields.has(key), - ); - if (invalidField) { - return { - ok: false, - detail: `Unsupported profile field: ${invalidField}`, - }; - } - - const update: { - display_name?: string | null; - organisation?: string | null; - title_model?: string; - tabular_model?: string; - legal_research_us?: boolean; - updated_at: string; - } = { updated_at: new Date().toISOString() }; - - if ("displayName" in raw) { - if (raw.displayName !== null && typeof raw.displayName !== "string") { - return { - ok: false, - detail: "displayName must be a string or null", - }; - } - update.display_name = raw.displayName?.trim() || null; - } - - if ("organisation" in raw) { - if (raw.organisation !== null && typeof raw.organisation !== "string") { - return { - ok: false, - detail: "organisation must be a string or null", - }; - } - update.organisation = raw.organisation?.trim() || null; - } - - if ("tabularModel" in raw) { - if (typeof raw.tabularModel !== "string") { - return { ok: false, detail: "tabularModel must be a string" }; - } - const resolved = resolveModel(raw.tabularModel, ""); - if (!resolved) { - return { ok: false, detail: "Unsupported tabularModel" }; - } - update.tabular_model = resolved; - } - - if ("titleModel" in raw) { - if (typeof raw.titleModel !== "string") { - return { ok: false, detail: "titleModel must be a string" }; - } - const resolved = resolveModel(raw.titleModel, ""); - if (!resolved) { - return { ok: false, detail: "Unsupported titleModel" }; - } - update.title_model = resolved; - } - - if ("legalResearchUs" in raw) { - if (typeof raw.legalResearchUs !== "boolean") { - return { - ok: false, - detail: "legalResearchUs must be a boolean", - }; - } - update.legal_research_us = raw.legalResearchUs; - } - - return { ok: true, update }; -} - -function readBooleanBodyField( - body: unknown, - field: string, -): { ok: true; value: boolean } | { ok: false; detail: string } { - if (!body || typeof body !== "object" || Array.isArray(body)) { - return { ok: false, detail: "Expected a JSON object" }; - } - - const raw = body as Record; - const invalidField = Object.keys(raw).find((key) => key !== field); - if (invalidField) { - return { ok: false, detail: `Unsupported field: ${invalidField}` }; - } - if (typeof raw[field] !== "boolean") { - return { ok: false, detail: `${field} must be a boolean` }; - } - - return { ok: true, value: raw[field] }; -} - -async function userHasVerifiedTotpFactor( - db: ReturnType, - userId: string, -) { - const { data, error } = await db.auth.admin.getUserById(userId); - if (error) return { ok: false as const, error }; - - const factors = data.user?.factors ?? []; - return { - ok: true as const, - hasVerifiedTotp: factors.some( - (factor) => - factor.factor_type === "totp" && factor.status === "verified", - ), - }; -} - -async function ensureProfileRow( - db: ReturnType, - userId: string, -) { - const { error } = await db - .from("user_profiles") - .upsert( - { user_id: userId }, - { onConflict: "user_id", ignoreDuplicates: true }, - ); - return error; -} - -async function loadProfile( - db: ReturnType, - userId: string, - options: { repairMissing?: boolean; apiKeyStatus?: ApiKeyStatus } = {}, -) { - let { data, error } = await selectProfile(db, userId, "maybe"); - - if (error) return { data: null, error }; - if (!data) { - if (!options.repairMissing) { - return { data: null, error: new Error("Profile not found") }; - } - - const ensureError = await ensureProfileRow(db, userId); - if (ensureError) return { data: null, error: ensureError }; - - const created = await selectProfile(db, userId, "single"); - if (created.error) return { data: null, error: created.error }; - data = created.data; - } - - let row = data as UserProfileRow; - if ( - row.credits_reset_date && - new Date() > new Date(row.credits_reset_date) - ) { - const creditsResetDate = new Date(); - creditsResetDate.setDate(creditsResetDate.getDate() + 30); - const { error: resetError } = await db - .from("user_profiles") - .update({ - message_credits_used: 0, - credits_reset_date: creditsResetDate.toISOString(), - updated_at: new Date().toISOString(), - }) - .eq("user_id", userId); - - if (resetError) return { data: null, error: resetError }; - const { data: resetData, error: resetLoadError } = await selectProfile( - db, - userId, - "single", - ); - if (resetLoadError) return { data: null, error: resetLoadError }; - row = resetData as UserProfileRow; - } - - return { data: serializeProfile(row, options.apiKeyStatus), error: null }; -} - -// POST /user/profile -userRouter.post("/profile", requireAuth, async (_req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - const error = await ensureProfileRow(db, userId); - if (error) return void res.status(500).json({ detail: error.message }); - res.json({ ok: true }); -}); - -// GET /user/lookup?email=person@example.com -userRouter.get("/lookup", requireAuth, async (req, res) => { - const email = typeof req.query.email === "string" ? req.query.email : ""; - if (!email.trim()) { - return void res.status(400).json({ detail: "email is required" }); - } - - const db = createServerSupabase(); - const user = await findProfileUserByEmail(db, email); - res.json({ - exists: !!user, - email: user?.email ?? email.trim().toLowerCase(), - display_name: user?.display_name ?? null, - }); -}); - -// GET /user/profile -userRouter.get("/profile", requireAuth, async (_req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - const apiKeyStatus = await getUserApiKeyStatus(userId, db); - const { data, error } = await loadProfile(db, userId, { - repairMissing: true, - apiKeyStatus, - }); - if (error) return void res.status(500).json({ detail: error.message }); - res.json({ ...data, apiKeyStatus }); -}); - -// PATCH /user/profile -userRouter.patch("/profile", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const parsed = validateProfilePayload(req.body); - if (!parsed.ok) return void res.status(400).json({ detail: parsed.detail }); - - const db = createServerSupabase(); - const ensureError = await ensureProfileRow(db, userId); - if (ensureError) - return void res.status(500).json({ detail: ensureError.message }); - - const { error: updateError } = await db - .from("user_profiles") - .update(parsed.update) - .eq("user_id", userId); - if (updateError) - return void res.status(500).json({ detail: updateError.message }); - - const apiKeyStatus = await getUserApiKeyStatus(userId, db); - const { data, error } = await loadProfile(db, userId, { apiKeyStatus }); - if (error) return void res.status(500).json({ detail: error.message }); - res.json({ ...data, apiKeyStatus }); -}); - -// PATCH /user/security/mfa-login -userRouter.patch( - "/security/mfa-login", - requireAuth, - requireMfaIfEnrolled, - async (req, res) => { - const userId = res.locals.userId as string; - const parsed = readBooleanBodyField(req.body, "enabled"); - if (!parsed.ok) - return void res.status(400).json({ detail: parsed.detail }); - - const db = createServerSupabase(); - if (parsed.value) { - const factorCheck = await userHasVerifiedTotpFactor(db, userId); - if (!factorCheck.ok) { - return void res.status(500).json({ - detail: factorCheck.error.message, - }); - } - if (!factorCheck.hasVerifiedTotp) { - return void res.status(400).json({ - detail: "Set up an authenticator app before requiring verification on login.", - }); - } - } - - const ensureError = await ensureProfileRow(db, userId); - if (ensureError) - return void res.status(500).json({ detail: ensureError.message }); - - const { error: updateError } = await db - .from("user_profiles") - .update({ - mfa_on_login: parsed.value, - updated_at: new Date().toISOString(), - }) - .eq("user_id", userId); - if (updateError) - return void res.status(500).json({ detail: updateError.message }); - - const apiKeyStatus = await getUserApiKeyStatus(userId, db); - const { data, error } = await loadProfile(db, userId, { apiKeyStatus }); - if (error) return void res.status(500).json({ detail: error.message }); - res.json({ ...data, apiKeyStatus }); - }, -); - -// GET /user/api-keys -userRouter.get("/api-keys", requireAuth, async (_req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - const status = await getUserApiKeyStatus(userId, db); - res.json(status); -}); - -// PUT /user/api-keys/:provider -userRouter.put( - "/api-keys/:provider", - requireAuth, - requireMfaIfEnrolled, - async (req, res) => { - const userId = res.locals.userId as string; - const provider = normalizeApiKeyProvider(req.params.provider); - if (!provider) - return void res - .status(400) - .json({ detail: "Unsupported provider" }); - - const apiKey = - typeof req.body?.api_key === "string" ? req.body.api_key : null; - const db = createServerSupabase(); - try { - if (hasEnvApiKey(provider)) { - return void res.status(409).json({ - detail: "This provider is configured by the server environment and cannot be changed from the browser.", - }); - } - await saveUserApiKey(userId, provider, apiKey, db); - const status = await getUserApiKeyStatus(userId, db); - res.json(status); - } catch (err) { - const detail = errorMessage(err); - console.error("[user/api-keys] save failed", { - provider, - error: detail, - }); - res.status(500).json({ detail }); - } - }, -); - -// GET /user/mcp-connectors -userRouter.get("/mcp-connectors", requireAuth, async (_req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - try { - res.json( - await listUserMcpConnectors(userId, db, { includeTools: false }), - ); - } catch (err) { - const detail = errorMessage(err); - console.error("[user/mcp-connectors] list failed", { - userId, - error: detail, - }); - res.status(500).json({ detail }); - } -}); - -// GET /user/mcp-connectors/:connectorId -userRouter.get( - "/mcp-connectors/:connectorId", - requireAuth, - async (req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - try { - res.json( - await getUserMcpConnector(userId, req.params.connectorId, db), - ); - } catch (err) { - const detail = errorMessage(err); - console.error("[user/mcp-connectors] get failed", { - userId, - connectorId: req.params.connectorId, - error: detail, - }); - res.status(404).json({ detail }); - } - }, -); - -// POST /user/mcp-connectors -userRouter.post( - "/mcp-connectors", - requireAuth, - requireMfaIfEnrolled, - async (req, res) => { - const userId = res.locals.userId as string; - const name = typeof req.body?.name === "string" ? req.body.name : ""; - const serverUrl = - typeof req.body?.serverUrl === "string" ? req.body.serverUrl : ""; - const bearerToken = - typeof req.body?.bearerToken === "string" - ? req.body.bearerToken - : null; - const headers = - req.body?.headers && - typeof req.body.headers === "object" && - !Array.isArray(req.body.headers) - ? (req.body.headers as Record) - : undefined; - const db = createServerSupabase(); - try { - const connector = await createUserMcpConnector( - userId, - { name, serverUrl, bearerToken, headers }, - db, - ); - res.status(201).json(connector); - } catch (err) { - const detail = errorMessage(err); - console.error("[user/mcp-connectors] create failed", { - userId, - error: detail, - }); - res.status(400).json({ detail }); - } - }, -); - -// PATCH /user/mcp-connectors/:connectorId -userRouter.patch( - "/mcp-connectors/:connectorId", - requireAuth, - requireMfaIfEnrolled, - async (req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - const body = req.body ?? {}; - try { - const connector = await updateUserMcpConnector( - userId, - req.params.connectorId, - { - ...(typeof body.name === "string" - ? { name: body.name } - : {}), - ...(typeof body.serverUrl === "string" - ? { serverUrl: body.serverUrl } - : {}), - ...(typeof body.enabled === "boolean" - ? { enabled: body.enabled } - : {}), - ...("bearerToken" in body - ? { - bearerToken: - typeof body.bearerToken === "string" - ? body.bearerToken - : null, - } - : {}), - ...("headers" in body - ? { - headers: - body.headers && - typeof body.headers === "object" && - !Array.isArray(body.headers) - ? (body.headers as Record< - string, - unknown - >) - : {}, - } - : {}), - }, - db, - ); - res.json(connector); - } catch (err) { - const detail = errorMessage(err); - console.error("[user/mcp-connectors] update failed", { - userId, - connectorId: req.params.connectorId, - error: detail, - }); - res.status(400).json({ detail }); - } - }, -); - -// DELETE /user/mcp-connectors/:connectorId -userRouter.delete( - "/mcp-connectors/:connectorId", - requireAuth, - requireMfaIfEnrolled, - async (req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - try { - await deleteUserMcpConnector(userId, req.params.connectorId, db); - res.status(204).send(); - } catch (err) { - const detail = errorMessage(err); - console.error("[user/mcp-connectors] delete failed", { - userId, - connectorId: req.params.connectorId, - error: detail, - }); - res.status(500).json({ detail }); - } - }, -); - -// POST /user/mcp-connectors/:connectorId/oauth/start -userRouter.post( - "/mcp-connectors/:connectorId/oauth/start", - requireAuth, - requireMfaIfEnrolled, - async (req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - try { - const redirectUri = `${backendPublicUrl(req)}/user/mcp-connectors/oauth/callback`; - const result = await startUserMcpConnectorOAuth( - userId, - req.params.connectorId, - redirectUri, - db, - ); - res.json(result); - } catch (err) { - const detail = errorMessage(err); - console.error("[user/mcp-connectors] oauth start failed", { - userId, - connectorId: req.params.connectorId, - error: detail, - }); - res.status(400).json({ detail }); - } - }, -); - -// GET /user/mcp-connectors/oauth/callback -userRouter.get("/mcp-connectors/oauth/callback", async (req, res) => { - const nonce = crypto.randomBytes(16).toString("base64"); - const state = typeof req.query.state === "string" ? req.query.state : ""; - const code = typeof req.query.code === "string" ? req.query.code : ""; - const error = - typeof req.query.error === "string" ? req.query.error : undefined; - const db = createServerSupabase(); - try { - if (error) throw new Error(error); - if (!state || !code) - throw new Error("OAuth callback is missing state or code."); - const result = await completeUserMcpConnectorOAuth(state, code, db); - res.set("Content-Security-Policy", mcpOAuthPopupCsp(nonce)) - .type("html") - .send( - mcpOAuthPopupHtml( - { - success: true, - connectorId: result.connectorId, - }, - nonce, - ), - ); - } catch (err) { - const detail = errorMessage(err); - console.error("[user/mcp-connectors] oauth callback failed", { - error: detail, - stateHash: shortHash(state), - hasCode: !!code, - hasError: !!error, - issuer: - typeof req.query.iss === "string" ? req.query.iss : undefined, - scope: - typeof req.query.scope === "string" - ? req.query.scope - : undefined, - }); - res.status(400) - .set("Content-Security-Policy", mcpOAuthPopupCsp(nonce)) - .type("html") - .send(mcpOAuthPopupHtml({ success: false, detail }, nonce)); - } -}); - -// POST /user/mcp-connectors/:connectorId/refresh-tools -userRouter.post( - "/mcp-connectors/:connectorId/refresh-tools", - requireAuth, - requireMfaIfEnrolled, - async (req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - try { - const connector = await refreshUserMcpConnectorTools( - userId, - req.params.connectorId, - db, - ); - res.json(connector); - } catch (err) { - const detail = errorMessage(err); - console.error("[user/mcp-connectors] refresh failed", { - userId, - connectorId: req.params.connectorId, - error: detail, - }); - if (err instanceof McpOAuthRequiredError) { - return void res.status(401).json({ - code: err.code, - detail, - }); - } - res.status(400).json({ detail }); - } - }, -); - -// PATCH /user/mcp-connectors/:connectorId/tools/:toolId -userRouter.patch( - "/mcp-connectors/:connectorId/tools/:toolId", - requireAuth, - requireMfaIfEnrolled, - async (req, res) => { - const userId = res.locals.userId as string; - const parsed = readBooleanBodyField(req.body, "enabled"); - if (!parsed.ok) - return void res.status(400).json({ detail: parsed.detail }); - - const db = createServerSupabase(); - try { - const connector = await setUserMcpToolEnabled( - userId, - req.params.connectorId, - req.params.toolId, - parsed.value, - db, - ); - res.json(connector); - } catch (err) { - const detail = errorMessage(err); - console.error("[user/mcp-connectors] tool toggle failed", { - userId, - connectorId: req.params.connectorId, - toolId: req.params.toolId, - error: detail, - }); - res.status(400).json({ detail }); - } - }, -); - -// DELETE /user/account -userRouter.delete( - "/account", - requireAuth, - requireMfaIfEnrolled, - async (_req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const db = createServerSupabase(); - try { - await deleteUserAccountData(db, userId, userEmail); - const { error } = await db.auth.admin.deleteUser(userId); - if (error) - return void res.status(500).json({ detail: error.message }); - res.status(204).send(); - } catch (err) { - const detail = errorMessage(err); - console.error("[user/account] delete failed", { - userId, - error: detail, - }); - res.status(500).json({ detail }); - } - }, -); - -// DELETE /user/chats -userRouter.delete( - "/chats", - requireAuth, - requireMfaIfEnrolled, - async (_req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - try { - await deleteAllUserChats(db, userId); - res.status(204).send(); - } catch (err) { - const detail = errorMessage(err); - console.error("[user/chats] delete failed", { - userId, - error: detail, - }); - res.status(500).json({ detail }); - } - }, -); - -// DELETE /user/projects -userRouter.delete( - "/projects", - requireAuth, - requireMfaIfEnrolled, - async (_req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - try { - await deleteUserProjects(db, userId); - res.status(204).send(); - } catch (err) { - const detail = errorMessage(err); - console.error("[user/projects] delete failed", { - userId, - error: detail, - }); - res.status(500).json({ detail }); - } - }, -); - -// DELETE /user/tabular-reviews -userRouter.delete( - "/tabular-reviews", - requireAuth, - requireMfaIfEnrolled, - async (_req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - try { - await deleteAllUserTabularReviews(db, userId); - res.status(204).send(); - } catch (err) { - const detail = errorMessage(err); - console.error("[user/tabular-reviews] delete failed", { - userId, - error: detail, - }); - res.status(500).json({ detail }); - } - }, -); - -// GET /user/export -userRouter.get( - "/export", - requireAuth, - requireMfaIfEnrolled, - async (_req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const db = createServerSupabase(); - try { - const data = await buildUserAccountExport(db, userId, userEmail); - res.setHeader("Content-Type", "application/json; charset=utf-8"); - res.setHeader( - "Content-Disposition", - `attachment; filename="${userExportFilename("account", userId)}"`, - ); - res.json(data); - } catch (err) { - const detail = errorMessage(err); - console.error("[user/export] failed", { userId, error: detail }); - res.status(500).json({ detail }); - } - }, -); - -// GET /user/chats/export -userRouter.get( - "/chats/export", - requireAuth, - requireMfaIfEnrolled, - async (_req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const db = createServerSupabase(); - try { - const data = await buildUserChatsExport(db, userId, userEmail); - res.setHeader("Content-Type", "application/json; charset=utf-8"); - res.setHeader( - "Content-Disposition", - `attachment; filename="${userExportFilename("chats", userId)}"`, - ); - res.json(data); - } catch (err) { - const detail = errorMessage(err); - console.error("[user/chats/export] failed", { - userId, - error: detail, - }); - res.status(500).json({ detail }); - } - }, -); - -// GET /user/tabular-reviews/export -userRouter.get( - "/tabular-reviews/export", - requireAuth, - requireMfaIfEnrolled, - async (_req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const db = createServerSupabase(); - try { - const data = await buildUserTabularReviewsExport( - db, - userId, - userEmail, - ); - res.setHeader("Content-Type", "application/json; charset=utf-8"); - res.setHeader( - "Content-Disposition", - `attachment; filename="${userExportFilename("tabular-reviews", userId)}"`, - ); - res.json(data); - } catch (err) { - const detail = errorMessage(err); - console.error("[user/tabular-reviews/export] failed", { - userId, - error: detail, - }); - res.status(500).json({ detail }); - } - }, -); diff --git a/backend/src/workers/__tests__/extractionWorker.test.ts b/backend/src/workers/__tests__/extractionWorker.test.ts index f73fb5326..f193c0fd7 100644 --- a/backend/src/workers/__tests__/extractionWorker.test.ts +++ b/backend/src/workers/__tests__/extractionWorker.test.ts @@ -6,7 +6,7 @@ vi.mock("../../lib/supabase", () => ({ const loadReviewRow = vi.fn(); const loadRowDocumentText = vi.fn(); -vi.mock("../../lib/tabular/tabular.rows", () => ({ +vi.mock("../../modules/tabular/tabular.rows", () => ({ loadReviewRow: (...a: unknown[]) => loadReviewRow(...a), loadRowDocumentText: (...a: unknown[]) => loadRowDocumentText(...a), })); @@ -19,7 +19,7 @@ vi.mock("../../lib/userSettings", () => ({ })); const queryTabularAllColumns = vi.fn(); -vi.mock("../../lib/tabular/tabular.extract", () => ({ +vi.mock("../../modules/tabular/tabular.extract", () => ({ queryTabularAllColumns: (...a: unknown[]) => queryTabularAllColumns(...a), })); diff --git a/backend/src/workers/extractionWorker.ts b/backend/src/workers/extractionWorker.ts index ec91f5551..9a9331293 100644 --- a/backend/src/workers/extractionWorker.ts +++ b/backend/src/workers/extractionWorker.ts @@ -9,9 +9,9 @@ import { type CellUpdate, } from "../lib/queue/runProgress"; import { getUserModelSettings } from "../lib/userSettings"; -import { extractRowColumns } from "../lib/tabular/tabular.extractRow"; -import { loadReviewRow } from "../lib/tabular/tabular.rows"; -import type { Column } from "../lib/tabular/tabular.shared"; +import { extractRowColumns } from "../modules/tabular/tabular.extractRow"; +import { loadReviewRow } from "../modules/tabular/tabular.rows"; +import type { Column } from "../modules/tabular/tabular.shared"; import { createServerSupabase } from "../lib/supabase"; type Db = ReturnType;