From c8ae5b58b947b3b25d989d11f9bb417205a4e5c2 Mon Sep 17 00:00:00 2001
From: Lorenzo Galassi <lorenzo.galassi@quinck.io>
Date: Wed, 10 Jun 2026 21:36:00 +0200
Subject: [PATCH] fix(queue): share one ioredis connection across BullMQ queues
 and workers

BullMQ instantiates a fresh ioredis client per Queue/Worker when handed a
plain {host, port} config object, and under sustained ZIM ingestion the
embed pipeline leaked ~1 client/sec until Redis maxclients was exhausted.
Pass a single shared ioredis instance (maxRetriesPerRequest: null, as
required by BullMQ) so all queues and workers reuse one client pool.
Workers still duplicate the connection once for their blocking client,
which is expected and bounded.

Closes #885
---
 admin/app/services/queue_service.ts | 10 +++++-----
 admin/config/queue.ts               | 22 +++++++++++++++++-----
 admin/package-lock.json             |  1 +
 admin/package.json                  |  1 +
 4 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/admin/app/services/queue_service.ts b/admin/app/services/queue_service.ts
index bad976b9..5e31e2f6 100644
--- a/admin/app/services/queue_service.ts
+++ b/admin/app/services/queue_service.ts
@@ -1,11 +1,11 @@
 import { Queue } from 'bullmq'
 import queueConfig from '#config/queue'
 
-// Process-wide singleton. Each `Queue` opens two ioredis connections (one for
-// commands, one blocking). Instantiating a fresh QueueService per dispatch /
-// status lookup leaks both, and under sustained job churn (e.g. multi-batch ZIM
-// ingestion enqueueing a continuation every few seconds) it saturates Redis's
-// maxclients within hours.
+// Process-wide singleton. Instantiating a fresh QueueService per dispatch /
+// status lookup leaks connections, and under sustained job churn (e.g.
+// multi-batch ZIM ingestion enqueueing a continuation every few seconds) it
+// saturates Redis's maxclients within hours. All queues additionally reuse the
+// single shared ioredis instance exported from #config/queue (#885).
 export class QueueService {
   private queues: Map<string, Queue> = new Map()
 
diff --git a/admin/config/queue.ts b/admin/config/queue.ts
index 02745637..3c6a112a 100644
--- a/admin/config/queue.ts
+++ b/admin/config/queue.ts
@@ -1,11 +1,23 @@
 import env from '#start/env'
+import { Redis } from 'ioredis'
+
+// BullMQ treats a plain `{host, port}` connection object as a recipe: every
+// Queue / Worker instantiates its own ioredis client from it, and script
+// commands executed against those clients can spawn further short-lived
+// connections. Under sustained ZIM ingestion this leaked ~1 client/sec until
+// Redis maxclients was exhausted (#885). Passing a single shared ioredis
+// instance instead gives BullMQ a pool to reuse — Workers still duplicate it
+// once for their blocking client, which is expected and bounded.
+// `maxRetriesPerRequest: null` is mandatory for connections shared with BullMQ.
+const sharedConnection = new Redis({
+  host: env.get('REDIS_HOST'),
+  port: env.get('REDIS_PORT') ?? 6379,
+  db: env.get('REDIS_DB') ?? 0,
+  maxRetriesPerRequest: null,
+})
 
 const queueConfig = {
-  connection: {
-    host: env.get('REDIS_HOST'),
-    port: env.get('REDIS_PORT') ?? 6379,
-    db: env.get('REDIS_DB') ?? 0,
-  },
+  connection: sharedConnection,
 }
 
 export default queueConfig
diff --git a/admin/package-lock.json b/admin/package-lock.json
index 6e3c1a19..a26b28c2 100644
--- a/admin/package-lock.json
+++ b/admin/package-lock.json
@@ -47,6 +47,7 @@
         "edge.js": "6.4.0",
         "fast-xml-parser": "5.7.0",
         "fuse.js": "7.1.0",
+        "ioredis": "5.10.1",
         "ipaddr.js": "2.4.0",
         "jszip": "3.10.1",
         "luxon": "3.7.2",
diff --git a/admin/package.json b/admin/package.json
index bb0146be..f4635aef 100644
--- a/admin/package.json
+++ b/admin/package.json
@@ -100,6 +100,7 @@
     "edge.js": "6.4.0",
     "fast-xml-parser": "5.7.0",
     "fuse.js": "7.1.0",
+    "ioredis": "5.10.1",
     "ipaddr.js": "2.4.0",
     "jszip": "3.10.1",
     "luxon": "3.7.2",