fix: lower mine worker retries (#695)

Author: arcoraven

src/db/transactions/db.ts

@@ -1,7 +1,6 @@
 import superjson from "superjson";
-import { env } from "../../utils/env";
-import { redis } from "../../utils/redis/redis";
-import { AnyTransaction } from "../../utils/transaction/types";
+import { MAX_REDIS_BATCH_SIZE, redis } from "../../utils/redis/redis";
+import type { AnyTransaction } from "../../utils/transaction/types";
 
 /**
  * Schemas
@@ -111,13 +110,9 @@ export class TransactionDB {
     }
 
     const result: AnyTransaction[] = [];
-    for (
-      let i = 0;
-      i < queueIds.length;
-      i += env.__EXPERIMENTAL_REDIS_BATCH_SIZE
-    ) {
+    for (let i = 0; i < queueIds.length; i += MAX_REDIS_BATCH_SIZE) {
       const keys = queueIds
-        .slice(i, i + env.__EXPERIMENTAL_REDIS_BATCH_SIZE)
+        .slice(i, i + MAX_REDIS_BATCH_SIZE)
         .map(this.transactionDetailsKey);
       const vals = await redis.mget(...keys);
 
@@ -141,13 +136,9 @@ export class TransactionDB {
     }
 
     let numDeleted = 0;
-    for (
-      let i = 0;
-      i < queueIds.length;
-      i += env.__EXPERIMENTAL_REDIS_BATCH_SIZE
-    ) {
+    for (let i = 0; i < queueIds.length; i += MAX_REDIS_BATCH_SIZE) {
       const keys = queueIds
-        .slice(i, i + env.__EXPERIMENTAL_REDIS_BATCH_SIZE)
+        .slice(i, i + MAX_REDIS_BATCH_SIZE)
         .map(this.transactionDetailsKey);
       numDeleted += await redis.unlink(...keys);
     }

src/utils/env.ts

@@ -90,18 +90,14 @@ export const env = createEnv({
     // Sets the max amount of memory Redis can use.
     // "0" means use all available memory.
     REDIS_MAXMEMORY: z.string().default("0"),
-    // Sets the max batch Redis will handle in batch operations like MGET and UNLINK.
-    // This will be removed if a consistent batch size works for all use cases.
-    // ioredis has issues with batches over 100k+ (source: https://github.com/redis/ioredis/issues/801).
-    __EXPERIMENTAL_REDIS_BATCH_SIZE: z.coerce.number().default(50_000),
     // Sets the number of recent transactions to store. Older transactions are pruned periodically.
     // In testing, 100k transactions consumes ~300mb memory.
     TRANSACTION_HISTORY_COUNT: z.coerce.number().default(100_000),
     // Sets the number of recent completed jobs in each queue.
     QUEUE_COMPLETE_HISTORY_COUNT: z.coerce.number().default(2_000),
     // Sets the number of recent failed jobs in each queue.
     // These limits are higher to debug failed jobs.
-    QUEUE_FAIL_HISTORY_COUNT: z.coerce.number().default(20_000),
+    QUEUE_FAIL_HISTORY_COUNT: z.coerce.number().default(10_000),
     // Sets the number of recent nonces to map to queue IDs.
     NONCE_MAP_COUNT: z.coerce.number().default(10_000),
   },
@@ -134,8 +130,6 @@ export const env = createEnv({
       process.env.CONFIRM_TRANSACTION_QUEUE_CONCURRENCY,
     ENGINE_MODE: process.env.ENGINE_MODE,
     REDIS_MAXMEMORY: process.env.REDIS_MAXMEMORY,
-    __EXPERIMENTAL_REDIS_BATCH_SIZE:
-      process.env.__EXPERIMENTAL_REDIS_BATCH_SIZE,
     TRANSACTION_HISTORY_COUNT: process.env.TRANSACTION_HISTORY_COUNT,
     GLOBAL_RATE_LIMIT_PER_MIN: process.env.GLOBAL_RATE_LIMIT_PER_MIN,
     DD_TRACER_ACTIVATED: process.env.DD_TRACER_ACTIVATED,

src/utils/redis/redis.ts

@@ -2,6 +2,9 @@ import Redis from "ioredis";
 import { env } from "../env";
 import { logger } from "../logger";
 
+// ioredis has issues with batches over 100k+ (source: https://github.com/redis/ioredis/issues/801).
+export const MAX_REDIS_BATCH_SIZE = 50_000;
+
 export const redis = new Redis(env.REDIS_URL, {
   enableAutoPipelining: true,
   maxRetriesPerRequest: null,

src/worker/queues/mineTransactionQueue.ts

@@ -23,7 +23,7 @@ export class MineTransactionQueue {
     const jobId = this.jobId(data);
     await this.q.add(jobId, serialized, {
       jobId,
-      attempts: 200, // > 30 minutes with the backoffStrategy defined on the worker
+      attempts: 100, // > 30 minutes with the backoffStrategy defined on the worker
       backoff: { type: "custom" },
     });
   };

src/worker/tasks/mineTransactionWorker.ts

@@ -271,8 +271,8 @@ export const initMineTransactionWorker = () => {
     connection: redis,
     settings: {
       backoffStrategy: (attemptsMade: number) => {
-        // Retries after: 2s, 4s, 6s, 8s, 10s, 10s, 10s, 10s, ...
-        return Math.min(attemptsMade * 2_000, 10_000);
+        // Retries at 2s, 4s, 6s, ..., 18s, 20s, 20s, 20s, ...
+        return Math.min(attemptsMade * 2_000, 20_000);
       },
     },
   });