perf: use a redis cluster (#596)

Author: MartinKolarik

.dockerignore

@@ -5,3 +5,4 @@ coverage/
 dist/
 node_modules/
 probes-stats/
+test/

.github/workflows/ci.yml

@@ -15,16 +15,6 @@ jobs:
       NODE_ENV: test
 
     services:
-      redis:
-        image: redis/redis-stack-server:latest
-        ports:
-          - 16379:6379
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
       mariadb:
         image: mariadb:10.11.5
         ports:
@@ -45,6 +35,10 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: 20.x
+      - name: Set up Redis
+        run: |
+          cp config/redis/.env.redis ./
+          docker compose up -d
       - name: Build
         run: |
           npm ci

.github/workflows/e2e.yml

@@ -15,16 +15,6 @@ jobs:
       NODE_ENV: test
 
     services:
-      redis:
-        image: redis/redis-stack-server:latest
-        ports:
-          - 16379:6379
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
       mariadb:
         image: mariadb:10.11.5
         ports:
@@ -51,6 +41,10 @@ jobs:
           sudo mv daemon.json /etc/docker/
           sudo systemctl restart docker
           docker restart $(docker ps -aq)
+      - name: Set up Redis
+        run: |
+          cp config/redis/.env.redis ./
+          docker compose up -d
       - name: Build
         run: |
           npm ci

.gitignore

@@ -6,6 +6,7 @@ coverage/
 dist/
 tmp/
 config/local*
+data/redis
 data/DOMAIN_BLACKLIST.json
 data/IP_BLACKLIST.json
 data/AWS_IP_RANGES.json
@@ -18,3 +19,4 @@ probes-stats/all-result.csv
 probes-stats/all-result.json
 .eslintcache
 .env
+/.env.redis

.husky/pre-commit

@@ -6,4 +6,4 @@ if ! type cygpath > /dev/null 2>&1; then
 	PATH="$PATH:$(type -ap git | grep 'cmd/git' | sed 's$cmd/git$usr/bin$')"
 fi
 
-node_modules/.bin/lint-staged --no-stash --quiet
+node_modules/.bin/lint-staged --quiet

CONTRIBUTING.md

@@ -10,16 +10,17 @@ Hi! We're really excited that you're interested in contributing! Before submitti
 
 ## Project setup
 
-In order to run the Globalping API locally you will need Node.js 20 and Redis with [RedisJSON](https://oss.redis.com/redisjson/) module and MariaDB. All of them are included in docker-compose.yml file. You will also need to run a development instance of the [Globalping Probe](https://github.com/jsdelivr/globalping-probe) at the same time when testing.
+In order to run the Globalping API locally you will need Node.js 20 and Redis with [RedisJSON](https://oss.redis.com/redisjson/) module and MariaDB. All of them are included in docker-compose.dev.yml file. You will also need to run a development instance of the [Globalping Probe](https://github.com/jsdelivr/globalping-probe) at the same time when testing.
 
 The API uses 3000 port by default. This can be overridden by `PORT` environment variable.
 
 You can run the project by following these steps:
 
 1. Clone this repository.
-2. `docker-compose up -d` - Run Redis and MariaDB
-3. `npm install && npm run download:files`
-4. Run `npm run start:dev`
+2. [Enable host networking in Docker Desktop](https://docs.docker.com/engine/network/drivers/host/#docker-desktop) if you haven't already.
+3. `docker compose -f docker-compose.dev.yml up -d` - Run Redis and MariaDB
+4. `npm install && npm run download:files`
+5. Run `npm run start:dev`
 
 Once the API is live, you can spin up a probe instance by running as described at https://github.com/jsdelivr/globalping-probe/blob/master/CONTRIBUTING.md.
 
@@ -48,3 +49,4 @@ Most IDEs have plugins integrating the used linter (eslint), including support f
 - `SYSTEM_API_KEY={value}` used for integration with the dashboard
 - `SERVER_SESSION_COOKIE_SECRET={value}` used to read the shared session cookie
 - `DB_CONNECTION_HOST`, `DB_CONNECTION_USER`, `DB_CONNECTION_PASSWORD`, and `DB_CONNECTION_DATABASE` database connection details
+- `REDIS_STANDALONE_PERSISTENT_URL`, `REDIS_STANDALONE_NON_PERSISTENT_URL`, `REDIS_CLUSTER_MEASUREMENTS_NODES_0`, `REDIS_CLUSTER_MEASUREMENTS_NODES_1`, `REDIS_CLUSTER_MEASUREMENTS_NODES_2`, and `REDIS_SHARED_OPTIONS_PASSWORD` - redis connection details

Dockerfile

@@ -1,7 +1,7 @@
 FROM node:20-bullseye-slim AS builder
 RUN apt-get update -y && apt-get install util-linux curl git -y
 
-ENV NODE_ENV production
+ENV NODE_ENV=production
 
 COPY package.json package-lock.json /app/
 WORKDIR /app
@@ -12,7 +12,7 @@ RUN npm run build
 FROM node:20-bullseye-slim
 RUN apt-get update -y && apt-get install tini util-linux curl -y
 
-ENV NODE_ENV production
+ENV NODE_ENV=production
 
 COPY package.json package-lock.json /app/
 WORKDIR /app

config/custom-environment-variables.cjs

@@ -1,25 +1,2 @@
-const _ = require('lodash');
 const df = require('./default.cjs');
-
-function mapEnvConfig (object, prefix = '') {
-	return _.mapValues(object, (value, key) => {
-		const currentKey = (prefix ? `${prefix}_` : '') + _.snakeCase(key).toUpperCase();
-
-		if (_.isObject(value)) {
-			return mapEnvConfig(value, currentKey);
-		}
-
-		if (typeof value === 'number' || typeof value === 'boolean') {
-			return {
-				__name: currentKey,
-				__format: typeof value,
-			};
-		}
-
-		return currentKey;
-	});
-}
-
-const mapped = mapEnvConfig(df);
-
-module.exports = mapped;
+module.exports = require('config-mapper-env')(df);

config/default.cjs

@@ -17,9 +17,26 @@ module.exports = {
 		},
 	},
 	redis: {
-		url: 'redis://localhost:6379',
-		socket: {
-			tls: false,
+		standalonePersistent: {
+			url: 'redis://localhost:7001',
+		},
+		standaloneNonPersistent: {
+			url: 'redis://localhost:7002',
+		},
+		clusterMeasurements: {
+			// listing three nodes here is enough, the rest will be discovered automatically
+			nodes: {
+				0: 'redis://localhost:7101',
+				1: 'redis://localhost:7102',
+				2: 'redis://localhost:7103',
+			},
+			options: {},
+		},
+		sharedOptions: {
+			password: 'PASSWORD',
+			socket: {
+				tls: false,
+			},
 		},
 	},
 	db: {

config/development.cjs

@@ -4,12 +4,6 @@ module.exports = {
 			cookieSecret: 'xxx',
 		},
 	},
-	redis: {
-		url: 'redis://localhost:16379',
-		socket: {
-			tls: false,
-		},
-	},
 	db: {
 		connection: {
 			port: 13306,

config/redis/.env.redis

@@ -0,0 +1,6 @@
+# standalone:
+REDIS_ARGS="--requirepass PASSWORD"
+
+# cluster:
+REDIS_PASSWORD=PASSWORD
+REDIS_PUBLIC_IP=127.0.0.1

redis/setup.md config/redis/README.md

@@ -12,9 +12,16 @@ swapon /swapfile
 echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab
 ```
 
-## Config
+### Docker config
 
-1. Download redis.conf to /etc/redis/
-2. Download zip file with json module to same folder
-3. Unzip
-4. Restart
+Assuming you start in this directory:
+
+```
+cp .env.redis ../../
+```
+
+Set the redis password and return to the project root. Then:
+
+```
+docker compose up -d
+```