feat: webhooks implementation & queue system

services/workflows-service/docker-compose.redis.yml

@@ -0,0 +1,22 @@
+version: '3.8'
+services:
+  redis:
+    image: redis:alpine
+    ports:
+      - '${REDIS_PORT}:6379'
+    volumes:
+      - redis-data:/data
+    command: >
+      --requirepass ${REDIS_PASSWORD}
+      --appendonly yes
+    environment:
+      - REDIS_PASSWORD=${REDIS_PASSWORD}
+      - REDIS_PORT=${REDIS_PORT}
+    networks:
+      - app-network
+volumes:
+  redis-data:
+    driver: local
+networks:
+  app-network:
+    driver: bridge

services/workflows-service/package.json

@@ -5,7 +5,7 @@
   "description": "workflow-service",
   "scripts": {
     "spellcheck": "cspell \"*\"",
-    "setup": "npm run docker:db:down && npm run docker:db && wait-on tcp:5432 && npm run db:reset && npm run seed",
+    "setup": "npm run docker:db:down && npm run docker:db && wait-on tcp:5432 && npm run docker:redis:down && npm run docker:redis && npm run db:reset && npm run seed",
     "format": "prettier --write . '!**/*.{md,hbs}'",
     "format:check": "prettier --check . '!**/*.{md,hbs}'",
     "lint": "eslint . --fix",
@@ -32,6 +32,8 @@
     "db:reset:dev:with-data": "npm run db:reset:dev && npm run db:data-migration:migrate && npm run db:data-sync",
     "db:init": "npm run db:migrate-dev -- --name 'initial version' && npm run db:migrate-up seed",
     "prisma:generate": "prisma generate",
+    "docker:redis": "docker compose -f docker-compose.redis.yml up -d --wait",
+    "docker:redis:down": "docker compose -f docker-compose.redis.yml down --volumes",
     "docker:db": "docker compose -f docker-compose.db.yml up -d --wait",
     "docker:db:down": "docker compose -f docker-compose.db.yml down --volumes",
     "docker:build": "docker build .",
@@ -53,6 +55,8 @@
     "@ballerine/common": "0.9.70",
     "@ballerine/workflow-core": "0.6.89",
     "@ballerine/workflow-node-sdk": "0.6.89",
+    "@bull-board/api": "^6.7.1",
+    "@bull-board/express": "^6.7.1",
     "@faker-js/faker": "^7.6.0",
     "@nestjs/axios": "^2.0.0",
     "@nestjs/common": "^9.3.12",
@@ -81,6 +85,7 @@
     "ballerine-nestjs-typebox": "3.0.2-next.11",
     "base64-stream": "^1.0.0",
     "bcrypt": "5.1.0",
+    "bullmq": "^5.13.2",
     "class-transformer": "0.5.1",
     "class-validator": "0.14.0",
     "concat-stream": "^2.0.0",
@@ -93,6 +98,7 @@
     "helmet": "^6.0.1",
     "i18n-iso-countries": "^7.6.0",
     "i18next": "^22.4.9",
+    "ioredis": "^5.4.2",
     "is-what": "^4.1.15",
     "js-base64": "^3.7.6",
     "json-stable-stringify": "^1.1.1",

services/workflows-service/src/alert/alert.module.ts

@@ -10,20 +10,16 @@ import { AlertRepository } from '@/alert/alert.repository';
 import { AlertService } from '@/alert/alert.service';
 import { AlertControllerExternal } from '@/alert/alert.controller.external';
 import { PrismaModule } from '@/prisma/prisma.module';
-import {
-  WebhookHttpService,
-  WebhookManagerService,
-} from '@/alert/webhook-manager/webhook-manager.service';
 import { HttpModule, HttpService } from '@nestjs/axios';
 import { AppLoggerService } from '@/common/app-logger/app-logger.service';
 import axiosRetry from 'axios-retry';
 import { isAxiosError } from 'axios';
 import { getHttpStatusFromAxiosError, interceptAxiosRequests } from '@/common/http-service/utils';
-import { WebhookEventEmitterService } from './webhook-manager/webhook-event-emitter.service';
 import { ProjectModule } from '@/project/project.module';
 import { UserRepository } from '@/user/user.repository';
 import { AlertDefinitionModule } from '@/alert-definition/alert-definition.module';
 import { SentryModule } from '@/sentry/sentry.module';
+import { WebhooksModule } from '@/webhooks/webhooks.module';
 
 @Module({
   imports: [
@@ -32,6 +28,7 @@ import { SentryModule } from '@/sentry/sentry.module';
     PrismaModule,
     SentryModule,
     ProjectModule,
+    WebhooksModule,
     HttpModule.register({
       timeout: 5000,
       maxRedirects: 10,
@@ -48,21 +45,15 @@ import { SentryModule } from '@/sentry/sentry.module';
   ],
   controllers: [AlertControllerInternal, AlertControllerExternal],
   providers: [
-    {
-      provide: WebhookHttpService,
-      useExisting: HttpService,
-    },
     AlertService,
     AlertRepository,
     AlertDefinitionRepository,
-    WebhookManagerService,
-    WebhookEventEmitterService,
     // TODO: Export to user module
     UserService,
     UserRepository,
     PasswordService,
   ],
-  exports: [ACLModule, AlertRepository, AlertService, WebhookEventEmitterService],
+  exports: [ACLModule, AlertRepository, AlertService],
 })
 export class AlertModule {
   constructor(

services/workflows-service/src/app.module.ts

@@ -32,18 +32,19 @@ import { CollectionFlowModule } from '@/collection-flow/collection-flow.module';
 import { SalesforceModule } from '@/salesforce/salesforce.module';
 import { UiDefinitionModule } from '@/ui-definition/ui-definition.module';
 import { multerFactory } from '@/common/multer';
-import { initHttpMoudle } from '@/common/http-service/http-config.service';
+import { initHttpModule } from '@/common/http-service/http-config.service';
 import { DataMigrationModule } from '@/data-migration/data-migration.module';
 import { CaseManagementModule } from '@/case-management/case-management.module';
 import { WorkflowModule } from '@/workflow/workflow.module';
 import { TransactionModule } from '@/transaction/transaction.module';
 import { AlertModule } from '@/alert/alert.module';
 import { SwaggerController } from './swagger/swagger.controller';
 import { WebhooksModule } from '@/webhooks/webhooks.module';
+import { IncomingWebhooksModule } from '@/webhooks-incoming/webhooks-incoming.module';
 import { BusinessReportModule } from '@/business-report/business-report.module';
 import { ScheduleModule } from '@nestjs/schedule';
 import { CronModule } from '@/workflow/cron/cron.module';
-import z from 'zod';
+import { z } from 'zod';
 import { hashKey } from './customer/api-key/utils';
 import { RuleEngineModule } from './rule-engine/rule-engine.module';
 import { NotionModule } from '@/notion/notion.module';
@@ -92,6 +93,7 @@ export const validate = async (config: Record<string, unknown>) => {
     UserModule,
     WorkflowModule,
     WebhooksModule,
+    IncomingWebhooksModule,
     NoteModule,
     UiDefinitionModule,
     StorageModule,
@@ -129,7 +131,7 @@ export const validate = async (config: Record<string, unknown>) => {
     BusinessReportModule,
     CronModule,
     ScheduleModule.forRoot(),
-    initHttpMoudle(),
+    initHttpModule(),
     RuleEngineModule,
     NotionModule,
     SecretsManagerModule,

services/workflows-service/src/business-report/business-report.controller.external.ts

@@ -18,7 +18,7 @@ import { CustomerService } from '@/customer/customer.service';
 import { BusinessService } from '@/business/business.service';
 import { CurrentProject } from '@/common/decorators/current-project.decorator';
 import type { TProjectId } from '@/types';
-import { GetLatestBusinessReportDto } from '@/business-report/get-latest-business-report.dto';
+import { GetLatestBusinessReportDto } from '@/business-report/dtos/get-latest-business-report.dto';
 import {
   BusinessReportListRequestParamDto,
   BusinessReportListResponseDto,

services/workflows-service/src/common/http-service/http-config.service.ts

@@ -23,7 +23,7 @@ export class HttpConfigService implements HttpModuleOptionsFactory {
   }
 }
 
-export const initHttpMoudle = () =>
+export const initHttpModule = () =>
   HttpModule.registerAsync({
     imports: [ConfigModule],
     // useClass: HttpConfigService,

services/workflows-service/src/env.ts

@@ -94,6 +94,13 @@ export const serverEnvSchema = {
   IN_MEMORIES_SECRET_PRIVATE_KEY: z.string().optional(),
   IN_MEMORIES_SECRET_CONSUMER_KEY: z.string().optional(),
   SYNC_UNIFIED_API: z.string().optional().default('true'),
+  REDIS_HOST: z.string(),
+  REDIS_PORT: z.coerce.number(),
+  REDIS_PASSWORD: z.string().optional(),
+  QUEUE_SYSTEM_ENABLED: z
+    .string()
+    .optional()
+    .transform(value => value === 'true'),
 };
 
 if (!process.env['ENVIRONMENT_NAME'] || process.env['ENVIRONMENT_NAME'] === 'local') {