[Feat] API Server 프로메테우스/그라파나 모니터링 추가

apps/api/.env.example

@@ -61,4 +61,8 @@ RABBITMQ_RECORD_SYNC_QUEUE=
 # Firebase
 FIREBASE_PROJECT_ID=
 FIREBASE_CLIENT_EMAIL=
-FIREBASE_PRIVATE_KEY=
+FIREBASE_PRIVATE_KEY=
+
+# Grafana
+GRAFANA_ADMIN_USER=
+GRAFANA_ADMIN_PASSWORD=

apps/api/docker-compose-dev.yml

@@ -63,10 +63,48 @@ services:
     networks:
       - locus-network
 
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: locus-prometheus
+    ports:
+      - '9090:9090'
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+      - '--web.console.templates=/usr/share/prometheus/consoles'
+    restart: unless-stopped
+    networks:
+      - locus-network
+
+  # Grafana 대시보드
+  grafana:
+    image: grafana/grafana:latest
+    container_name: locus-grafana
+    ports:
+      - '3001:3000'
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin123
+      # - GF_USERS_ALLOW_SIGN_UP=false
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning
+    restart: unless-stopped
+    networks:
+      - locus-network
+    depends_on:
+      - prometheus
+
 volumes:
   elasticsearch_data:
   postgres_data:
   redis_data:
+  prometheus_data:
+  grafana_data:
 
 networks:
   locus-network:

apps/api/package.json

@@ -41,6 +41,7 @@
     "@nestjs/swagger": "^11.2.4",
     "@prisma/adapter-pg": "^7.2.0",
     "@prisma/client": "^7.2.0",
+    "@willsoto/nestjs-prometheus": "^6.0.2",
     "amqp-connection-manager": "^5.0.0",
     "amqplib": "^0.10.9",
     "axios": "^1.13.4",
@@ -57,6 +58,7 @@
     "passport-kakao": "^1.0.1",
     "passport-oauth2": "^1.8.0",
     "pg": "^8.16.3",
+    "prom-client": "^15.1.3",
     "redis": "^5.10.0",
     "reflect-metadata": "^0.2.2",
     "rxjs": "^7.8.1",

apps/api/prometheus.yml

@@ -0,0 +1,18 @@
+global:
+  scrape_interval: 15s # 15초마다 메트릭 수집
+  evaluation_interval: 15s
+
+scrape_configs:
+  # NestJS 애플리케이션
+  - job_name: 'nestjs-app'
+    static_configs:
+      - targets: ['host.docker.internal:3000'] # Docker 내부에서 호스트 접근
+        labels:
+          service: 'nestjs'
+          env: 'development'
+    metrics_path: '/api/metrics' # GET /api/metrics 엔드포인트
+
+  # Prometheus 자체 모니터링
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:9090']

apps/api/src/app.module.ts

@@ -19,13 +19,15 @@ import { TagsModule } from './tags/tags.module';
 import { MapsModule } from './maps/maps.module';
 import { NotificationModule } from './notification/notification.module';
 import { ImagesModule } from './images/images.module';
+import { PrometheusModule } from './infra/monitoring/prometheus.module';
 import { DuckModule } from './duck/duck.module';
 
 @Module({
   imports: [
     ScheduleModule.forRoot(),
     ConfigModule.forRoot({ isGlobal: true }),
     PrismaModule,
+    PrometheusModule,
     RabbitMqModule,
     OutboxModule,
     RedisModule,

apps/api/src/common/interceptors/response-tranform.interceptor.ts

@@ -7,6 +7,7 @@ import {
 import { map, Observable } from 'rxjs';
 import { ApiResponseType } from '../type/api-response.types';
 import { ApiResponse } from '../utils/api-response.helper';
+import { Request } from 'express';
 
 @Injectable()
 export class ResponseTransformInterceptor<T>
@@ -16,6 +17,10 @@ export class ResponseTransformInterceptor<T>
     context: ExecutionContext,
     next: CallHandler,
   ): Observable<ApiResponseType<T>> {
+    const request = context.switchToHttp().getRequest<Request>();
+    if (request.url.includes('/metrics')) {
+      return next.handle() as Observable<ApiResponseType<T>>;
+    }
     return next.handle().pipe(map((data: T) => ApiResponse.success(data)));
   }
 }

apps/api/src/infra/monitoring/constants/metrics.constants.ts

@@ -0,0 +1,24 @@
+export const API_METRICS = {
+  HTTP_REQUESTS_TOTAL: 'http_requests_total',
+  HTTP_REQUEST_DURATION_SEC: 'http_request_duration_seconds',
+};
+
+export const OUTBOX_METRICS = {
+  OUTBOX_EVENTS_PUBLISHED_TOTAL: 'outbox_events_published_total',
+  STATUS_TRANSITIONS_TOTAL: 'outbox_status_transitions_total',
+  OUTBOX_PROCESSING_DURATION_SEC: 'outbox_processing_duration_seconds',
+  OUTBOX_DEAD_LETTER_TOTAL: 'outbox_dead_letter_total',
+};
+
+export const RABBITMQ_METRICS = {
+  MESSAGES_PUBLISHED_TOTAL: 'messages_published_total',
+  MESSAGES_CONSUMED_TOTAL: 'messages_consumed_total',
+  MESSAGES_PROCESSING_DURATION_SEC: 'message_processing_duration_seconds',
+  MESSAGES_IN_FLIGHT: 'messages_in_flight',
+};
+
+export const ELASTICSEARCH_METRICS = {
+  OPERATIONS_TOTAL: 'elasticsearch_operations_total',
+  OPERATION_DURATION_SEC: 'elasticsearch_operation_duration_seconds',
+  RECORD_SYNC_EVENTS_TOTAL: 'record_sync_events_total',
+};

apps/api/src/infra/monitoring/interceptor/api-metrics.interceptor.ts

@@ -0,0 +1,61 @@
+import {
+  Injectable,
+  NestInterceptor,
+  ExecutionContext,
+  CallHandler,
+} from '@nestjs/common';
+import { Observable } from 'rxjs';
+import { tap } from 'rxjs/operators';
+import { ApiMetricsService } from '../services/api-metrics.service';
+import { Request, Response } from 'express';
+
+@Injectable()
+export class ApiMetricsInterceptor implements NestInterceptor {
+  private readonly EXCLUDE_PATHS = ['/api/metrics'];
+
+  constructor(private readonly apiMetricsService: ApiMetricsService) {}
+
+  intercept(context: ExecutionContext, next: CallHandler): Observable<any> {
+    const request = context.switchToHttp().getRequest<Request>();
+    const method: string = request.method;
+    const url: string = request.url;
+    const route = request.route as { path: string } | undefined;
+    const path: string = route?.path ?? url;
+
+    if (this.EXCLUDE_PATHS.some((excludePath) => path.includes(excludePath))) {
+      return next.handle();
+    }
+
+    const startTime = Date.now();
+
+    return next.handle().pipe(
+      tap({
+        next: () => {
+          const response = context.switchToHttp().getResponse<Response>();
+          const statusCode = response.statusCode;
+          const duration = (Date.now() - startTime) / 1000;
+          this.recordApiMetrics(method, path, statusCode, duration);
+        },
+        error: (error: { status?: number; statusCode?: number }) => {
+          const statusCode = error.status ?? error.statusCode ?? 500;
+          const duration = (Date.now() - startTime) / 1000;
+          this.recordApiMetrics(method, path, statusCode, duration);
+        },
+      }),
+    );
+  }
+  private recordApiMetrics(
+    method: string,
+    path: string,
+    status: number,
+    duration: number,
+  ) {
+    this.apiMetricsService.recordRequest(method, path, status);
+    this.apiMetricsService.recordAPIDurationTime(
+      method,
+      path,
+      status,
+      duration,
+    );
+  }
+}

apps/api/src/infra/monitoring/metrics.provider.ts

@@ -0,0 +1,110 @@
+import {
+  makeCounterProvider,
+  makeGaugeProvider,
+  makeHistogramProvider,
+} from '@willsoto/nestjs-prometheus';
+import {
+  API_METRICS,
+  ELASTICSEARCH_METRICS,
+  OUTBOX_METRICS,
+  RABBITMQ_METRICS,
+} from './constants/metrics.constants';
+
+export const metricsProviders = [
+  // HTTP 요청 총 개수
+  makeCounterProvider({
+    name: API_METRICS.HTTP_REQUESTS_TOTAL,
+    help: 'Total number of HTTP requests',
+    labelNames: ['method', 'path', 'status'],
+  }),
+
+  // HTTP 요청 응답 시간 (히스토그램)
+  makeHistogramProvider({
+    name: API_METRICS.HTTP_REQUEST_DURATION_SEC,
+    help: 'Duration of HTTP requests in seconds',
+    labelNames: ['method', 'path', 'status'],
+    buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
+  }),
+];
+
+export const outboxMetricsProviders = [
+  makeCounterProvider({
+    name: OUTBOX_METRICS.OUTBOX_EVENTS_PUBLISHED_TOTAL,
+    help: 'Total number of outbox events published to RabbitMQ',
+    labelNames: ['status', 'event_type'],
+  }),
+
+  makeCounterProvider({
+    name: OUTBOX_METRICS.STATUS_TRANSITIONS_TOTAL,
+    help: 'Total number of outbox status transitions',
+    labelNames: ['from_status', 'to_status', 'event_type'],
+  }),
+
+  makeHistogramProvider({
+    name: OUTBOX_METRICS.OUTBOX_PROCESSING_DURATION_SEC,
+    help: 'Duration from outbox event creation to publication',
+    labelNames: ['event_type'],
+    buckets: [0.1, 0.5, 1, 5, 10, 30, 60],
+  }),
+
+  makeCounterProvider({
+    name: OUTBOX_METRICS.OUTBOX_DEAD_LETTER_TOTAL,
+    help: 'Total number of events moved to dead letter queue',
+    labelNames: ['event_type'],
+  }),
+];
+
+export const rabbitMQMetricsProviders = [
+  makeCounterProvider({
+    name: RABBITMQ_METRICS.MESSAGES_PUBLISHED_TOTAL,
+    help: 'Total number of messages published',
+    labelNames: ['pattern', 'status'],
+  }),
+
+  makeCounterProvider({
+    name: RABBITMQ_METRICS.MESSAGES_CONSUMED_TOTAL,
+    help: 'Total number of messages consumed',
+    labelNames: ['pattern', 'status'],
+  }),
+
+  makeHistogramProvider({
+    name: RABBITMQ_METRICS.MESSAGES_PROCESSING_DURATION_SEC,
+    help: 'Duration of message processing',
+    labelNames: ['pattern', 'event_type'],
+    buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10],
+  }),
+
+  makeGaugeProvider({
+    name: RABBITMQ_METRICS.MESSAGES_IN_FLIGHT,
+    help: 'Number of messages currently in flight',
+    labelNames: ['pattern'],
+  }),
+];
+
+export const elasticSearchMetricsProviders = [
+  makeCounterProvider({
+    name: ELASTICSEARCH_METRICS.OPERATIONS_TOTAL,
+    help: 'Total number of Elasticsearch operations',
+    labelNames: ['operation', 'status'],
+  }),
+
+  makeHistogramProvider({
+    name: ELASTICSEARCH_METRICS.OPERATION_DURATION_SEC,
+    help: 'Duration of Elasticsearch operations',
+    labelNames: ['operation'],
+    buckets: [0.01, 0.1, 0.5, 1, 5],
+  }),
+
+  makeCounterProvider({
+    name: ELASTICSEARCH_METRICS.RECORD_SYNC_EVENTS_TOTAL,
+    help: 'Total number of record sync events processed',
+    labelNames: ['event_type'],
+  }),
+];
+
+export const allMetricsProviders = [
+  ...metricsProviders,
+  ...outboxMetricsProviders,
+  ...rabbitMQMetricsProviders,
+  ...elasticSearchMetricsProviders,
+];

apps/api/src/infra/monitoring/prometheus.module.ts

@@ -0,0 +1,37 @@
+import { Module } from '@nestjs/common';
+import { PrometheusModule as NestPrometheusModule } from '@willsoto/nestjs-prometheus';
+import { allMetricsProviders } from './metrics.provider';
+import { ApiMetricsInterceptor } from '@/infra/monitoring/interceptor/api-metrics.interceptor';
+import { APP_INTERCEPTOR } from '@nestjs/core';
+import { ApiMetricsService } from './services/api-metrics.service';
+import { OutboxMetricsService } from './services/outbox-metrics.service';
+import { RabbitMQMetricsService } from './services/rabbitmq-metrics.service';
+import { ElasticsearchMetricsService } from './services/elasticsearch-metrics.service';
+
+@Module({
+  imports: [
+    NestPrometheusModule.register({
+      defaultMetrics: { enabled: true },
+      path: '/metrics',
+      defaultLabels: { app: 'locus' },
+    }),
+  ],
+  providers: [
+    ...allMetricsProviders,
+    ApiMetricsService,
+    OutboxMetricsService,
+    RabbitMQMetricsService,
+    ElasticsearchMetricsService,
+    {
+      provide: APP_INTERCEPTOR,
+      useClass: ApiMetricsInterceptor,
+    },
+  ],
+  exports: [
+    NestPrometheusModule,
+    OutboxMetricsService,
+    RabbitMQMetricsService,
+    ElasticsearchMetricsService,
+  ],
+})
+export class PrometheusModule {}

apps/api/src/infra/monitoring/services/api-metrics.service.ts

@@ -0,0 +1,32 @@
+import { Injectable } from '@nestjs/common';
+import { InjectMetric } from '@willsoto/nestjs-prometheus';
+import { Counter, Histogram } from 'prom-client';
+import { API_METRICS } from '../constants/metrics.constants';
+
+@Injectable()
+export class ApiMetricsService {
+  constructor(
+    @InjectMetric(API_METRICS.HTTP_REQUESTS_TOTAL)
+    private readonly requestsTotal: Counter<string>,
+
+    @InjectMetric(API_METRICS.HTTP_REQUEST_DURATION_SEC)
+    private readonly requestDuration: Histogram<string>,
+  ) {}
+
+  // request 기록
+  recordRequest(method: string, path: string, statusCode: number): void {
+    this.requestsTotal.inc({ method, path, status: statusCode.toString() });
+  }
+
+  recordAPIDurationTime(
+    method: string,
+    path: string,
+    statusCode: number,
+    durationSeconds: number,
+  ): void {
+    this.requestDuration.observe(
+      { method, path, status: statusCode.toString() },
+      durationSeconds,
+    );
+  }
+}