diff --git a/docs/analytics.md b/docs/analytics.md new file mode 100644 index 00000000..c36c2f40 --- /dev/null +++ b/docs/analytics.md @@ -0,0 +1,71 @@ +# Analytics Module + +The Analytics Module provides event tracking and reporting capabilities for the StellAIverse platform. + +## Event Schema + +All events are stored in the `analytics_events` table and represented by the `AnalyticsEvent` entity. + +### Standard Properties +- `eventType`: Enum specifying the type of event (`page_view`, `click`, `transaction`, etc.) +- `eventName`: String describing the specific event (e.g., "submit_order") +- `userId`: Identifier for the user who triggered the event +- `sessionId`: Session identifier +- `properties`: JSON object for arbitrary custom event data +- `idempotencyKey`: Unique string to prevent duplicate ingestion + +### Context Properties +- `page`: Page path or URL +- `referrer`: Referrer URL +- `userAgent`: Raw user agent string +- `device`, `browser`, `os`: Parsed client metadata +- `ipAddress`, `country`: Location data + +## Retention Policy + +Events are stored in the primary transactional database (`analytics_events` table). +- Raw events are kept indefinitely by default, but a cleanup job could be introduced to prune events older than 90 days. +- Aggregated metrics (DAU, daily event counts) are precomputed daily and stored in `daily_metrics`. These are kept indefinitely for long-term trend analysis. +- The `analytics_events` table is optimized with a `BRIN` index on the `createdAt` column to support fast time-series queries. + +## Adding New Events + +To add a new event type: +1. Update the `EventType` enum in `src/analytics/entities/analytics-event.entity.ts`. +2. Fire the event from the client side or backend service using the ingestion API. + +### Ingestion API + +**Single Event Ingestion** +`POST /analytics/events` +```json +{ + "eventType": "custom", + "eventName": "feature_unlocked", + "properties": { "feature": "advanced_trading" }, + "idempotencyKey": "unique-uuid-1234" +} +``` + +**Batch Event Ingestion** +`POST /analytics/events/batch` +```json +{ + "events": [ + { + "eventType": "page_view", + "page": "/dashboard", + "idempotencyKey": "unique-uuid-1235" + } + ] +} +``` + +## Reporting APIs + +The module provides reporting APIs for rendering dashboards: +- `GET /analytics/metrics/dau`: Daily active users over time. +- `GET /analytics/metrics/events`: Count of events grouped by type. +- `GET /analytics/metrics/top-events`: Top most frequent custom events. +- `GET /analytics/metrics/retention`: Basic cohort retention analysis. +- `GET /analytics/metrics/funnel`: Conversion rate across a sequence of events. diff --git a/package-lock.json b/package-lock.json index 25132e1d..6b560b2d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "@nestjs/passport": "^10.0.3", "@nestjs/platform-express": "^10.3.0", "@nestjs/platform-socket.io": "^10.4.22", + "@nestjs/schedule": "^6.1.3", "@nestjs/swagger": "^7.4.2", "@nestjs/terminus": "^11.1.1", "@nestjs/throttler": "^6.5.0", @@ -3519,6 +3520,19 @@ "node": ">=10.2.0" } }, + "node_modules/@nestjs/schedule": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-6.1.3.tgz", + "integrity": "sha512-RflMFOpR16Dwd1jAUbeB4mfGTCh65fvEdL4mSjQPJChpkRGRjIXjb+6YQcK2faQrVT60c9DmLmoVR7/ONCtuYQ==", + "license": "MIT", + "dependencies": { + "cron": "4.4.0" + }, + "peerDependencies": { + "@nestjs/common": "^10.0.0 || ^11.0.0", + "@nestjs/core": "^10.0.0 || ^11.0.0" + } + }, "node_modules/@nestjs/schematics": { "version": "10.2.3", "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-10.2.3.tgz", @@ -7121,6 +7135,12 @@ "@types/node": "*" } }, + "node_modules/@types/luxon": { + "version": "3.7.4", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.7.4.tgz", + "integrity": "sha512-V536ZAd6ZJztrrBlLcDFaaZrXNAL2E5uGmssWf/dpSiLkmkLScXUYhUBnWPmtW+cIqnNHzf6//TCMpIc9SCRRQ==", + "license": "MIT" + }, "node_modules/@types/memcached": { "version": "2.2.10", "resolved": "https://registry.npmjs.org/@types/memcached/-/memcached-2.2.10.tgz", @@ -9641,6 +9661,23 @@ "devOptional": true, "license": "MIT" }, + "node_modules/cron": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/cron/-/cron-4.4.0.tgz", + "integrity": "sha512-fkdfq+b+AHI4cKdhZlppHveI/mgz2qpiYxcm+t5E5TsxX7QrLS1VE0+7GENEk9z0EeGPcpSciGv6ez24duWhwQ==", + "license": "MIT", + "dependencies": { + "@types/luxon": "~3.7.0", + "luxon": "~3.7.0" + }, + "engines": { + "node": ">=18.x" + }, + "funding": { + "type": "ko-fi", + "url": "https://ko-fi.com/intcreator" + } + }, "node_modules/cron-parser": { "version": "4.9.0", "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", diff --git a/package.json b/package.json index 350e255c..dcab9519 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "@nestjs/passport": "^10.0.3", "@nestjs/platform-express": "^10.3.0", "@nestjs/platform-socket.io": "^10.4.22", + "@nestjs/schedule": "^6.1.3", "@nestjs/swagger": "^7.4.2", "@nestjs/terminus": "^11.1.1", "@nestjs/throttler": "^6.5.0", @@ -144,4 +145,4 @@ "tsconfig-paths": "^4.2.0", "typescript": "^5.9.3" } -} \ No newline at end of file +} diff --git a/src/analytics/analytics.cron.ts b/src/analytics/analytics.cron.ts new file mode 100644 index 00000000..dd048923 --- /dev/null +++ b/src/analytics/analytics.cron.ts @@ -0,0 +1,25 @@ +import { Injectable, Logger } from "@nestjs/common"; +import { Cron, CronExpression } from "@nestjs/schedule"; +import { AnalyticsService } from "./analytics.service"; + +@Injectable() +export class AnalyticsCronService { + private readonly logger = new Logger(AnalyticsCronService.name); + + constructor(private readonly analyticsService: AnalyticsService) {} + + @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT) + async handleDailyMetricsAggregation() { + this.logger.log("Starting daily analytics metrics aggregation..."); + try { + // Aggregate for yesterday, as this runs right at midnight + const dateToAggregate = new Date(); + dateToAggregate.setDate(dateToAggregate.getDate() - 1); + + await this.analyticsService.aggregateDailyMetrics(dateToAggregate); + this.logger.log("Successfully completed daily metrics aggregation."); + } catch (error) { + this.logger.error("Failed to aggregate daily metrics", error); + } + } +} diff --git a/src/analytics/analytics.module.ts b/src/analytics/analytics.module.ts index 54c0526b..c49380b3 100644 --- a/src/analytics/analytics.module.ts +++ b/src/analytics/analytics.module.ts @@ -4,11 +4,12 @@ import { AnalyticsService } from "./analytics.service"; import { AnalyticsController } from "./controllers/analytics.controller"; import { AnalyticsEvent } from "./entities/analytics-event.entity"; import { DailyMetric } from "./entities/daily-metric.entity"; +import { AnalyticsCronService } from "./analytics.cron"; @Module({ imports: [TypeOrmModule.forFeature([AnalyticsEvent, DailyMetric])], controllers: [AnalyticsController], - providers: [AnalyticsService], + providers: [AnalyticsService, AnalyticsCronService], exports: [AnalyticsService], }) export class AnalyticsModule {} diff --git a/src/analytics/analytics.service.spec.ts b/src/analytics/analytics.service.spec.ts new file mode 100644 index 00000000..f20a8956 --- /dev/null +++ b/src/analytics/analytics.service.spec.ts @@ -0,0 +1,91 @@ +import { Test, TestingModule } from "@nestjs/testing"; +import { AnalyticsService } from "./analytics.service"; +import { getRepositoryToken } from "@nestjs/typeorm"; +import { AnalyticsEvent, EventType } from "./entities/analytics-event.entity"; +import { DailyMetric } from "./entities/daily-metric.entity"; +import { Repository } from "typeorm"; + +describe("AnalyticsService", () => { + let service: AnalyticsService; + + const mockQueryBuilder = { + insert: jest.fn().mockReturnThis(), + into: jest.fn().mockReturnThis(), + values: jest.fn().mockReturnThis(), + orIgnore: jest.fn().mockReturnThis(), + execute: jest.fn().mockResolvedValue({ identifiers: [{ id: "test-id" }] }), + select: jest.fn().mockReturnThis(), + addSelect: jest.fn().mockReturnThis(), + where: jest.fn().mockReturnThis(), + andWhere: jest.fn().mockReturnThis(), + groupBy: jest.fn().mockReturnThis(), + orderBy: jest.fn().mockReturnThis(), + limit: jest.fn().mockReturnThis(), + getRawMany: jest.fn().mockResolvedValue([]), + getRawOne: jest.fn().mockResolvedValue({ count: "10" }), + getCount: jest.fn().mockResolvedValue(5), + }; + + const mockEventRepository = { + create: jest.fn().mockImplementation((dto) => dto), + save: jest.fn(), + find: jest.fn(), + update: jest.fn(), + createQueryBuilder: jest.fn().mockReturnValue(mockQueryBuilder), + query: jest.fn().mockResolvedValue([]), + }; + + const mockMetricRepository = { + findOne: jest.fn(), + create: jest.fn(), + save: jest.fn(), + find: jest.fn(), + }; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + AnalyticsService, + { + provide: getRepositoryToken(AnalyticsEvent), + useValue: mockEventRepository, + }, + { + provide: getRepositoryToken(DailyMetric), + useValue: mockMetricRepository, + }, + ], + }).compile(); + + service = module.get(AnalyticsService); + }); + + it("should be defined", () => { + expect(service).toBeDefined(); + }); + + describe("ingestEvent", () => { + it("should gracefully handle deduplication via orIgnore", async () => { + const result = await service.ingestEvent( + { eventType: EventType.CLICK, idempotencyKey: "123" }, + { userId: "user-1" }, + ); + + expect(result.idempotencyKey).toBe("123"); + expect(result.userId).toBe("user-1"); + expect(mockQueryBuilder.orIgnore).toHaveBeenCalled(); + }); + }); + + describe("ingestBatch", () => { + it("should gracefully handle batch deduplication", async () => { + const result = await service.ingestBatch( + { events: [{ eventType: EventType.CLICK, idempotencyKey: "123" }] }, + { userId: "user-1" }, + ); + + expect(result.accepted).toBe(1); + expect(mockQueryBuilder.orIgnore).toHaveBeenCalled(); + }); + }); +}); diff --git a/src/analytics/analytics.service.ts b/src/analytics/analytics.service.ts index df2b4777..f3bf594b 100644 --- a/src/analytics/analytics.service.ts +++ b/src/analytics/analytics.service.ts @@ -44,7 +44,18 @@ export class AnalyticsService { optedOut: false, }); - return this.eventRepository.save(event); + const result = await this.eventRepository + .createQueryBuilder() + .insert() + .into(AnalyticsEvent) + .values(event) + .orIgnore() + .execute(); + + if (result.identifiers && result.identifiers.length > 0) { + event.id = result.identifiers[0].id; + } + return event; } /** @@ -62,6 +73,10 @@ export class AnalyticsService { os?: string; }, ): Promise<{ accepted: number; rejected: number }> { + if (!dto.events.length) { + return { accepted: 0, rejected: 0 }; + } + const events = dto.events.map((eventDto) => this.eventRepository.create({ ...eventDto, @@ -77,9 +92,20 @@ export class AnalyticsService { }), ); - const result = await this.eventRepository.save(events); - this.logger.log(`Batch ingested ${result.length} events`); - return { accepted: result.length, rejected: 0 }; + const result = await this.eventRepository + .createQueryBuilder() + .insert() + .into(AnalyticsEvent) + .values(events) + .orIgnore() + .execute(); + + // result.identifiers might not match input length if duplicates were ignored + const acceptedCount = result.identifiers?.length || 0; + const rejectedCount = events.length - acceptedCount; + + this.logger.log(`Batch ingested ${acceptedCount} events, rejected ${rejectedCount} duplicates`); + return { accepted: acceptedCount, rejected: rejectedCount }; } /** @@ -157,6 +183,100 @@ export class AnalyticsService { return results; } + /** + * Get top events by frequency + */ + async getTopEvents(startDate: Date, endDate: Date, limit: number = 10): Promise<{ eventName: string; count: number }[]> { + const result = await this.eventRepository + .createQueryBuilder("event") + .select("event.eventName", "eventName") + .addSelect("COUNT(*)", "count") + .where("event.createdAt BETWEEN :startDate AND :endDate", { startDate, endDate }) + .andWhere("event.eventName IS NOT NULL") + .andWhere("event.optedOut = false") + .groupBy("event.eventName") + .orderBy("count", "DESC") + .limit(limit) + .getRawMany(); + + return result.map((r) => ({ + eventName: r.eventName, + count: parseInt(r.count, 10), + })); + } + + /** + * Get basic retention cohorts (by day) + */ + async getRetentionCohorts(startDate: Date, endDate: Date): Promise { + // This is a simplified retention query. In production with a huge DB, + // it's better to precalculate this or use specific analytics DB like ClickHouse. + const query = ` + WITH user_first_seen AS ( + SELECT "userId", DATE("createdAt") AS first_day + FROM "analytics_events" + WHERE "userId" IS NOT NULL AND "optedOut" = false + GROUP BY "userId" + ), + retention_data AS ( + SELECT + u.first_day AS cohort_day, + DATE(e."createdAt") - u.first_day AS day_offset, + COUNT(DISTINCT e."userId") AS active_users + FROM user_first_seen u + JOIN "analytics_events" e ON u."userId" = e."userId" + WHERE e."createdAt" BETWEEN $1 AND $2 + AND e."optedOut" = false + GROUP BY u.first_day, DATE(e."createdAt") - u.first_day + ) + SELECT + cohort_day, + day_offset, + active_users + FROM retention_data + WHERE day_offset >= 0 + ORDER BY cohort_day ASC, day_offset ASC + `; + + const result = await this.eventRepository.query(query, [startDate, endDate]); + + // Group by cohort_day + const cohorts: Record }> = {}; + + result.forEach((row: any) => { + const cohortDay = new Date(row.cohort_day).toISOString().split('T')[0]; + const offset = parseInt(row.day_offset, 10); + const count = parseInt(row.active_users, 10); + + if (!cohorts[cohortDay]) { + cohorts[cohortDay] = { size: 0, retention: {} }; + } + + if (offset === 0) { + cohorts[cohortDay].size = count; + } + + cohorts[cohortDay].retention[offset] = count; + }); + + // Format output + return Object.keys(cohorts).map((day) => { + const cohort = cohorts[day]; + const retentionRates: Record = {}; + + Object.keys(cohort.retention).forEach((offsetStr) => { + const offset = parseInt(offsetStr, 10); + retentionRates[offset] = Math.round((cohort.retention[offset] / cohort.size) * 10000) / 100; + }); + + return { + cohortDay: day, + size: cohort.size, + retentionRates, + }; + }); + } + /** * Aggregate daily metrics */ diff --git a/src/analytics/controllers/analytics.controller.spec.ts b/src/analytics/controllers/analytics.controller.spec.ts new file mode 100644 index 00000000..d7502229 --- /dev/null +++ b/src/analytics/controllers/analytics.controller.spec.ts @@ -0,0 +1,70 @@ +import { Test, TestingModule } from "@nestjs/testing"; +import { AnalyticsController } from "./analytics.controller"; +import { AnalyticsService } from "../analytics.service"; +import { EventType } from "../entities/analytics-event.entity"; +import { IngestEventDto, BatchIngestEventsDto } from "../dto/ingest-events.dto"; + +describe("AnalyticsController", () => { + let controller: AnalyticsController; + let service: AnalyticsService; + + const mockAnalyticsService = { + ingestEvent: jest.fn(), + ingestBatch: jest.fn(), + getDailyActiveUsers: jest.fn(), + getEventCountsByType: jest.fn(), + getFunnelConversion: jest.fn(), + getTopEvents: jest.fn(), + getRetentionCohorts: jest.fn(), + getMetrics: jest.fn(), + optOut: jest.fn(), + }; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [AnalyticsController], + providers: [ + { + provide: AnalyticsService, + useValue: mockAnalyticsService, + }, + ], + }).compile(); + + controller = module.get(AnalyticsController); + service = module.get(AnalyticsService); + }); + + it("should be defined", () => { + expect(controller).toBeDefined(); + }); + + describe("ingestEvent", () => { + it("should accept a single event", async () => { + const dto: IngestEventDto = { eventType: EventType.PAGE_VIEW, idempotencyKey: "123" }; + const req = { headers: {}, ip: "127.0.0.1" }; + + mockAnalyticsService.ingestEvent.mockResolvedValue({ id: "event-id" }); + + const result = await controller.ingestEvent(dto, req); + expect(result).toEqual({ status: "accepted", eventId: "event-id" }); + expect(service.ingestEvent).toHaveBeenCalled(); + }); + }); + + describe("reporting endpoints", () => { + it("should get top events", async () => { + mockAnalyticsService.getTopEvents.mockResolvedValue([{ eventName: "test", count: 10 }]); + const result = await controller.getTopEvents("2026-01-01", "2026-01-31", 5); + expect(result).toEqual([{ eventName: "test", count: 10 }]); + expect(service.getTopEvents).toHaveBeenCalledWith(expect.any(Date), expect.any(Date), 5); + }); + + it("should get retention cohorts", async () => { + mockAnalyticsService.getRetentionCohorts.mockResolvedValue([{ cohortDay: "2026-01-01", size: 100, retentionRates: { 1: 50 } }]); + const result = await controller.getRetentionCohorts("2026-01-01", "2026-01-31"); + expect(result[0].size).toBe(100); + expect(service.getRetentionCohorts).toHaveBeenCalled(); + }); + }); +}); diff --git a/src/analytics/controllers/analytics.controller.ts b/src/analytics/controllers/analytics.controller.ts index 34d4f705..179ec405 100644 --- a/src/analytics/controllers/analytics.controller.ts +++ b/src/analytics/controllers/analytics.controller.ts @@ -106,6 +106,41 @@ export class AnalyticsController { ); } + @Get("metrics/top-events") + @UseGuards(JwtAuthGuard) + @ApiBearerAuth() + @ApiOperation({ summary: "Get top events by frequency" }) + @ApiQuery({ name: "startDate", required: true, type: String }) + @ApiQuery({ name: "endDate", required: true, type: String }) + @ApiQuery({ name: "limit", required: false, type: Number }) + async getTopEvents( + @Query("startDate") startDate: string, + @Query("endDate") endDate: string, + @Query("limit") limit?: number, + ) { + return this.analyticsService.getTopEvents( + new Date(startDate), + new Date(endDate), + limit ? Number(limit) : 10, + ); + } + + @Get("metrics/retention") + @UseGuards(JwtAuthGuard) + @ApiBearerAuth() + @ApiOperation({ summary: "Get retention cohorts" }) + @ApiQuery({ name: "startDate", required: true, type: String }) + @ApiQuery({ name: "endDate", required: true, type: String }) + async getRetentionCohorts( + @Query("startDate") startDate: string, + @Query("endDate") endDate: string, + ) { + return this.analyticsService.getRetentionCohorts( + new Date(startDate), + new Date(endDate), + ); + } + @Get("metrics/summary") @UseGuards(JwtAuthGuard) @ApiBearerAuth() diff --git a/src/analytics/dto/ingest-events.dto.ts b/src/analytics/dto/ingest-events.dto.ts index 4d008730..50fc7fc5 100644 --- a/src/analytics/dto/ingest-events.dto.ts +++ b/src/analytics/dto/ingest-events.dto.ts @@ -37,6 +37,11 @@ export class IngestEventDto { @IsString() @IsOptional() timestamp?: string; + + @ApiPropertyOptional({ description: "Idempotency key for deduplication" }) + @IsString() + @IsOptional() + idempotencyKey?: string; } export class BatchIngestEventsDto { diff --git a/src/analytics/entities/analytics-event.entity.ts b/src/analytics/entities/analytics-event.entity.ts index e6bb2976..9c1cfe13 100644 --- a/src/analytics/entities/analytics-event.entity.ts +++ b/src/analytics/entities/analytics-event.entity.ts @@ -75,6 +75,10 @@ export class AnalyticsEvent { @Column({ type: "boolean", default: false }) optedOut: boolean; + @Column({ type: "varchar", length: 255, nullable: true }) + @Index({ unique: true }) + idempotencyKey: string; + @CreateDateColumn() createdAt: Date; } diff --git a/src/app.module.ts b/src/app.module.ts index 370dedca..26a7aa10 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -11,6 +11,7 @@ import { APP_GUARD } from "@nestjs/core"; import { ThrottlerModule } from "@nestjs/throttler"; import { TerminusModule } from "@nestjs/terminus"; import { EventEmitterModule } from "@nestjs/event-emitter"; +import { ScheduleModule } from "@nestjs/schedule"; import { AppController } from "./app.controller"; import { AppService } from "./app.service"; @@ -98,6 +99,7 @@ import { QuotaGuard } from "./common/guard/quota.guard"; }), EventEmitterModule.forRoot(), + ScheduleModule.forRoot(), // ✅ ONLY ONE TypeORM CONFIG (Async) TypeOrmModule.forRootAsync({ diff --git a/src/migrations/1724076000000-AnalyticsModuleUpdates.ts b/src/migrations/1724076000000-AnalyticsModuleUpdates.ts new file mode 100644 index 00000000..28b6ffec --- /dev/null +++ b/src/migrations/1724076000000-AnalyticsModuleUpdates.ts @@ -0,0 +1,23 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AnalyticsModuleUpdates1724076000000 implements MigrationInterface { + name = 'AnalyticsModuleUpdates1724076000000' + + public async up(queryRunner: QueryRunner): Promise { + // Add idempotencyKey column + await queryRunner.query(`ALTER TABLE "analytics_events" ADD "idempotencyKey" character varying(255)`); + + // Add unique index for deduplication + await queryRunner.query(`CREATE UNIQUE INDEX "IDX_analytics_events_idempotencyKey" ON "analytics_events" ("idempotencyKey")`); + + // Add BRIN index for time-series optimization on createdAt (great for analytics) + // Note: IF NOT EXISTS is not standard for CREATE INDEX in all Postgres versions, but we'll create the BRIN index specifically. + await queryRunner.query(`CREATE INDEX "IDX_analytics_events_createdAt_brin" ON "analytics_events" USING BRIN ("createdAt")`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "public"."IDX_analytics_events_createdAt_brin"`); + await queryRunner.query(`DROP INDEX "public"."IDX_analytics_events_idempotencyKey"`); + await queryRunner.query(`ALTER TABLE "analytics_events" DROP COLUMN "idempotencyKey"`); + } +}