Merge pull request #25 from davedumto/feat/data-normalization

feat: add normalizers for Alpha Vantage etc

Author: Villarley

apps/aggregator/README.md

@@ -426,6 +426,100 @@ export class MyCustomAggregator implements IAggregator {
   }
 }
 ```
+apps/aggregator/
+├── src/
+│   ├── main.ts                    # Application entry point
+│   ├── app.module.ts              # Root module
+│   ├── interfaces/                # Type definitions
+│   │   ├── normalized-price.interface.ts
+│   │   └── normalizer.interface.ts
+│   ├── normalizers/               # Source-specific normalizers
+│   │   ├── base.normalizer.ts
+│   │   ├── alpha-vantage.normalizer.ts
+│   │   ├── finnhub.normalizer.ts
+│   │   ├── yahoo-finance.normalizer.ts
+│   │   └── mock.normalizer.ts
+│   ├── services/                  # Business logic
+│   │   └── normalization.service.ts
+│   ├── modules/                   # Feature modules
+│   │   └── normalization.module.ts
+│   └── exceptions/                # Custom exceptions
+│       └── normalization.exception.ts
+├── .env.example         # Example environment variables
+├── nest-cli.json        # NestJS CLI configuration
+├── package.json         # Dependencies and scripts
+├── tsconfig.json        # TypeScript configuration
+└── README.md           # This file
+```
+
+## Data Normalization
+
+### NormalizedPrice Interface
+
+The standard internal format for normalized price data:
+
+```typescript
+interface NormalizedPrice {
+  symbol: string;           // Normalized ticker (e.g., 'AAPL')
+  price: number;            // Price rounded to 4 decimal places
+  timestamp: string;        // ISO 8601 UTC (e.g., '2024-01-15T14:30:00.000Z')
+  originalTimestamp: number; // Original Unix timestamp in milliseconds
+  source: NormalizedSource; // Enum: 'alpha_vantage' | 'finnhub' | 'yahoo_finance' | 'mock'
+  metadata: {
+    originalSource: string;    // Original source string
+    originalSymbol: string;    // Original symbol before normalization
+    normalizedAt: string;      // When normalization occurred
+    normalizerVersion: string; // Version of normalizer used
+    wasTransformed: boolean;   // Whether transformations were applied
+    transformations: string[]; // List of transformations applied
+  };
+}
+```
+
+### Supported Sources and Transformations
+
+| Source | Detected By | Symbol Transformations |
+|--------|-------------|------------------------|
+| **Alpha Vantage** | `alphavantage`, `alpha_vantage`, `alpha-vantage` | Removes `.US`, `.NYSE`, `.NASDAQ`, `.LSE`, `.TSX`, `.ASX`, `.HK` suffixes |
+| **Finnhub** | `finnhub` | Removes `US-`, `CRYPTO-`, `FX-`, `INDICES-` prefixes |
+| **Yahoo Finance** | `yahoo`, `yahoofinance`, `yahoo_finance`, `yahoo-finance` | Removes `.L`, `.T`, `.AX`, `.HK`, `.SI`, `.KS`, `.TW`, `.NS`, `.BO`, `.TO`, `.DE`, `.PA` suffixes; removes `^` index prefix |
+| **Mock** | `mock` | Basic cleanup (trim, uppercase) |
+
+### Common Transformations
+
+All normalizers apply these transformations:
+- **Symbol**: Trimmed and uppercased
+- **Price**: Rounded to 4 decimal places
+- **Timestamp**: Converted to ISO 8601 UTC format
+
+### Usage Example
+
+```typescript
+import { NormalizationService } from './services/normalization.service';
+
+// Inject via NestJS DI
+constructor(private readonly normalizationService: NormalizationService) {}
+
+// Normalize a single price
+const rawPrice = {
+  symbol: 'AAPL.US',
+  price: 150.123456,
+  timestamp: Date.now(),
+  source: 'AlphaVantage',
+};
+const normalized = this.normalizationService.normalize(rawPrice);
+// Result: { symbol: 'AAPL', price: 150.1235, timestamp: '2024-01-15T14:30:00.000Z', ... }
+
+// Normalize multiple prices (skips failures)
+const results = this.normalizationService.normalizeMany(rawPrices);
+
+// Normalize with error tracking
+const { successful, failed } = this.normalizationService.normalizeManyWithErrors(rawPrices);
+```
+
+## Status
+
+🚧 Under construction - Aggregation and filtering logic will be implemented in subsequent issues.
 
 2. Register in `AggregationService` constructor:
 

apps/aggregator/package.json

@@ -21,6 +21,7 @@
     "@nestjs/core": "^10.0.0",
     "@nestjs/event-emitter": "^3.0.1",
     "@nestjs/platform-express": "^10.0.0",
+    "@oracle-stocks/shared": "*",
     "@nestjs/terminus": "^10.0.0",
     "axios": "^1.6.0",
     "ioredis": "^5.3.2",
@@ -69,6 +70,9 @@
       "**/*.(t|j)s"
     ],
     "coverageDirectory": "../coverage",
-    "testEnvironment": "node"
+    "testEnvironment": "node",
+    "moduleNameMapper": {
+      "^@oracle-stocks/shared$": "<rootDir>/../../packages/shared/src"
+    }
   }
 }

apps/aggregator/src/__mocks__/raw-price.fixtures.ts

@@ -0,0 +1,101 @@
+import { RawPrice } from '@oracle-stocks/shared';
+
+/**
+ * Test fixtures for raw price data from different sources
+ */
+export const mockRawPrices: Record<string, RawPrice> = {
+  alphaVantage: {
+    symbol: 'AAPL.US',
+    price: 150.1234567,
+    timestamp: 1705329000000, // 2024-01-15T14:30:00.000Z
+    source: 'AlphaVantage',
+  },
+  alphaVantageNYSE: {
+    symbol: 'MSFT.NYSE',
+    price: 380.5,
+    timestamp: 1705330200000,
+    source: 'alpha_vantage',
+  },
+  finnhub: {
+    symbol: 'US-GOOGL',
+    price: 140.999,
+    timestamp: 1705330200000,
+    source: 'Finnhub',
+  },
+  finnhubCrypto: {
+    symbol: 'CRYPTO-BTC',
+    price: 42000.0,
+    timestamp: 1705330200000,
+    source: 'finnhub',
+  },
+  yahooFinance: {
+    symbol: 'MSFT.L',
+    price: 380.12345,
+    timestamp: 1705330200000,
+    source: 'Yahoo Finance',
+  },
+  yahooFinanceIndex: {
+    symbol: '^DJI',
+    price: 37500.0,
+    timestamp: 1705330200000,
+    source: 'yahoo_finance',
+  },
+  yahooFinanceAustralia: {
+    symbol: 'BHP.AX',
+    price: 45.67,
+    timestamp: 1705330200000,
+    source: 'YahooFinance',
+  },
+  mock: {
+    symbol: 'TSLA',
+    price: 250.5,
+    timestamp: 1705330200000,
+    source: 'MockProvider',
+  },
+  mockLowercase: {
+    symbol: '  aapl  ',
+    price: 150.0,
+    timestamp: 1705330200000,
+    source: 'mock',
+  },
+  unknown: {
+    symbol: 'BTC',
+    price: 42000.0,
+    timestamp: 1705330200000,
+    source: 'UnknownSource',
+  },
+};
+
+/**
+ * Malformed price data for testing validation
+ */
+export const malformedPrices: Array<Partial<RawPrice> | null | undefined> = [
+  { symbol: '', price: 100, timestamp: Date.now(), source: 'Test' },
+  { symbol: 'TEST', price: NaN, timestamp: Date.now(), source: 'Test' },
+  { symbol: 'TEST', price: -100, timestamp: Date.now(), source: 'Test' },
+  { symbol: 'TEST', price: 100, timestamp: null as unknown as number, source: 'Test' },
+  { symbol: 'TEST', price: 100, timestamp: Date.now(), source: '' },
+  { price: 100, timestamp: Date.now(), source: 'Test' } as Partial<RawPrice>,
+  { symbol: 'TEST', timestamp: Date.now(), source: 'Test' } as Partial<RawPrice>,
+  null,
+  undefined,
+];
+
+/**
+ * Valid raw prices for batch testing
+ */
+export const validRawPrices: RawPrice[] = [
+  mockRawPrices.alphaVantage,
+  mockRawPrices.finnhub,
+  mockRawPrices.yahooFinance,
+  mockRawPrices.mock,
+];
+
+/**
+ * Mixed valid and invalid prices for error handling tests
+ */
+export const mixedRawPrices: RawPrice[] = [
+  mockRawPrices.alphaVantage,
+  mockRawPrices.unknown,
+  mockRawPrices.finnhub,
+];

apps/aggregator/src/app.module.ts

@@ -1,4 +1,5 @@
 import { Module } from '@nestjs/common';
+import { NormalizationModule } from './modules/normalization.module';
 import { ConfigModule } from '@nestjs/config';
 import { AggregationService } from './services/aggregation.service';
 import { WeightedAverageAggregator } from './strategies/aggregators/weighted-average.aggregator';
@@ -10,6 +11,7 @@ import { DebugModule } from './debug/debug.module';
 
 @Module({
   imports: [
+    NormalizationModule,
     ConfigModule.forRoot({ isGlobal: true, envFilePath: '.env' }),
     HealthModule,
     MetricsModule,

apps/aggregator/src/exceptions/index.ts

@@ -0,0 +1 @@
+export * from './normalization.exception';

apps/aggregator/src/exceptions/normalization.exception.ts

@@ -0,0 +1,36 @@
+import { RawPrice } from '@oracle-stocks/shared';
+
+/**
+ * Base exception for normalization errors
+ */
+export class NormalizationException extends Error {
+  constructor(
+    message: string,
+    public readonly rawPrice?: RawPrice,
+    public readonly cause?: Error,
+  ) {
+    super(message);
+    this.name = 'NormalizationException';
+    Error.captureStackTrace(this, this.constructor);
+  }
+}
+
+/**
+ * Exception for validation failures
+ */
+export class ValidationException extends NormalizationException {
+  constructor(message: string, rawPrice?: RawPrice) {
+    super(message, rawPrice);
+    this.name = 'ValidationException';
+  }
+}
+
+/**
+ * Exception when no normalizer is found for a source
+ */
+export class NoNormalizerFoundException extends NormalizationException {
+  constructor(source: string, rawPrice?: RawPrice) {
+    super(`No normalizer found for source: ${source}`, rawPrice);
+    this.name = 'NoNormalizerFoundException';
+  }
+}

apps/aggregator/src/interfaces/index.ts

@@ -0,0 +1,2 @@
+export * from './normalized-price.interface';
+export * from './normalizer.interface';

apps/aggregator/src/interfaces/normalized-price.interface.ts

@@ -1,3 +1,62 @@
+/**
+ * Enum for standardized source identifiers
+ */
+export enum NormalizedSource {
+  ALPHA_VANTAGE = 'alpha_vantage',
+  FINNHUB = 'finnhub',
+  YAHOO_FINANCE = 'yahoo_finance',
+  MOCK = 'mock',
+  UNKNOWN = 'unknown',
+}
+
+/**
+ * Metadata tracking normalization processing
+ */
+export interface NormalizationMetadata {
+  /** Original source string before normalization */
+  originalSource: string;
+
+  /** Original symbol before normalization (e.g., 'AAPL.US') */
+  originalSymbol: string;
+
+  /** ISO 8601 timestamp when normalization was performed */
+  normalizedAt: string;
+
+  /** Version of the normalization logic used */
+  normalizerVersion: string;
+
+  /** Whether any transformations were applied */
+  wasTransformed: boolean;
+
+  /** List of transformations applied (for debugging/audit) */
+  transformations: string[];
+}
+
+/**
+ * Represents a fully normalized price record with standard formatting
+ * and metadata for audit/tracking purposes.
+ * Used as output from the normalization service.
+ */
+export interface NormalizedPriceRecord {
+  /** Normalized ticker symbol (e.g., 'AAPL' - stripped of exchange suffixes) */
+  symbol: string;
+
+  /** Price value normalized to minimum 4 decimal places */
+  price: number;
+
+  /** ISO 8601 UTC timestamp string (e.g., '2024-01-15T14:30:00.000Z') */
+  timestamp: string;
+
+  /** Original Unix timestamp in milliseconds (preserved for precision) */
+  originalTimestamp: number;
+
+  /** Normalized source identifier */
+  source: NormalizedSource;
+
+  /** Metadata for tracking and audit purposes */
+  metadata: NormalizationMetadata;
+}
+
 /**
  * Normalized price data structure from various sources
  * This is the input to the aggregation engine

apps/aggregator/src/interfaces/normalizer.interface.ts

@@ -0,0 +1,55 @@
+import { RawPrice } from '@oracle-stocks/shared';
+import { NormalizedPriceRecord, NormalizedSource } from './normalized-price.interface';
+
+/**
+ * Interface for source-specific normalization strategies.
+ * Each data source should implement this interface.
+ */
+export interface Normalizer {
+  /** Unique identifier for this normalizer */
+  readonly name: string;
+
+  /** The source this normalizer handles */
+  readonly source: NormalizedSource;
+
+  /** Version string for tracking normalization logic changes */
+  readonly version: string;
+
+  /**
+   * Check if this normalizer can handle the given raw price
+   * @param rawPrice - The raw price to check
+   * @returns true if this normalizer supports the source
+   */
+  canNormalize(rawPrice: RawPrice): boolean;
+
+  /**
+   * Normalize a single raw price record
+   * @param rawPrice - The raw price to normalize
+   * @returns Normalized price or throws NormalizationException
+   */
+  normalize(rawPrice: RawPrice): NormalizedPriceRecord;
+
+  /**
+   * Normalize multiple raw price records
+   * @param rawPrices - Array of raw prices to normalize
+   * @returns Array of normalized prices (invalid entries filtered out)
+   */
+  normalizeMany(rawPrices: RawPrice[]): NormalizedPriceRecord[];
+}
+
+/**
+ * Result type for batch normalization with error tracking
+ */
+export interface NormalizationResult {
+  successful: NormalizedPriceRecord[];
+  failed: NormalizationFailure[];
+}
+
+/**
+ * Represents a failed normalization attempt
+ */
+export interface NormalizationFailure {
+  rawPrice: RawPrice;
+  error: string;
+  timestamp: string;
+}