diff --git a/.env.example b/.env.example index fcad466..1e38c63 100644 --- a/.env.example +++ b/.env.example @@ -7,6 +7,16 @@ NAA_ENV=development NAA_DATABASE_PATH=naa.sqlite +# LangSmith / LangGraph tracing +# Set LANGSMITH_TRACING=true and LANGSMITH_API_KEY to trace graph runs. +LANGSMITH_TRACING=false +LANGSMITH_API_KEY= +LANGSMITH_ENDPOINT=https://api.smith.langchain.com +LANGSMITH_PROJECT=narrative-alpha-agent +LANGSMITH_WORKSPACE_ID= +LANGSMITH_RUN_NAME=narrative-alpha-agent +LANGSMITH_TAGS=naa,langgraph,narrative-replay + # LLM provider selection. # Supported values: # local, openai, anthropic, deepseek, openrouter, google, cohere, mistral, @@ -79,3 +89,8 @@ TWITTER_BEARER_TOKEN= TELEGRAM_BOT_TOKEN= TELEGRAM_CHAT_ID= NEWS_API_KEY= + +# Alert notifiers +DISCORD_WEBHOOK_URL= +DISCORD_USERNAME=Narrative Alpha Agent +DISCORD_AVATAR_URL= diff --git a/README.md b/README.md index bccbc60..b451e1b 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,8 @@ pnpm run test - [Backtesting](docs/BACKTESTING.md) - [Operations](docs/OPERATIONS.md) - [Providers and Secrets](docs/PROVIDERS.md) +- [Observability](docs/OBSERVABILITY.md) +- [Notifications](docs/NOTIFICATIONS.md) - [Contributing](CONTRIBUTING.md) - [Security](SECURITY.md) - [Changelog](CHANGELOG.md) diff --git a/docs/NOTIFICATIONS.md b/docs/NOTIFICATIONS.md new file mode 100644 index 0000000..0cd21ac --- /dev/null +++ b/docs/NOTIFICATIONS.md @@ -0,0 +1,42 @@ +# Notifications + +Narrative Alpha Agent emits alerts through the `Notifier` port in `src/types/ports.ts`. + +## Built-in Notifiers + +- `ConsoleNotifier`: writes alerts to stdout +- `TelegramNotifierStub`: captures alert messages for future Telegram integration +- `DiscordNotifier`: sends rich embeds to a Discord webhook + +## Discord + +Set these variables in `.env`: + +```bash +DISCORD_WEBHOOK_URL=https://discord.com/api/webhooks/... +DISCORD_USERNAME=Narrative Alpha Agent +DISCORD_AVATAR_URL= +``` + +When `DISCORD_WEBHOOK_URL` is unset, Discord is not enabled. This keeps tests and replay offline by default. + +Discord alerts include: + +- narrative name +- alert reason +- NIP score +- lifecycle state +- document count +- timestamp + +## Adding More Notifiers + +Implement `Notifier`: + +```ts +interface Notifier { + notify(narrative: Narrative, message: string): Promise; +} +``` + +Then inject it through `createRuntime({ notifiers: [...] })` or add a typed environment-backed config in `src/config/notifications.ts`. diff --git a/docs/OBSERVABILITY.md b/docs/OBSERVABILITY.md new file mode 100644 index 0000000..916f0dd --- /dev/null +++ b/docs/OBSERVABILITY.md @@ -0,0 +1,55 @@ +# Observability + +Narrative Alpha Agent uses LangGraph as its execution layer, so graph runs can be traced with LangSmith through LangChain's standard tracing environment variables. + +Tracing is disabled by default to keep local development and replay offline. + +## Enable LangSmith + +Copy `.env.example` to `.env` and set: + +```bash +LANGSMITH_TRACING=true +LANGSMITH_API_KEY=lsv2_... +LANGSMITH_PROJECT=narrative-alpha-agent +LANGSMITH_RUN_NAME=narrative-alpha-agent +LANGSMITH_TAGS=naa,langgraph,narrative-replay +``` + +Optional: + +```bash +LANGSMITH_ENDPOINT=https://api.smith.langchain.com +LANGSMITH_WORKSPACE_ID=... +``` + +The runtime reads these values in `src/config/observability.ts`. + +## What Gets Traced + +`NarrativeGraphRunner` passes LangChain runnable config into every graph invocation: + +- `runName` +- `tags` +- metadata containing app name, replay timestamp, document count, project, and tracing status +- `thread_id` for LangGraph checkpointing and replay re-entry + +When LangSmith tracing is enabled, LangGraph/LangChain records the graph run and child node spans under the configured project. + +## Replay Tracing + +Replay uses a timestamp-specific thread ID: + +```text +replay-1735689600000 +replay-1735693200000 +... +``` + +This makes historical checkpoints searchable in LangSmith by run metadata and thread ID. + +## Local Safety + +- No traces are emitted unless `LANGSMITH_TRACING=true`. +- No LangSmith API key is required for tests, replay, or local development. +- Do not commit real `LANGSMITH_API_KEY` values. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 72c606b..3084374 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -34,9 +34,12 @@ The default runtime uses: - `ConsoleNotifier` - `TelegramNotifierStub` +- `DiscordNotifier` when `DISCORD_WEBHOOK_URL` is configured Real notifier implementations should implement `Notifier` and be wired through runtime construction. +See [Notifications](NOTIFICATIONS.md). + ## LLM and Embeddings The default MVP path does not require external paid APIs. @@ -45,6 +48,14 @@ LLM credentials are configured with `.env.example`. The provider registry suppor The deterministic local embedding provider is used by default for replay stability. +## Observability + +LangSmith tracing is configured through `.env.example`. + +Set `LANGSMITH_TRACING=true`, `LANGSMITH_API_KEY`, and `LANGSMITH_PROJECT` to trace LangGraph runs. Runtime metadata includes replay timestamp, document count, tags, project, and thread ID. + +See [Observability](OBSERVABILITY.md). + ## CI GitHub Actions runs: diff --git a/docs/PROVIDERS.md b/docs/PROVIDERS.md index 9a479fc..a4e01c9 100644 --- a/docs/PROVIDERS.md +++ b/docs/PROVIDERS.md @@ -100,3 +100,5 @@ Real source connectors should implement `DocumentSourceConnector` from `src/type - Scope keys to the minimum provider permissions. - Rotate keys after accidental disclosure. - Keep production secrets in a secret manager, not in local files. + +LangSmith and notifier secrets are also listed in `.env.example`. diff --git a/src/agents/createRuntime.ts b/src/agents/createRuntime.ts index 32fceb6..cacfb02 100644 --- a/src/agents/createRuntime.ts +++ b/src/agents/createRuntime.ts @@ -1,8 +1,15 @@ import { defaultConfig, type AppConfig } from "../config/defaults.js"; +import { readNotificationConfigFromEnv, type NotificationConfig } from "../config/notifications.js"; +import { readLangSmithConfigFromEnv, type LangSmithConfig } from "../config/observability.js"; import { readProviderConfigFromEnv, type ProviderSecretConfig } from "../config/secrets.js"; import { SqliteNarrativeRepository } from "../db/SqliteNarrativeRepository.js"; import { NarrativeGraphRunner } from "../graph/agentGraph.js"; -import { AlertService, ConsoleNotifier, TelegramNotifierStub } from "../services/AlertService.js"; +import { + AlertService, + ConsoleNotifier, + DiscordNotifier, + TelegramNotifierStub +} from "../services/AlertService.js"; import { ClusteringService } from "../services/ClusteringService.js"; import { DeterministicEmbeddingProvider } from "../services/DeterministicEmbeddingProvider.js"; import { InMemoryVectorStore } from "../services/InMemoryVectorStore.js"; @@ -11,6 +18,7 @@ import { NarrativeLifecycleService } from "../services/NarrativeLifecycleService import { ScoringService } from "../services/ScoringService.js"; import { SentimentService } from "../services/SentimentService.js"; import type { NarrativeRepository } from "../types/ports.js"; +import type { Notifier } from "../types/ports.js"; export type RuntimeOptions = { config?: Partial; @@ -18,6 +26,9 @@ export type RuntimeOptions = { databasePath?: string; enableConsoleAlerts?: boolean; llmProvider?: ProviderSecretConfig; + notifications?: NotificationConfig; + langSmith?: LangSmithConfig; + notifiers?: Notifier[]; }; export const createRuntime = (options: RuntimeOptions = {}): NarrativeGraphRunner => { @@ -29,6 +40,7 @@ export const createRuntime = (options: RuntimeOptions = {}): NarrativeGraphRunne ...options.config?.scoringWeights } }; + const langSmith = options.langSmith ?? readLangSmithConfigFromEnv(); const repository = options.repository ?? new SqliteNarrativeRepository(options.databasePath ?? "naa.sqlite"); repository.initialize(); @@ -44,18 +56,45 @@ export const createRuntime = (options: RuntimeOptions = {}): NarrativeGraphRunne }); void llm; const scoring = new ScoringService(config.scoringWeights); + const notifications = options.notifications ?? readNotificationConfigFromEnv(); const notifiers = - options.enableConsoleAlerts === false - ? [new TelegramNotifierStub()] - : [new ConsoleNotifier(), new TelegramNotifierStub()]; + options.notifiers ?? + buildNotifiers({ + enableConsoleAlerts: options.enableConsoleAlerts, + notifications + }); const alerts = new AlertService(notifiers); - return new NarrativeGraphRunner({ - config, - clustering, - lifecycle, - scoring, - repository, - alerts - }); + return new NarrativeGraphRunner( + { + config, + clustering, + lifecycle, + scoring, + repository, + alerts + }, + langSmith + ); +}; + +const buildNotifiers = (options: { + enableConsoleAlerts?: boolean | undefined; + notifications: NotificationConfig; +}): Notifier[] => { + const notifiers: Notifier[] = []; + if (options.enableConsoleAlerts !== false) { + notifiers.push(new ConsoleNotifier()); + } + notifiers.push(new TelegramNotifierStub()); + if (options.notifications.discordWebhookUrl) { + notifiers.push( + new DiscordNotifier({ + webhookUrl: options.notifications.discordWebhookUrl, + username: options.notifications.discordUsername, + avatarUrl: options.notifications.discordAvatarUrl + }) + ); + } + return notifiers; }; diff --git a/src/config/notifications.ts b/src/config/notifications.ts new file mode 100644 index 0000000..7bd737d --- /dev/null +++ b/src/config/notifications.ts @@ -0,0 +1,13 @@ +export type NotificationConfig = { + discordWebhookUrl?: string | undefined; + discordUsername?: string | undefined; + discordAvatarUrl?: string | undefined; +}; + +export const readNotificationConfigFromEnv = ( + env: NodeJS.ProcessEnv = process.env +): NotificationConfig => ({ + discordWebhookUrl: env.DISCORD_WEBHOOK_URL, + discordUsername: env.DISCORD_USERNAME ?? "Narrative Alpha Agent", + discordAvatarUrl: env.DISCORD_AVATAR_URL +}); diff --git a/src/config/observability.ts b/src/config/observability.ts new file mode 100644 index 0000000..2c1cc56 --- /dev/null +++ b/src/config/observability.ts @@ -0,0 +1,27 @@ +export type LangSmithConfig = { + tracingEnabled: boolean; + apiKey?: string | undefined; + endpoint?: string | undefined; + project?: string | undefined; + workspaceId?: string | undefined; + runName: string; + tags: string[]; +}; + +export const readLangSmithConfigFromEnv = ( + env: NodeJS.ProcessEnv = process.env +): LangSmithConfig => ({ + tracingEnabled: env.LANGSMITH_TRACING === "true" || env.LANGCHAIN_TRACING_V2 === "true", + apiKey: env.LANGSMITH_API_KEY, + endpoint: env.LANGSMITH_ENDPOINT ?? "https://api.smith.langchain.com", + project: env.LANGSMITH_PROJECT ?? "narrative-alpha-agent", + workspaceId: env.LANGSMITH_WORKSPACE_ID, + runName: env.LANGSMITH_RUN_NAME ?? "narrative-alpha-agent", + tags: parseTags(env.LANGSMITH_TAGS ?? "naa,langgraph,narrative-replay") +}); + +const parseTags = (value: string): string[] => + value + .split(",") + .map((tag) => tag.trim()) + .filter((tag) => tag.length > 0); diff --git a/src/graph/agentGraph.ts b/src/graph/agentGraph.ts index a7ddec6..6f4dc47 100644 --- a/src/graph/agentGraph.ts +++ b/src/graph/agentGraph.ts @@ -1,5 +1,6 @@ import { END, MemorySaver, START, StateGraph } from "@langchain/langgraph"; import type { AppConfig } from "../config/defaults.js"; +import type { LangSmithConfig } from "../config/observability.js"; import type { AlertService } from "../services/AlertService.js"; import type { ClusteringService } from "../services/ClusteringService.js"; import type { NarrativeLifecycleService } from "../services/NarrativeLifecycleService.js"; @@ -53,12 +54,28 @@ export const createNarrativeGraph = (dependencies: GraphDependencies) => { export class NarrativeGraphRunner { private readonly graph: ReturnType; - constructor(dependencies: GraphDependencies) { + constructor( + dependencies: GraphDependencies, + private readonly langSmith: LangSmithConfig = { + tracingEnabled: false, + runName: "narrative-alpha-agent", + tags: [] + } + ) { this.graph = createNarrativeGraph(dependencies); } async run(input: SystemState, threadId = "default"): Promise { const result = await this.graph.invoke(input, { + runName: this.langSmith.runName, + tags: this.langSmith.tags, + metadata: { + app: "narrative-alpha-agent", + timestamp: input.timestamp, + documentCount: input.documents.length, + langSmithProject: this.langSmith.project, + tracingEnabled: this.langSmith.tracingEnabled + }, configurable: { thread_id: threadId } diff --git a/src/index.ts b/src/index.ts index 3a4aa5a..1e6b5bb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,8 @@ export { ReplayEngine, type ReplaySnapshot } from "./backtest/ReplayEngine.js"; export { demoDataset, demoTimeline } from "./backtest/demoDataset.js"; export { createRuntime } from "./agents/createRuntime.js"; +export { readNotificationConfigFromEnv } from "./config/notifications.js"; +export { readLangSmithConfigFromEnv } from "./config/observability.js"; export { readProviderConfigFromEnv, type AiProvider } from "./config/secrets.js"; export { createNarrativeGraph, NarrativeGraphRunner } from "./graph/agentGraph.js"; export { @@ -11,6 +13,7 @@ export { GeminiLlmClient, LocalEchoLlmClient } from "./services/MultiProviderLlmClient.js"; +export { DiscordNotifier, type DiscordNotifierOptions } from "./services/AlertService.js"; export { OpenAICompatibleLlmClient } from "./services/OpenAICompatibleLlmClient.js"; export type { Cluster, Document, Narrative, SystemState } from "./types/domain.js"; export type { EmbeddingProvider, LlmClient, NarrativeRepository, Notifier } from "./types/ports.js"; diff --git a/src/services/AlertService.ts b/src/services/AlertService.ts index f96f524..be7beb6 100644 --- a/src/services/AlertService.ts +++ b/src/services/AlertService.ts @@ -15,6 +15,64 @@ export class TelegramNotifierStub implements Notifier { } } +export type DiscordNotifierOptions = { + webhookUrl?: string | undefined; + username?: string | undefined; + avatarUrl?: string | undefined; + fetchFn?: typeof fetch | undefined; +}; + +export class DiscordNotifier implements Notifier { + constructor(private readonly options: DiscordNotifierOptions) {} + + async notify(narrative: Narrative, message: string): Promise { + if (!this.options.webhookUrl) { + return; + } + + const fetchFn = this.options.fetchFn ?? fetch; + const response = await fetchFn(this.options.webhookUrl, { + method: "POST", + headers: { + "content-type": "application/json" + }, + body: JSON.stringify({ + username: this.options.username ?? "Narrative Alpha Agent", + avatar_url: this.options.avatarUrl, + embeds: [ + { + title: narrative.name, + description: message, + color: colorForNarrative(narrative.nipScore), + fields: [ + { + name: "NIP", + value: narrative.nipScore.toFixed(3), + inline: true + }, + { + name: "State", + value: narrative.state, + inline: true + }, + { + name: "Documents", + value: String(narrative.documents.length), + inline: true + } + ], + timestamp: new Date(narrative.updatedAt).toISOString() + } + ] + }) + }); + + if (!response.ok) { + throw new Error(`Discord webhook request failed with status ${response.status}`); + } + } +} + export class AlertService { constructor(private readonly notifiers: Notifier[]) {} @@ -23,3 +81,13 @@ export class AlertService { await Promise.all(this.notifiers.map((notifier) => notifier.notify(narrative, message))); } } + +const colorForNarrative = (nipScore: number): number => { + if (nipScore >= 0.85) { + return 0xff4d4f; + } + if (nipScore >= 0.7) { + return 0xfaad14; + } + return 0x52c41a; +}; diff --git a/tests/notifications.test.ts b/tests/notifications.test.ts new file mode 100644 index 0000000..08df135 --- /dev/null +++ b/tests/notifications.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; +import { readNotificationConfigFromEnv } from "../src/config/notifications.js"; +import { DiscordNotifier } from "../src/services/AlertService.js"; +import type { Narrative } from "../src/types/domain.js"; + +const narrative: Narrative = { + id: "narrative_1", + name: "Agent Payments", + description: "test", + createdAt: 1_735_689_600_000, + updatedAt: 1_735_696_800_000, + state: "accelerating", + documents: ["doc_1", "doc_2", "doc_3"], + metrics: { + velocity: 0.8, + sourceDiversity: 1, + sentiment: 0.75 + }, + nipScore: 0.86 +}; + +describe("notification configuration", () => { + it("reads Discord webhook settings from env", () => { + const config = readNotificationConfigFromEnv({ + DISCORD_WEBHOOK_URL: "https://discord.com/api/webhooks/test", + DISCORD_USERNAME: "NAA", + DISCORD_AVATAR_URL: "https://example.com/avatar.png" + }); + + expect(config.discordWebhookUrl).toBe("https://discord.com/api/webhooks/test"); + expect(config.discordUsername).toBe("NAA"); + expect(config.discordAvatarUrl).toBe("https://example.com/avatar.png"); + }); + + it("posts a Discord embed when configured", async () => { + const requests: Array<{ url: string; body: unknown }> = []; + const notifier = new DiscordNotifier({ + webhookUrl: "https://discord.com/api/webhooks/test", + username: "NAA", + fetchFn: async (input, init) => { + const body = typeof init?.body === "string" ? init.body : ""; + requests.push({ + url: input instanceof Request ? input.url : input.toString(), + body: JSON.parse(body) as unknown + }); + return new Response(null, { status: 204 }); + } + }); + + await notifier.notify(narrative, "Narrative crossed threshold"); + + expect(requests).toHaveLength(1); + expect(requests[0]?.url).toBe("https://discord.com/api/webhooks/test"); + expect(requests[0]?.body).toMatchObject({ + username: "NAA", + embeds: [ + { + title: "Agent Payments", + description: "Narrative crossed threshold" + } + ] + }); + }); + + it("does nothing when Discord is not configured", async () => { + let calls = 0; + const notifier = new DiscordNotifier({ + fetchFn: async () => { + calls += 1; + return new Response(null, { status: 204 }); + } + }); + + await notifier.notify(narrative, "Narrative crossed threshold"); + + expect(calls).toBe(0); + }); +}); diff --git a/tests/observability.test.ts b/tests/observability.test.ts new file mode 100644 index 0000000..991b9bb --- /dev/null +++ b/tests/observability.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest"; +import { readLangSmithConfigFromEnv } from "../src/config/observability.js"; + +describe("LangSmith observability configuration", () => { + it("defaults tracing off with stable project metadata", () => { + const config = readLangSmithConfigFromEnv({}); + + expect(config.tracingEnabled).toBe(false); + expect(config.project).toBe("narrative-alpha-agent"); + expect(config.tags).toContain("langgraph"); + }); + + it("reads LangSmith tracing settings from env", () => { + const config = readLangSmithConfigFromEnv({ + LANGSMITH_TRACING: "true", + LANGSMITH_API_KEY: "lsv2_test", + LANGSMITH_PROJECT: "naa-prod", + LANGSMITH_RUN_NAME: "naa-live", + LANGSMITH_TAGS: "prod,alpha" + }); + + expect(config.tracingEnabled).toBe(true); + expect(config.apiKey).toBe("lsv2_test"); + expect(config.project).toBe("naa-prod"); + expect(config.runName).toBe("naa-live"); + expect(config.tags).toEqual(["prod", "alpha"]); + }); +});