diff --git a/cloud/bun.lock b/cloud/bun.lock index e5261ee5ed..a2179a2f7d 100644 --- a/cloud/bun.lock +++ b/cloud/bun.lock @@ -4,6 +4,7 @@ "": { "name": "@mirascope/cloud", "dependencies": { + "@clickhouse/client-web": "^1.16.0", "@effect/platform": "^0.93.3", "@effect/schema": "^0.75.5", "@effect/sql": "^0.48.6", @@ -150,6 +151,10 @@ "@bcoe/v8-coverage": ["@bcoe/v8-coverage@1.0.2", "", {}, "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA=="], + "@clickhouse/client-common": ["@clickhouse/client-common@1.16.0", "", {}, "sha512-qMzkI1NmV29ZjFkNpVSvGNfA0c7sCExlufAQMv+V+5xtNeYXnRfdqzmBLIQoq6Pf1ij0kw/wGLD3HQrl7pTFLA=="], + + "@clickhouse/client-web": ["@clickhouse/client-web@1.16.0", "", { "dependencies": { "@clickhouse/client-common": "1.16.0" } }, "sha512-47+H8GsXXlultL3HFkTu94M7BGXJF8FOwiOUafPpvgCFoqra3o82I0YbWs3vTRqHIOuFjCvuOzrBNFvcT732eg=="], + "@cloudflare/kv-asset-handler": ["@cloudflare/kv-asset-handler@0.4.1", "", { "dependencies": { "mime": "^3.0.0" } }, "sha512-Nu8ahitGFFJztxUml9oD/DLb7Z28C8cd8F46IVQ7y5Btz575pvMY8AqZsXkX7Gds29eCKdMgIHjIvzskHgPSFg=="], "@cloudflare/unenv-preset": ["@cloudflare/unenv-preset@2.7.13", "", { "peerDependencies": { "unenv": "2.0.0-rc.24", "workerd": "^1.20251202.0" }, "optionalPeers": ["workerd"] }, "sha512-NulO1H8R/DzsJguLC0ndMuk4Ufv0KSlN+E54ay9rn9ZCQo0kpAPwwh3LhgpZ96a3Dr6L9LqW57M4CqC34iLOvw=="], diff --git a/cloud/clickhouse/client.test.ts b/cloud/clickhouse/client.test.ts new file mode 100644 index 0000000000..b136e49422 --- /dev/null +++ b/cloud/clickhouse/client.test.ts @@ -0,0 +1,214 @@ +import { describe, it, expect } from "@/tests/clickhouse"; +import { Effect, Layer } from "effect"; +import { ClickHouse, ClickHouseLive } from "@/clickhouse/client"; +import { SettingsService, type Settings } from "@/settings"; +import { ClickHouseError } from "@/errors"; +import { vi } from "vitest"; + +const createTestSettings = (overrides: Partial = {}): Settings => ({ + env: "local", + CLICKHOUSE_URL: process.env.CLICKHOUSE_URL ?? "http://localhost:8123", + CLICKHOUSE_USER: process.env.CLICKHOUSE_USER ?? "default", + CLICKHOUSE_PASSWORD: process.env.CLICKHOUSE_PASSWORD ?? "clickhouse", + CLICKHOUSE_DATABASE: process.env.CLICKHOUSE_DATABASE ?? "mirascope_analytics", + CLICKHOUSE_TLS_ENABLED: false, + CLICKHOUSE_TLS_HOSTNAME_VERIFY: true, + ...overrides, +}); + +const makeTestSettingsLayer = (settings: Settings) => + Layer.succeed(SettingsService, settings); + +const createClickHouseLayer = (settings: Settings) => + ClickHouseLive.pipe(Layer.provide(makeTestSettingsLayer(settings))); + +describe("ClickHouse", () => { + describe("ClickHouseLive", () => { + it.effect("creates a Layer successfully", () => + Effect.gen(function* () { + const client = yield* ClickHouse; + expect(client).toBeDefined(); + expect(client.unsafeQuery).toBeDefined(); + expect(client.insert).toBeDefined(); + expect(client.command).toBeDefined(); + }), + ); + + it.effect("executes unsafeQuery successfully", () => + Effect.gen(function* () { + const client = yield* ClickHouse; + + const result = yield* client.unsafeQuery<{ n: number }>( + "SELECT 1 as n", + ); + + expect(result).toHaveLength(1); + expect(result[0]?.n).toBe(1); + }), + ); + + it.effect("handles unsafeQuery errors with ClickHouseError", () => + Effect.gen(function* () { + const client = yield* ClickHouse; + + const error = yield* client + .unsafeQuery("SELECT * FROM non_existent_table_xyz_123") + .pipe(Effect.flip); + + expect(error).toBeInstanceOf(ClickHouseError); + expect(error.message).toContain("ClickHouse operation failed"); + }), + ); + + it.effect("executes command successfully", () => + Effect.gen(function* () { + const client = yield* ClickHouse; + yield* client.command("SELECT 1"); + }), + ); + + it.effect("skips insert for empty rows", () => + Effect.gen(function* () { + const client = yield* ClickHouse; + yield* client.insert("any_table", []); + }), + ); + }); + + describe("TLS configuration validation", () => { + it("throws error when TLS_SKIP_VERIFY is true", async () => { + const settings = createTestSettings({ + CLICKHOUSE_TLS_ENABLED: true, + CLICKHOUSE_TLS_SKIP_VERIFY: true, + }); + + const program = Effect.gen(function* () { + yield* ClickHouse; + }); + + await expect( + Effect.runPromise( + program.pipe(Effect.provide(createClickHouseLayer(settings))), + ), + ).rejects.toThrow("CLICKHOUSE_TLS_SKIP_VERIFY=true is not supported"); + }); + + it("throws error when TLS_HOSTNAME_VERIFY is false", async () => { + const settings = createTestSettings({ + CLICKHOUSE_TLS_ENABLED: true, + CLICKHOUSE_TLS_HOSTNAME_VERIFY: false, + }); + + const program = Effect.gen(function* () { + yield* ClickHouse; + }); + + await expect( + Effect.runPromise( + program.pipe(Effect.provide(createClickHouseLayer(settings))), + ), + ).rejects.toThrow( + "CLICKHOUSE_TLS_HOSTNAME_VERIFY=false is not supported", + ); + }); + + it("throws error when TLS_CA is provided", async () => { + const settings = createTestSettings({ + CLICKHOUSE_TLS_ENABLED: true, + CLICKHOUSE_TLS_CA: "/nonexistent/ca.pem", + }); + + const program = Effect.gen(function* () { + yield* ClickHouse; + }); + + await expect( + Effect.runPromise( + program.pipe(Effect.provide(createClickHouseLayer(settings))), + ), + ).rejects.toThrow("CLICKHOUSE_TLS_CA is not supported"); + }); + + it("logs warning when TLS_MIN_VERSION is non-default", async () => { + const consoleSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + const settings = createTestSettings({ + CLICKHOUSE_TLS_ENABLED: true, + CLICKHOUSE_TLS_MIN_VERSION: "TLSv1.3", + }); + + const program = Effect.gen(function* () { + yield* ClickHouse; + }); + + await expect( + Effect.runPromise( + program.pipe(Effect.provide(createClickHouseLayer(settings))), + ), + ).rejects.toThrow(); + + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining("CLICKHOUSE_TLS_MIN_VERSION=TLSv1.3"), + ); + + consoleSpy.mockRestore(); + }); + }); + + describe("ClickHouse.layer", () => { + it("creates a layer with provided configuration", () => { + const layer = ClickHouse.layer({ + url: "http://localhost:8123", + user: "default", + password: "test", + database: "test_db", + }); + + expect(layer).toBeDefined(); + expect(Layer.isLayer(layer)).toBe(true); + }); + + it("layer works with unsafeQuery", async () => { + const program = Effect.gen(function* () { + const client = yield* ClickHouse; + return yield* client.unsafeQuery<{ n: number }>("SELECT 1 as n"); + }); + + const result = await Effect.runPromise( + program.pipe( + Effect.provide( + ClickHouse.layer({ + url: process.env.CLICKHOUSE_URL ?? "http://localhost:8123", + user: process.env.CLICKHOUSE_USER ?? "default", + password: process.env.CLICKHOUSE_PASSWORD ?? "clickhouse", + database: + process.env.CLICKHOUSE_DATABASE ?? "mirascope_analytics", + }), + ), + ), + ); + + expect(result).toHaveLength(1); + expect(result[0]?.n).toBe(1); + }); + }); + + describe("ClickHouseLive in production", () => { + it("validates https in production", async () => { + const settings = createTestSettings({ + env: "production", + CLICKHOUSE_URL: "http://clickhouse.example.com", + }); + + const program = Effect.gen(function* () { + yield* ClickHouse; + }); + + await expect( + Effect.runPromise( + program.pipe(Effect.provide(createClickHouseLayer(settings))), + ), + ).rejects.toThrow("must use https://"); + }); + }); +}); diff --git a/cloud/clickhouse/client.ts b/cloud/clickhouse/client.ts new file mode 100644 index 0000000000..bee4773a37 --- /dev/null +++ b/cloud/clickhouse/client.ts @@ -0,0 +1,297 @@ +/** + * @fileoverview ClickHouse client service for analytics data access. + * + * Provides an Effect-based ClickHouse client using @clickhouse/client-web + * (Fetch + Web Streams) compatible with Cloudflare Workers. + * + * ## Architecture + * + * ``` + * ClickHouse.layer (HTTP connection) + * └── ClickHouse (Effect Service) + * ``` + * + * ## Usage + * + * - Local dev / CI: Use `ClickHouseLive` + * - Workers (production): Use `ClickHouseLive` + * + * ## TLS Constraints + * + * - System CA only (no custom CA support) + * + * @example + * ```ts + * import { ClickHouse, ClickHouseLive } from "@/clickhouse/client"; + * + * const program = Effect.gen(function* () { + * const client = yield* ClickHouse; + * const spans = yield* client.unsafeQuery( + * "SELECT * FROM spans_analytics LIMIT 10" + * ); + * return spans; + * }); + * + * await Effect.runPromise( + * program.pipe(Effect.provide(ClickHouseLive)) + * ); + * ``` + */ + +import { Context, Effect, Layer } from "effect"; +import { + createClient, + type ClickHouseClient as WebClickHouseClient, +} from "@clickhouse/client-web"; +import { ClickHouseError } from "@/errors"; +import { SettingsService, type Settings } from "@/settings"; + +// ============================================================================= +// Service Interface +// ============================================================================= + +/** + * ClickHouse configuration options. + */ +export interface ClickHouseConfig { + /** ClickHouse HTTP URL (e.g., http://localhost:8123) */ + url: string; + + /** ClickHouse username */ + user: string; + + /** ClickHouse password */ + password?: string; + + /** ClickHouse database name */ + database: string; +} + +/** + * ClickHouse service interface type. + * + * Provides convenience methods for query, insert, and command operations. + */ +export interface ClickHouseClient { + /** + * Execute a raw SQL query without parameterization. + * WARNING: This method is unsafe and should only be used for trusted SQL. + */ + readonly unsafeQuery: ( + sql: string, + params?: Record, + ) => Effect.Effect; + + /** Insert rows into a table in JSONEachRow format. */ + readonly insert: >( + table: string, + rows: T[], + ) => Effect.Effect; + + /** Execute a DDL/DML command (CREATE, ALTER, etc.). */ + readonly command: (sql: string) => Effect.Effect; +} + +/** + * ClickHouse service. + */ +export class ClickHouse extends Context.Tag("ClickHouse")< + ClickHouse, + ClickHouseClient +>() { + /** + * Default layer using SettingsService for configuration. + * Requires SettingsService to be provided. + * + * Uses @clickhouse/client-web over the ClickHouse HTTP interface. + */ + static Default = Layer.effect( + ClickHouse, + Effect.gen(function* () { + const settings = yield* SettingsService; + const client = createWebClickHouseClient(settings); + + const toClickHouseError = (error: unknown): ClickHouseError => { + if (error instanceof ClickHouseError) return error; + if ( + error && + typeof error === "object" && + "_tag" in error && + error._tag === "SqlError" + ) { + return new ClickHouseError({ + message: "ClickHouse operation failed", + cause: error, + }); + } + return new ClickHouseError({ + message: `ClickHouse operation failed: ${ + error instanceof Error ? error.message : String(error) + }`, + cause: error instanceof Error ? error : undefined, + }); + }; + + const mapClickHouseError = ( + effect: Effect.Effect, + ): Effect.Effect => + effect.pipe(Effect.mapError((error) => toClickHouseError(error))); + + return { + unsafeQuery: ( + query: string, + params?: Record, + ): Effect.Effect => + mapClickHouseError( + Effect.tryPromise({ + try: async () => { + const result = await client.query({ + query, + format: "JSONEachRow", + query_params: params, + }); + return result.json(); + }, + catch: (error) => error, + }), + ), + insert: >( + table: string, + rows: T[], + ): Effect.Effect => { + if (rows.length === 0) return Effect.void; + return mapClickHouseError( + Effect.tryPromise({ + try: async () => { + await client.insert({ + table, + values: rows, + format: "JSONEachRow", + }); + }, + catch: (error) => error, + }), + ); + }, + command: (query: string): Effect.Effect => + mapClickHouseError( + Effect.tryPromise({ + try: async () => { + await client.command({ query }); + }, + catch: (error) => error, + }), + ), + }; + }), + ); + + /** + * Creates a layer with direct configuration. + * Does not require SettingsService. + * + * @param config - ClickHouse connection configuration + */ + static layer = (config: ClickHouseConfig, env: Settings["env"] = "local") => { + const settings: Settings = { + env, + + CLICKHOUSE_URL: config.url, + + CLICKHOUSE_USER: config.user, + + CLICKHOUSE_PASSWORD: config.password, + + CLICKHOUSE_DATABASE: config.database, + }; + + return ClickHouse.Default.pipe( + Layer.provide(Layer.succeed(SettingsService, settings)), + ); + }; +} + +/** + * Validates TLS settings for compatibility with @clickhouse/client-web. + * + * @param settings - Application settings including ClickHouse configuration + * @throws Error if unsupported TLS settings are configured + */ +const validateTLSSettings = (settings: Settings): void => { + if (settings.CLICKHOUSE_TLS_ENABLED) { + if (settings.CLICKHOUSE_TLS_SKIP_VERIFY) { + throw new Error( + "CLICKHOUSE_TLS_SKIP_VERIFY=true is not supported by @clickhouse/client-web. " + + "The client always verifies certificates when TLS is enabled.", + ); + } + + if (settings.CLICKHOUSE_TLS_HOSTNAME_VERIFY === false) { + throw new Error( + "CLICKHOUSE_TLS_HOSTNAME_VERIFY=false is not supported by @clickhouse/client-web. " + + "The client always performs hostname verification.", + ); + } + + if (settings.CLICKHOUSE_TLS_CA) { + throw new Error( + "CLICKHOUSE_TLS_CA is not supported by @clickhouse/client-web. " + + "Custom CA certificates are not available in Fetch-based environments.", + ); + } + + if ( + settings.CLICKHOUSE_TLS_MIN_VERSION && + settings.CLICKHOUSE_TLS_MIN_VERSION !== "TLSv1.2" + ) { + console.warn( + `CLICKHOUSE_TLS_MIN_VERSION=${settings.CLICKHOUSE_TLS_MIN_VERSION} is not supported by @clickhouse/client-web.`, + ); + throw new Error( + "CLICKHOUSE_TLS_MIN_VERSION is not supported by @clickhouse/client-web.", + ); + } + } +}; + +/** + * Creates a ClickHouse web client from Settings. + * + * @param settings - Application settings including ClickHouse configuration + * @returns ClickHouse web client + */ +const createWebClickHouseClient = (settings: Settings): WebClickHouseClient => { + validateTLSSettings(settings); + + if ( + settings.env === "production" && + !settings.CLICKHOUSE_URL?.startsWith("https://") + ) { + throw new Error( + "CLICKHOUSE_URL must use https:// in production (Workers environment)", + ); + } + + return createClient({ + url: settings.CLICKHOUSE_URL, + + username: settings.CLICKHOUSE_USER, + + password: settings.CLICKHOUSE_PASSWORD, + + database: settings.CLICKHOUSE_DATABASE, + + request_timeout: 30000, + + max_open_connections: 10, + }); +}; + +// ============================================================================= +// Default Export +// ============================================================================= + +/** + * Default ClickHouse layer for local development and testing. + */ +export const ClickHouseLive = ClickHouse.Default; diff --git a/cloud/errors.ts b/cloud/errors.ts index bc58e5832d..b83d7fcea2 100644 --- a/cloud/errors.ts +++ b/cloud/errors.ts @@ -252,6 +252,43 @@ export class ProxyError extends Schema.TaggedError()("ProxyError", { static readonly status = 502 as const; } +// ============================================================================= +// Analytics Errors +// ============================================================================= + +/** + * Error that occurs during ClickHouse operations. + * + * This error wraps any failures from ClickHouse client operations, including: + * - Connection errors + * - Query execution errors + * - Insert failures + * - Timeout errors + * + * @example + * ```ts + * const result = yield* clickhouse.unsafeQuery("SELECT * FROM spans").pipe( + * Effect.catchTag("ClickHouseError", (error) => { + * console.error("ClickHouse operation failed:", error.message); + * return Effect.succeed([]); + * }) + * ); + * ``` + */ +export class ClickHouseError extends Schema.TaggedError()( + "ClickHouseError", + { + message: Schema.String, + cause: Schema.optional(Schema.Unknown), + }, +) { + static readonly status = 500 as const; +} + +// ============================================================================= +// Pricing Errors +// ============================================================================= + /** * Error that occurs when pricing data is unavailable for cost estimation. * diff --git a/cloud/package.json b/cloud/package.json index 2e6625fc10..8f25273d55 100644 --- a/cloud/package.json +++ b/cloud/package.json @@ -30,6 +30,7 @@ "lint:eslint:fix": "eslint . --fix" }, "dependencies": { + "@clickhouse/client-web": "^1.16.0", "@effect/platform": "^0.93.3", "@effect/schema": "^0.75.5", "@effect/sql": "^0.48.6", diff --git a/cloud/settings.test.ts b/cloud/settings.test.ts new file mode 100644 index 0000000000..e936b5aa87 --- /dev/null +++ b/cloud/settings.test.ts @@ -0,0 +1,185 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { + getSettings, + getSettingsFromEnvironment, + type CloudflareEnvironment, +} from "@/settings"; + +describe("settings", () => { + describe("getSettings", () => { + const originalEnv = process.env; + + beforeEach(() => { + vi.resetModules(); + process.env = { ...originalEnv }; + }); + + afterEach(() => { + process.env = originalEnv; + }); + + it("returns default values when environment variables are not set", () => { + delete process.env.ENVIRONMENT; + delete process.env.CLICKHOUSE_URL; + delete process.env.CLICKHOUSE_USER; + delete process.env.CLICKHOUSE_DATABASE; + + const settings = getSettings(); + + expect(settings.env).toBe("local"); + expect(settings.CLICKHOUSE_URL).toBe("http://localhost:8123"); + expect(settings.CLICKHOUSE_USER).toBe("default"); + expect(settings.CLICKHOUSE_DATABASE).toBe("mirascope_analytics"); + }); + + it("reads ClickHouse settings from environment variables", () => { + process.env.CLICKHOUSE_URL = "https://ch.example.com"; + process.env.CLICKHOUSE_USER = "testuser"; + process.env.CLICKHOUSE_PASSWORD = "testpass"; + process.env.CLICKHOUSE_DATABASE = "testdb"; + + const settings = getSettings(); + + expect(settings.CLICKHOUSE_URL).toBe("https://ch.example.com"); + expect(settings.CLICKHOUSE_USER).toBe("testuser"); + expect(settings.CLICKHOUSE_PASSWORD).toBe("testpass"); + expect(settings.CLICKHOUSE_DATABASE).toBe("testdb"); + }); + + it("reads TLS settings from environment variables", () => { + process.env.CLICKHOUSE_TLS_ENABLED = "true"; + process.env.CLICKHOUSE_TLS_CA = "/path/to/ca.pem"; + process.env.CLICKHOUSE_TLS_SKIP_VERIFY = "false"; + process.env.CLICKHOUSE_TLS_HOSTNAME_VERIFY = "true"; + process.env.CLICKHOUSE_TLS_MIN_VERSION = "TLSv1.3"; + + const settings = getSettings(); + + expect(settings.CLICKHOUSE_TLS_ENABLED).toBe(true); + expect(settings.CLICKHOUSE_TLS_CA).toBe("/path/to/ca.pem"); + expect(settings.CLICKHOUSE_TLS_SKIP_VERIFY).toBe(false); + expect(settings.CLICKHOUSE_TLS_HOSTNAME_VERIFY).toBe(true); + expect(settings.CLICKHOUSE_TLS_MIN_VERSION).toBe("TLSv1.3"); + }); + + it("defaults TLS_HOSTNAME_VERIFY to true when not set", () => { + delete process.env.CLICKHOUSE_TLS_HOSTNAME_VERIFY; + + const settings = getSettings(); + + expect(settings.CLICKHOUSE_TLS_HOSTNAME_VERIFY).toBe(true); + }); + + it("sets TLS_HOSTNAME_VERIFY to false when explicitly set to 'false'", () => { + process.env.CLICKHOUSE_TLS_HOSTNAME_VERIFY = "false"; + + const settings = getSettings(); + + expect(settings.CLICKHOUSE_TLS_HOSTNAME_VERIFY).toBe(false); + }); + + it("throws error in production when TLS is not enabled", () => { + process.env.ENVIRONMENT = "production"; + process.env.CLICKHOUSE_TLS_ENABLED = "false"; + + expect(() => getSettings()).toThrow( + "CLICKHOUSE_TLS_ENABLED=true is required in production", + ); + }); + + it("throws error in production when TLS_SKIP_VERIFY is true", () => { + process.env.ENVIRONMENT = "production"; + process.env.CLICKHOUSE_TLS_ENABLED = "true"; + process.env.CLICKHOUSE_TLS_SKIP_VERIFY = "true"; + + expect(() => getSettings()).toThrow( + "CLICKHOUSE_TLS_SKIP_VERIFY=true is not allowed in production", + ); + }); + + it("throws error in production when TLS_HOSTNAME_VERIFY is false", () => { + process.env.ENVIRONMENT = "production"; + process.env.CLICKHOUSE_TLS_ENABLED = "true"; + process.env.CLICKHOUSE_TLS_SKIP_VERIFY = "false"; + process.env.CLICKHOUSE_TLS_HOSTNAME_VERIFY = "false"; + + expect(() => getSettings()).toThrow( + "CLICKHOUSE_TLS_HOSTNAME_VERIFY=true is required in production", + ); + }); + + it("succeeds in production with valid TLS settings", () => { + process.env.ENVIRONMENT = "production"; + process.env.CLICKHOUSE_TLS_ENABLED = "true"; + process.env.CLICKHOUSE_TLS_SKIP_VERIFY = "false"; + process.env.CLICKHOUSE_TLS_HOSTNAME_VERIFY = "true"; + + const settings = getSettings(); + + expect(settings.env).toBe("production"); + expect(settings.CLICKHOUSE_TLS_ENABLED).toBe(true); + }); + }); + + describe("getSettingsFromEnvironment", () => { + it("defaults env to 'local' when ENVIRONMENT is not set", () => { + const env: CloudflareEnvironment = {}; + const settings = getSettingsFromEnvironment(env); + + expect(settings.env).toBe("local"); + }); + + it("uses ENVIRONMENT from env bindings", () => { + const env: CloudflareEnvironment = { ENVIRONMENT: "production" }; + const settings = getSettingsFromEnvironment(env); + + expect(settings.env).toBe("production"); + }); + + it("maps all ClickHouse settings from env", () => { + const env: CloudflareEnvironment = { + CLICKHOUSE_URL: "https://ch.example.com", + CLICKHOUSE_USER: "user", + CLICKHOUSE_PASSWORD: "pass", + CLICKHOUSE_DATABASE: "analytics", + }; + const settings = getSettingsFromEnvironment(env); + + expect(settings.CLICKHOUSE_URL).toBe("https://ch.example.com"); + expect(settings.CLICKHOUSE_USER).toBe("user"); + expect(settings.CLICKHOUSE_PASSWORD).toBe("pass"); + expect(settings.CLICKHOUSE_DATABASE).toBe("analytics"); + }); + + it("returns default ClickHouse values when not set", () => { + const env: CloudflareEnvironment = {}; + const settings = getSettingsFromEnvironment(env); + + expect(settings.CLICKHOUSE_URL).toBe("http://localhost:8123"); + expect(settings.CLICKHOUSE_USER).toBe("default"); + expect(settings.CLICKHOUSE_DATABASE).toBe("mirascope_analytics"); + }); + + it("maps TLS settings from env", () => { + const env: CloudflareEnvironment = { + CLICKHOUSE_TLS_ENABLED: "true", + CLICKHOUSE_TLS_CA: "/path/to/ca.pem", + CLICKHOUSE_TLS_SKIP_VERIFY: "true", + CLICKHOUSE_TLS_HOSTNAME_VERIFY: "false", + }; + const settings = getSettingsFromEnvironment(env); + + expect(settings.CLICKHOUSE_TLS_ENABLED).toBe(true); + expect(settings.CLICKHOUSE_TLS_CA).toBe("/path/to/ca.pem"); + expect(settings.CLICKHOUSE_TLS_SKIP_VERIFY).toBe(true); + expect(settings.CLICKHOUSE_TLS_HOSTNAME_VERIFY).toBe(false); + }); + + it("defaults TLS_HOSTNAME_VERIFY to true when not set", () => { + const env: CloudflareEnvironment = {}; + const settings = getSettingsFromEnvironment(env); + + expect(settings.CLICKHOUSE_TLS_HOSTNAME_VERIFY).toBe(true); + }); + }); +}); diff --git a/cloud/settings.ts b/cloud/settings.ts index 3df9ef3c1b..b86186dcf9 100644 --- a/cloud/settings.ts +++ b/cloud/settings.ts @@ -14,6 +14,17 @@ export type Settings = { readonly ANTHROPIC_API_KEY?: string; readonly GEMINI_API_KEY?: string; readonly OPENAI_API_KEY?: string; + // ClickHouse settings + readonly CLICKHOUSE_URL?: string; + readonly CLICKHOUSE_USER?: string; + readonly CLICKHOUSE_PASSWORD?: string; + readonly CLICKHOUSE_DATABASE?: string; + // TLS settings (Node.js environment only) + readonly CLICKHOUSE_TLS_ENABLED?: boolean; + readonly CLICKHOUSE_TLS_CA?: string; + readonly CLICKHOUSE_TLS_SKIP_VERIFY?: boolean; + readonly CLICKHOUSE_TLS_HOSTNAME_VERIFY?: boolean; + readonly CLICKHOUSE_TLS_MIN_VERSION?: string; }; export class SettingsService extends Context.Tag("SettingsService")< @@ -22,7 +33,10 @@ export class SettingsService extends Context.Tag("SettingsService")< >() {} export function getSettings(): Settings { - return { + const clickhouseTlsHostnameVerifyEnv = + process.env.CLICKHOUSE_TLS_HOSTNAME_VERIFY; + + const settings: Settings = { env: process.env.ENVIRONMENT || "local", DATABASE_URL: process.env.DATABASE_URL, GITHUB_CLIENT_ID: process.env.GITHUB_CLIENT_ID, @@ -36,5 +50,82 @@ export function getSettings(): Settings { ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY, GEMINI_API_KEY: process.env.GEMINI_API_KEY, OPENAI_API_KEY: process.env.OPENAI_API_KEY, + // ClickHouse settings + CLICKHOUSE_URL: process.env.CLICKHOUSE_URL || "http://localhost:8123", + CLICKHOUSE_USER: process.env.CLICKHOUSE_USER || "default", + CLICKHOUSE_PASSWORD: process.env.CLICKHOUSE_PASSWORD, + CLICKHOUSE_DATABASE: + process.env.CLICKHOUSE_DATABASE || "mirascope_analytics", + // TLS settings + CLICKHOUSE_TLS_ENABLED: process.env.CLICKHOUSE_TLS_ENABLED === "true", + CLICKHOUSE_TLS_CA: process.env.CLICKHOUSE_TLS_CA, + CLICKHOUSE_TLS_SKIP_VERIFY: + process.env.CLICKHOUSE_TLS_SKIP_VERIFY === "true", + CLICKHOUSE_TLS_HOSTNAME_VERIFY: clickhouseTlsHostnameVerifyEnv !== "false", + CLICKHOUSE_TLS_MIN_VERSION: + process.env.CLICKHOUSE_TLS_MIN_VERSION || "TLSv1.2", }; + + // Production environment validation (Node.js environment) + if (settings.env === "production") { + if (!settings.CLICKHOUSE_TLS_ENABLED) { + throw new Error("CLICKHOUSE_TLS_ENABLED=true is required in production"); + } + if (settings.CLICKHOUSE_TLS_SKIP_VERIFY) { + throw new Error( + "CLICKHOUSE_TLS_SKIP_VERIFY=true is not allowed in production", + ); + } + if (!settings.CLICKHOUSE_TLS_HOSTNAME_VERIFY) { + throw new Error( + "CLICKHOUSE_TLS_HOSTNAME_VERIFY=true is required in production", + ); + } + } + + return settings; +} + +/** + * Cloudflare Workers environment type for settings extraction. + * Extends as needed for additional bindings. + */ +export type CloudflareEnvironment = { + ENVIRONMENT?: string; + DATABASE_URL?: string; + CLICKHOUSE_URL?: string; + CLICKHOUSE_USER?: string; + CLICKHOUSE_PASSWORD?: string; + CLICKHOUSE_DATABASE?: string; + CLICKHOUSE_TLS_ENABLED?: string; + CLICKHOUSE_TLS_CA?: string; + CLICKHOUSE_TLS_SKIP_VERIFY?: string; + CLICKHOUSE_TLS_HOSTNAME_VERIFY?: string; + // Add other bindings as needed +}; + +/** + * Get settings from Cloudflare Workers environment bindings. + * Used by Queue Consumer, Cron Trigger, etc. + */ +export function getSettingsFromEnvironment( + env: CloudflareEnvironment, +): Settings { + const settings: Settings = { + env: env.ENVIRONMENT || "local", + DATABASE_URL: env.DATABASE_URL, + // ClickHouse settings + CLICKHOUSE_URL: env.CLICKHOUSE_URL || "http://localhost:8123", + CLICKHOUSE_USER: env.CLICKHOUSE_USER || "default", + CLICKHOUSE_PASSWORD: env.CLICKHOUSE_PASSWORD, + CLICKHOUSE_DATABASE: env.CLICKHOUSE_DATABASE || "mirascope_analytics", + // TLS settings (Workers environment has limited TLS control) + CLICKHOUSE_TLS_ENABLED: env.CLICKHOUSE_TLS_ENABLED === "true", + CLICKHOUSE_TLS_CA: env.CLICKHOUSE_TLS_CA, + CLICKHOUSE_TLS_SKIP_VERIFY: env.CLICKHOUSE_TLS_SKIP_VERIFY === "true", + CLICKHOUSE_TLS_HOSTNAME_VERIFY: + env.CLICKHOUSE_TLS_HOSTNAME_VERIFY !== "false", + }; + + return settings; } diff --git a/cloud/tests/clickhouse.ts b/cloud/tests/clickhouse.ts new file mode 100644 index 0000000000..c8ce5728ce --- /dev/null +++ b/cloud/tests/clickhouse.ts @@ -0,0 +1,85 @@ +import { Effect, Layer } from "effect"; +import { describe, expect, it as vitestIt } from "@effect/vitest"; +import { createCustomIt } from "@/tests/shared"; +import { ClickHouse } from "@/clickhouse/client"; +import { SettingsService, type Settings } from "@/settings"; + +// Re-export describe and expect for convenience +export { describe, expect }; + +// Environment variables for ClickHouse connection +const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL ?? "http://localhost:8123"; +const CLICKHOUSE_USER = process.env.CLICKHOUSE_USER ?? "default"; +const CLICKHOUSE_PASSWORD = process.env.CLICKHOUSE_PASSWORD ?? "clickhouse"; +const CLICKHOUSE_DATABASE = + process.env.CLICKHOUSE_DATABASE ?? "mirascope_analytics"; + +const isClickHouseAvailable = async (): Promise => { + try { + const response = await fetch(`${CLICKHOUSE_URL}/ping`, { + method: "GET", + signal: AbortSignal.timeout(2000), + }); + return response.ok; + } catch { + return false; + } +}; + +const clickhouseAvailable = await isClickHouseAvailable(); + +/** + * Test settings for ClickHouse. + */ +const testSettings: Settings = { + env: "local", + CLICKHOUSE_URL, + CLICKHOUSE_USER, + CLICKHOUSE_PASSWORD, + CLICKHOUSE_DATABASE, + CLICKHOUSE_TLS_ENABLED: false, +}; + +/** + * Settings layer for tests. + */ +const TestSettingsLayer = Layer.succeed(SettingsService, testSettings); + +/** + * ClickHouse layer for tests. + * Note: Layer may fail with SqlError | ConfigError during initialization. + */ +export const TestClickHouse = ClickHouse.Default.pipe( + Layer.provide(TestSettingsLayer), +); + +/** + * Services that are automatically provided to all `it.effect` tests. + */ +export type TestServices = ClickHouse; + +/** + * Wraps a test function to automatically provide TestClickHouse. + */ +const wrapEffectTest = + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (original: any) => + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (name: any, fn: any, timeout?: any) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const runner = clickhouseAvailable ? original : vitestIt.effect.skip; + // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-return + return runner( + name, + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unnecessary-type-assertion + () => (fn() as any).pipe(Effect.provide(TestClickHouse)), + timeout, + ); + }; + +/** + * Type-safe `it` with `it.effect` that automatically provides TestClickHouse. + * + * Use this instead of importing directly from @effect/vitest. + */ +export const it = createCustomIt(wrapEffectTest);