feat(clickhouse): add clickhouse migrations and migrate tooling

.github/workflows/ci.yml

@@ -184,6 +184,28 @@ jobs:
     defaults:
       run:
         working-directory: cloud
+    services:
+      clickhouse:
+        image: clickhouse/clickhouse-server:24.10
+        ports:
+          - 8123:8123
+          - 9000:9000
+        env:
+          CLICKHOUSE_DB: mirascope_analytics
+          CLICKHOUSE_USER: ci
+          CLICKHOUSE_PASSWORD: ci_pass
+          CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
+        options: >-
+          --health-cmd "wget --spider -q http://localhost:8123/ping"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      CLICKHOUSE_URL: http://localhost:8123
+      CLICKHOUSE_USER: ci
+      CLICKHOUSE_PASSWORD: ci_pass
+      CLICKHOUSE_DATABASE: mirascope_analytics
+      CLICKHOUSE_MIGRATE_NATIVE_PORT: "9000"
     steps:
       - uses: actions/checkout@v4
 
@@ -193,6 +215,9 @@ jobs:
       - name: Install dependencies
         run: bun install --frozen-lockfile
 
+      - name: Initialize ClickHouse
+        run: bun run clickhouse:migrate
+
       - name: Run `vitest` with code coverage
         run: bun run test:coverage
 

STRUCTURE.md

@@ -246,12 +246,23 @@ cloud/
 │   │   └── [other files]          # users.ts, organizations.ts, tests, etc.
 │   ├── errors.ts                  # Database errors
 │   └── utils.ts                   # Database utilities
+├── clickhouse/                    # ClickHouse analytics services
+│   ├── client.ts                  # Effect service for ClickHouse access
+│   ├── search.ts                  # Analytics/search queries and transforms
+│   ├── migrate.sh                 # Schema migration runner (run: bun run clickhouse:migrate)
+│   ├── migrations/                # SQL migration files (versioned, applied on startup)
+│   │   ├── *.up.sql
+│   │   └── *.down.sql
+│   └── [other files]              # utils.ts, tests, etc.
+├── workers/                       # Cloudflare Workers (cron)
+│   ├── clickhouseCron.ts          # Cron trigger for outbox sync
+│   └── outboxProcessor.ts         # Shared processing logic
 ├── tests/                         # Test utilities
 │   ├── api.ts                     # API test utilities
 │   └── db.ts                      # Database test utilities
 ├── docker/                        # Docker configuration
 │   ├── compose.yml
-│   └── data/                      # Docker data directory
+│   └── data/                      # Docker data directory (gitignored)
 ├── dist/                          # Build output
 │   ├── client/                    # Client build artifacts
 │   └── server/                    # Server build artifacts
@@ -267,6 +278,32 @@ cloud/
 ├── [other files]                  # e.g. package.json, tsconfig.json, vite.config.ts, etc.
 ```
 
+**ClickHouse Analytics Flow (Cloud)**:
+
+```text
+PostgreSQL (OLTP)
+  └── spans_outbox (insert)
+        └── Cron (Cloudflare scheduled event)
+              └── ClickHouse (analytics)
+```
+
+**ClickHouse Outbox Processing (Detailed)**:
+
+```text
+1) spans_outbox row is created when a span is inserted (db/traces.ts)
+2) Worker picks up the outbox row:
+   - Production: Cloudflare scheduled event -> clickhouseCron.ts
+   - Local dev: run cron via `bun run cron:dev` + `bun run cron:trigger`
+3) processOutboxMessages (outboxProcessor.ts):
+   - lock row (status=processing, lockedAt/lockedBy)
+   - load span + trace from Postgres
+   - transform to ClickHouse row
+   - bulk insert into ClickHouse
+4) Success -> mark completed + processedAt
+5) Failure -> increment retryCount + backoff via processAfter
+6) Cron (clickhouseCron.ts) reprocesses pending/stale rows
+```
+
 **Tooling Choices**:
 
 - **React 19**: Modern React features and performance improvements