diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..78c8e87 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + check: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 18 + cache: pnpm + + - run: pnpm install --frozen-lockfile + + - run: pnpm exec secretlint "**/*" + + - run: pnpm test -- --run diff --git a/__tests__/api-contract.test.ts b/__tests__/api-contract.test.ts new file mode 100644 index 0000000..3eb25ae --- /dev/null +++ b/__tests__/api-contract.test.ts @@ -0,0 +1,67 @@ +import { describe, it, expect } from "vitest"; +import { ok, fail, ApiValidationError } from "@/lib/opentrust/api-contract"; + +describe("ok()", () => { + it("wraps data in success envelope", () => { + const result = ok({ foo: "bar" }); + expect(result.ok).toBe(true); + expect(result.data).toEqual({ foo: "bar" }); + }); + + it("works with null data", () => { + const result = ok(null); + expect(result.ok).toBe(true); + expect(result.data).toBeNull(); + }); +}); + +describe("fail()", () => { + it("formats ApiValidationError with code and message", () => { + const error = new ApiValidationError("Bad input", { code: "invalid_input", status: 400 }); + const result = fail(error); + + expect(result.ok).toBe(false); + expect(result.error.code).toBe("invalid_input"); + expect(result.error.message).toBe("Bad input"); + expect(result.status).toBe(400); + }); + + it("includes details when provided", () => { + const error = new ApiValidationError("Missing field", { + code: "missing_field", + details: { field: "title" }, + }); + const result = fail(error); + expect(result.error.details).toEqual({ field: "title" }); + }); + + it("returns internal_error for unknown errors", () => { + const result = fail(new Error("something broke")); + expect(result.ok).toBe(false); + expect(result.error.code).toBe("internal_error"); + expect(result.status).toBe(500); + // Should NOT leak the original error message + expect(result.error.message).not.toContain("something broke"); + }); + + it("handles non-Error thrown values", () => { + const result = fail("string error"); + expect(result.ok).toBe(false); + expect(result.error.code).toBe("internal_error"); + }); +}); + +describe("ApiValidationError", () => { + it("defaults to status 400 and code invalid_request", () => { + const error = new ApiValidationError("test"); + expect(error.status).toBe(400); + expect(error.code).toBe("invalid_request"); + expect(error.name).toBe("ApiValidationError"); + }); + + it("accepts custom status and code", () => { + const error = new ApiValidationError("forbidden", { status: 403, code: "forbidden" }); + expect(error.status).toBe(403); + expect(error.code).toBe("forbidden"); + }); +}); diff --git a/__tests__/artifact-extract.test.ts b/__tests__/artifact-extract.test.ts new file mode 100644 index 0000000..c84927d --- /dev/null +++ b/__tests__/artifact-extract.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect } from "vitest"; +import { extractArtifactsFromText } from "@/lib/opentrust/artifact-extract"; + +describe("extractArtifactsFromText", () => { + it("extracts URLs", () => { + const text = "Check out https://example.com/docs and http://localhost:3000"; + const artifacts = extractArtifactsFromText(text); + + const urls = artifacts.filter((a) => a.kind === "url"); + expect(urls.length).toBeGreaterThanOrEqual(2); + expect(urls.some((a) => a.uri.includes("example.com"))).toBe(true); + }); + + it("extracts GitHub-style repo references", () => { + const text = "See OpenKnots/OpenTrust for details"; + const artifacts = extractArtifactsFromText(text); + + const repos = artifacts.filter((a) => a.kind === "repo"); + expect(repos.some((a) => a.uri === "OpenKnots/OpenTrust")).toBe(true); + }); + + it("extracts doc file references", () => { + const text = "Updated docs/ARCHITECTURE.md and lib/opentrust/db.ts"; + const artifacts = extractArtifactsFromText(text); + + const docs = artifacts.filter((a) => a.kind === "doc"); + expect(docs.some((a) => a.uri.includes("ARCHITECTURE.md"))).toBe(true); + expect(docs.some((a) => a.uri.includes("db.ts"))).toBe(true); + }); + + it("deduplicates artifacts by kind+uri", () => { + const text = "See https://example.com and also https://example.com again"; + const artifacts = extractArtifactsFromText(text); + + const urls = artifacts.filter((a) => a.kind === "url" && a.uri.includes("example.com")); + expect(urls).toHaveLength(1); + }); + + it("limits to 24 artifacts", () => { + const urls = Array.from({ length: 30 }, (_, i) => `https://example.com/page${i}`).join(" "); + const artifacts = extractArtifactsFromText(urls); + expect(artifacts.length).toBeLessThanOrEqual(24); + }); + + it("returns empty for text with no artifacts", () => { + const artifacts = extractArtifactsFromText("Just plain text with no links or references"); + // May or may not have matches depending on regex — at minimum should not throw + expect(Array.isArray(artifacts)).toBe(true); + }); +}); diff --git a/__tests__/auth-rate-limit.test.ts b/__tests__/auth-rate-limit.test.ts new file mode 100644 index 0000000..2e2c781 --- /dev/null +++ b/__tests__/auth-rate-limit.test.ts @@ -0,0 +1,80 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import { + getLoginRateLimit, + recordLoginFailure, + clearLoginFailures, + getRateLimitKey, +} from "@/lib/opentrust/auth-rate-limit"; + +describe("getRateLimitKey", () => { + it("returns the IP when provided", () => { + expect(getRateLimitKey("192.168.1.1")).toBe("192.168.1.1"); + }); + + it("trims whitespace", () => { + expect(getRateLimitKey(" 10.0.0.1 ")).toBe("10.0.0.1"); + }); + + it('returns "unknown" for null/undefined/empty', () => { + expect(getRateLimitKey(null)).toBe("unknown"); + expect(getRateLimitKey(undefined)).toBe("unknown"); + expect(getRateLimitKey("")).toBe("unknown"); + }); +}); + +describe("getLoginRateLimit", () => { + beforeEach(() => { + clearLoginFailures("test-ip-limit"); + }); + + it("starts with 5 remaining attempts", () => { + const state = getLoginRateLimit("test-ip-limit"); + expect(state.remaining).toBe(5); + expect(state.count).toBe(0); + }); + + it("returns a future resetAt timestamp", () => { + const state = getLoginRateLimit("test-ip-limit"); + expect(state.resetAt).toBeGreaterThan(Date.now()); + }); +}); + +describe("recordLoginFailure", () => { + beforeEach(() => { + clearLoginFailures("test-ip-fail"); + }); + + it("increments the failure count", () => { + recordLoginFailure("test-ip-fail"); + const state = getLoginRateLimit("test-ip-fail"); + expect(state.remaining).toBe(4); + }); + + it("marks as limited after 5 failures", () => { + for (let i = 0; i < 4; i++) { + const result = recordLoginFailure("test-ip-fail"); + expect(result.limited).toBe(false); + } + const fifth = recordLoginFailure("test-ip-fail"); + expect(fifth.limited).toBe(true); + }); + + it("returns 0 remaining after limit reached", () => { + for (let i = 0; i < 5; i++) { + recordLoginFailure("test-ip-fail"); + } + const state = getLoginRateLimit("test-ip-fail"); + expect(state.remaining).toBe(0); + }); +}); + +describe("clearLoginFailures", () => { + it("resets the failure count", () => { + recordLoginFailure("test-ip-clear"); + recordLoginFailure("test-ip-clear"); + clearLoginFailures("test-ip-clear"); + + const state = getLoginRateLimit("test-ip-clear"); + expect(state.remaining).toBe(5); + }); +}); diff --git a/__tests__/db.test.ts b/__tests__/db.test.ts new file mode 100644 index 0000000..eb38d6c --- /dev/null +++ b/__tests__/db.test.ts @@ -0,0 +1,123 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import Database from "better-sqlite3"; +import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs"; +import path from "node:path"; + +/** + * These tests exercise the database layer against a real SQLite instance + * using a temporary directory, bypassing the singleton to avoid polluting + * the development database. + */ + +const TEST_DIR = path.join(process.cwd(), "storage", "__test__"); +const TEST_DB_PATH = path.join(TEST_DIR, "test.sqlite"); +const MIGRATION_PATH = path.join(process.cwd(), "db", "0001_init.sql"); + +function freshDb() { + if (!existsSync(TEST_DIR)) mkdirSync(TEST_DIR, { recursive: true }); + if (existsSync(TEST_DB_PATH)) rmSync(TEST_DB_PATH); + + const db = new Database(TEST_DB_PATH); + db.exec("PRAGMA journal_mode=WAL;"); + db.exec("PRAGMA foreign_keys=ON;"); + return db; +} + +describe("database initialization", () => { + let db: Database.Database; + + beforeEach(() => { + db = freshDb(); + }); + + afterEach(() => { + db.close(); + if (existsSync(TEST_DB_PATH)) rmSync(TEST_DB_PATH); + }); + + it("enables WAL journal mode", () => { + const row = db.prepare("PRAGMA journal_mode;").get() as { journal_mode: string }; + expect(row.journal_mode).toBe("wal"); + }); + + it("enables foreign keys", () => { + const row = db.prepare("PRAGMA foreign_keys;").get() as { foreign_keys: number }; + expect(row.foreign_keys).toBe(1); + }); + + it("applies the init migration without errors", () => { + const migrationSql = readFileSync(MIGRATION_PATH, "utf8"); + expect(() => db.exec(migrationSql)).not.toThrow(); + }); + + it("creates expected core tables after migration", () => { + const migrationSql = readFileSync(MIGRATION_PATH, "utf8"); + db.exec(migrationSql); + + const tables = db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name;") + .all() as { name: string }[]; + + const tableNames = tables.map((t) => t.name); + + expect(tableNames).toContain("sessions"); + expect(tableNames).toContain("traces"); + expect(tableNames).toContain("events"); + expect(tableNames).toContain("tool_calls"); + expect(tableNames).toContain("workflow_runs"); + expect(tableNames).toContain("workflow_steps"); + expect(tableNames).toContain("artifacts"); + expect(tableNames).toContain("capabilities"); + }); + + it("migration is idempotent — running twice does not error", () => { + const migrationSql = readFileSync(MIGRATION_PATH, "utf8"); + db.exec(migrationSql); + expect(() => db.exec(migrationSql)).not.toThrow(); + }); +}); + +describe("transaction support", () => { + let db: Database.Database; + + beforeEach(() => { + db = freshDb(); + const migrationSql = readFileSync(MIGRATION_PATH, "utf8"); + db.exec(migrationSql); + }); + + afterEach(() => { + db.close(); + if (existsSync(TEST_DB_PATH)) rmSync(TEST_DB_PATH); + }); + + it("commits all writes on success", () => { + const run = db.transaction(() => { + db.prepare( + "INSERT INTO capabilities (id, kind, name, metadata_json) VALUES (?, ?, ?, ?)", + ).run("test:a", "skill", "a", "{}"); + db.prepare( + "INSERT INTO capabilities (id, kind, name, metadata_json) VALUES (?, ?, ?, ?)", + ).run("test:b", "skill", "b", "{}"); + }); + + run(); + + const rows = db.prepare("SELECT id FROM capabilities WHERE id LIKE 'test:%'").all() as { id: string }[]; + expect(rows).toHaveLength(2); + }); + + it("rolls back all writes on failure", () => { + const run = db.transaction(() => { + db.prepare( + "INSERT INTO capabilities (id, kind, name, metadata_json) VALUES (?, ?, ?, ?)", + ).run("test:c", "skill", "c", "{}"); + throw new Error("deliberate failure"); + }); + + expect(() => run()).toThrow("deliberate failure"); + + const rows = db.prepare("SELECT id FROM capabilities WHERE id = 'test:c'").all(); + expect(rows).toHaveLength(0); + }); +}); diff --git a/__tests__/ingestion-state.test.ts b/__tests__/ingestion-state.test.ts new file mode 100644 index 0000000..3704002 --- /dev/null +++ b/__tests__/ingestion-state.test.ts @@ -0,0 +1,91 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import Database from "better-sqlite3"; +import { existsSync, mkdirSync, rmSync } from "node:fs"; +import path from "node:path"; + +/** + * Tests for ingestion state tracking — cursor management for incremental imports. + * Uses a standalone SQLite instance to avoid polluting the development database. + */ + +const TEST_DIR = path.join(process.cwd(), "storage", "__test__"); +const TEST_DB_PATH = path.join(TEST_DIR, "ingestion-state-test.sqlite"); + +function setup() { + if (!existsSync(TEST_DIR)) mkdirSync(TEST_DIR, { recursive: true }); + if (existsSync(TEST_DB_PATH)) rmSync(TEST_DB_PATH); + + const db = new Database(TEST_DB_PATH); + db.exec("PRAGMA journal_mode=WAL;"); + db.exec("PRAGMA foreign_keys=ON;"); + db.exec(` + CREATE TABLE IF NOT EXISTS ingestion_state ( + source_key TEXT PRIMARY KEY, + source_kind TEXT NOT NULL, + cursor_text TEXT, + cursor_number INTEGER, + last_run_at TEXT, + last_status TEXT, + imported_count INTEGER NOT NULL DEFAULT 0, + metadata_json TEXT NOT NULL DEFAULT '{}' + ); + `); + return db; +} + +describe("ingestion_state table", () => { + let db: Database.Database; + + beforeEach(() => { + db = setup(); + }); + + afterEach(() => { + db.close(); + if (existsSync(TEST_DB_PATH)) rmSync(TEST_DB_PATH); + }); + + it("inserts a new ingestion state record", () => { + db.prepare(` + INSERT INTO ingestion_state (source_key, source_kind, cursor_text, cursor_number, last_run_at, last_status, imported_count, metadata_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `).run("test:source:1", "session-jsonl", "abc", 123, "2026-01-01T00:00:00Z", "ok", 5, "{}"); + + const row = db.prepare("SELECT * FROM ingestion_state WHERE source_key = ?").get("test:source:1") as Record; + expect(row).toBeTruthy(); + expect(row.source_kind).toBe("session-jsonl"); + expect(row.cursor_number).toBe(123); + expect(row.imported_count).toBe(5); + }); + + it("upserts on conflict — updates existing record", () => { + db.prepare(` + INSERT INTO ingestion_state (source_key, source_kind, cursor_number, last_status, imported_count) + VALUES (?, ?, ?, ?, ?) + `).run("test:source:2", "cron", 100, "ok", 3); + + db.prepare(` + INSERT INTO ingestion_state (source_key, source_kind, cursor_number, last_status, imported_count) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(source_key) DO UPDATE SET + cursor_number=excluded.cursor_number, + last_status=excluded.last_status, + imported_count=excluded.imported_count + `).run("test:source:2", "cron", 200, "ok", 7); + + const row = db.prepare("SELECT * FROM ingestion_state WHERE source_key = ?").get("test:source:2") as Record; + expect(row.cursor_number).toBe(200); + expect(row.imported_count).toBe(7); + }); + + it("handles null cursor values", () => { + db.prepare(` + INSERT INTO ingestion_state (source_key, source_kind, cursor_text, cursor_number, last_status, imported_count) + VALUES (?, ?, ?, ?, ?, ?) + `).run("test:source:3", "manual", null, null, "ok", 0); + + const row = db.prepare("SELECT * FROM ingestion_state WHERE source_key = ?").get("test:source:3") as Record; + expect(row.cursor_text).toBeNull(); + expect(row.cursor_number).toBeNull(); + }); +}); diff --git a/__tests__/memory-entries.test.ts b/__tests__/memory-entries.test.ts new file mode 100644 index 0000000..f8508b6 --- /dev/null +++ b/__tests__/memory-entries.test.ts @@ -0,0 +1,305 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import Database from "better-sqlite3"; +import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs"; +import path from "node:path"; + +/** + * Tests for memory entry CRUD operations. + * Uses standalone SQLite to avoid polluting the development database. + */ + +const TEST_DIR = path.join(process.cwd(), "storage", "__test__"); +const TEST_DB_PATH = path.join(TEST_DIR, "memory-entries-test.sqlite"); +const MIGRATION_PATH = path.join(process.cwd(), "db", "0001_init.sql"); + +function setup() { + if (!existsSync(TEST_DIR)) mkdirSync(TEST_DIR, { recursive: true }); + if (existsSync(TEST_DB_PATH)) rmSync(TEST_DB_PATH); + + const db = new Database(TEST_DB_PATH); + db.exec("PRAGMA journal_mode=WAL;"); + db.exec("PRAGMA foreign_keys=ON;"); + db.exec(readFileSync(MIGRATION_PATH, "utf8")); + + // Apply memory-specific tables from ensureMigrated() + db.exec(` + CREATE TABLE IF NOT EXISTS ingestion_state ( + source_key TEXT PRIMARY KEY, + source_kind TEXT NOT NULL, + cursor_text TEXT, + cursor_number INTEGER, + last_run_at TEXT, + last_status TEXT, + imported_count INTEGER NOT NULL DEFAULT 0, + metadata_json TEXT NOT NULL DEFAULT '{}' + ); + + CREATE TABLE IF NOT EXISTS memory_entries ( + id TEXT PRIMARY KEY, + kind TEXT NOT NULL, + title TEXT NOT NULL, + body TEXT NOT NULL, + summary TEXT, + retention_class TEXT NOT NULL, + review_status TEXT NOT NULL, + review_notes TEXT, + confidence_score REAL, + confidence_reason TEXT, + uncertainty_summary TEXT, + author_type TEXT NOT NULL, + author_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + reviewed_at TEXT, + reviewed_by TEXT, + archived_at TEXT + ); + + CREATE TABLE IF NOT EXISTS memory_entry_origins ( + memory_entry_id TEXT NOT NULL REFERENCES memory_entries(id) ON DELETE CASCADE, + origin_type TEXT NOT NULL, + origin_id TEXT NOT NULL, + relationship TEXT NOT NULL, + PRIMARY KEY (memory_entry_id, origin_type, origin_id, relationship) + ); + + CREATE TABLE IF NOT EXISTS memory_entry_tags ( + memory_entry_id TEXT NOT NULL REFERENCES memory_entries(id) ON DELETE CASCADE, + tag TEXT NOT NULL, + PRIMARY KEY (memory_entry_id, tag) + ); + + CREATE TABLE IF NOT EXISTS memory_entry_versions ( + id TEXT NOT NULL, + version INTEGER NOT NULL, + title TEXT NOT NULL, + body TEXT NOT NULL, + summary TEXT, + retention_class TEXT NOT NULL, + review_status TEXT NOT NULL, + changed_by_type TEXT NOT NULL, + changed_by_id TEXT, + changed_at TEXT NOT NULL, + change_reason TEXT, + PRIMARY KEY (id, version) + ); + `); + return db; +} + +function insertMemoryEntry(db: Database.Database, overrides: Partial> = {}) { + const defaults = { + id: `mem_test_${Date.now()}`, + kind: "memoryEntry", + title: "Test Entry", + body: "Test body content", + summary: null, + retention_class: "working", + review_status: "draft", + review_notes: null, + confidence_score: null, + confidence_reason: null, + uncertainty_summary: null, + author_type: "user", + author_id: null, + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + reviewed_at: null, + reviewed_by: null, + archived_at: null, + }; + const entry = { ...defaults, ...overrides }; + + db.prepare(` + INSERT INTO memory_entries ( + id, kind, title, body, summary, retention_class, review_status, + review_notes, confidence_score, confidence_reason, uncertainty_summary, + author_type, author_id, created_at, updated_at, reviewed_at, reviewed_by, archived_at + ) VALUES ( + :id, :kind, :title, :body, :summary, :retention_class, :review_status, + :review_notes, :confidence_score, :confidence_reason, :uncertainty_summary, + :author_type, :author_id, :created_at, :updated_at, :reviewed_at, :reviewed_by, :archived_at + ) + `).run(entry); + + return entry; +} + +describe("memory_entries table", () => { + let db: Database.Database; + + beforeEach(() => { + db = setup(); + }); + + afterEach(() => { + db.close(); + if (existsSync(TEST_DB_PATH)) rmSync(TEST_DB_PATH); + }); + + it("inserts and retrieves a memory entry", () => { + const entry = insertMemoryEntry(db, { id: "mem_crud_1", title: "My First Memory" }); + const row = db.prepare("SELECT * FROM memory_entries WHERE id = ?").get(entry.id) as Record; + + expect(row).toBeTruthy(); + expect(row.title).toBe("My First Memory"); + expect(row.retention_class).toBe("working"); + expect(row.review_status).toBe("draft"); + }); + + it("updates review status", () => { + const entry = insertMemoryEntry(db, { id: "mem_review_1" }); + + db.prepare("UPDATE memory_entries SET review_status = ?, reviewed_at = ? WHERE id = ?") + .run("approved", new Date().toISOString(), entry.id); + + const row = db.prepare("SELECT review_status, reviewed_at FROM memory_entries WHERE id = ?") + .get(entry.id) as Record; + expect(row.review_status).toBe("approved"); + expect(row.reviewed_at).toBeTruthy(); + }); + + it("deletes entry cascades to origins and tags", () => { + const entry = insertMemoryEntry(db, { id: "mem_cascade_1" }); + + db.prepare("INSERT INTO memory_entry_origins (memory_entry_id, origin_type, origin_id, relationship) VALUES (?, ?, ?, ?)") + .run(entry.id, "trace", "trace:1", "derived_from"); + db.prepare("INSERT INTO memory_entry_tags (memory_entry_id, tag) VALUES (?, ?)") + .run(entry.id, "test-tag"); + + // Verify they exist + expect(db.prepare("SELECT COUNT(*) as c FROM memory_entry_origins WHERE memory_entry_id = ?").get(entry.id)).toEqual({ c: 1 }); + expect(db.prepare("SELECT COUNT(*) as c FROM memory_entry_tags WHERE memory_entry_id = ?").get(entry.id)).toEqual({ c: 1 }); + + // Delete the entry + db.prepare("DELETE FROM memory_entries WHERE id = ?").run(entry.id); + + // Cascaded + expect(db.prepare("SELECT COUNT(*) as c FROM memory_entry_origins WHERE memory_entry_id = ?").get(entry.id)).toEqual({ c: 0 }); + expect(db.prepare("SELECT COUNT(*) as c FROM memory_entry_tags WHERE memory_entry_id = ?").get(entry.id)).toEqual({ c: 0 }); + }); + + it("filters by review status", () => { + insertMemoryEntry(db, { id: "mem_filter_1", review_status: "draft" }); + insertMemoryEntry(db, { id: "mem_filter_2", review_status: "approved" }); + insertMemoryEntry(db, { id: "mem_filter_3", review_status: "draft" }); + + const drafts = db.prepare("SELECT id FROM memory_entries WHERE review_status = ?").all("draft") as { id: string }[]; + expect(drafts).toHaveLength(2); + expect(drafts.map((r) => r.id)).toContain("mem_filter_1"); + expect(drafts.map((r) => r.id)).toContain("mem_filter_3"); + }); + + it("filters by retention class", () => { + insertMemoryEntry(db, { id: "mem_ret_1", retention_class: "ephemeral" }); + insertMemoryEntry(db, { id: "mem_ret_2", retention_class: "pinned" }); + insertMemoryEntry(db, { id: "mem_ret_3", retention_class: "ephemeral" }); + + const ephemerals = db.prepare("SELECT id FROM memory_entries WHERE retention_class = ?").all("ephemeral") as { id: string }[]; + expect(ephemerals).toHaveLength(2); + }); + + it("respects LIMIT", () => { + for (let i = 0; i < 10; i++) { + insertMemoryEntry(db, { id: `mem_limit_${i}` }); + } + + const limited = db.prepare("SELECT id FROM memory_entries ORDER BY updated_at DESC LIMIT ?").all(3); + expect(limited).toHaveLength(3); + }); +}); + +describe("memory_entry_origins", () => { + let db: Database.Database; + + beforeEach(() => { + db = setup(); + }); + + afterEach(() => { + db.close(); + if (existsSync(TEST_DB_PATH)) rmSync(TEST_DB_PATH); + }); + + it("links multiple origins to a single entry", () => { + const entry = insertMemoryEntry(db, { id: "mem_origins_1" }); + + db.prepare("INSERT INTO memory_entry_origins (memory_entry_id, origin_type, origin_id, relationship) VALUES (?, ?, ?, ?)") + .run(entry.id, "trace", "trace:abc", "derived_from"); + db.prepare("INSERT INTO memory_entry_origins (memory_entry_id, origin_type, origin_id, relationship) VALUES (?, ?, ?, ?)") + .run(entry.id, "event", "event:xyz", "derived_from"); + + const origins = db.prepare("SELECT * FROM memory_entry_origins WHERE memory_entry_id = ? ORDER BY origin_type").all(entry.id) as Record[]; + expect(origins).toHaveLength(2); + expect(origins[0].origin_type).toBe("event"); + expect(origins[1].origin_type).toBe("trace"); + }); +}); + +describe("memory_entry_tags", () => { + let db: Database.Database; + + beforeEach(() => { + db = setup(); + }); + + afterEach(() => { + db.close(); + if (existsSync(TEST_DB_PATH)) rmSync(TEST_DB_PATH); + }); + + it("attaches tags to entries", () => { + const entry = insertMemoryEntry(db, { id: "mem_tags_1" }); + + db.prepare("INSERT INTO memory_entry_tags (memory_entry_id, tag) VALUES (?, ?)").run(entry.id, "important"); + db.prepare("INSERT INTO memory_entry_tags (memory_entry_id, tag) VALUES (?, ?)").run(entry.id, "security"); + + const tags = db.prepare("SELECT tag FROM memory_entry_tags WHERE memory_entry_id = ? ORDER BY tag").all(entry.id) as { tag: string }[]; + expect(tags.map((t) => t.tag)).toEqual(["important", "security"]); + }); + + it("enforces unique tag per entry", () => { + const entry = insertMemoryEntry(db, { id: "mem_tag_dup_1" }); + + db.prepare("INSERT INTO memory_entry_tags (memory_entry_id, tag) VALUES (?, ?)").run(entry.id, "dupe"); + expect(() => + db.prepare("INSERT INTO memory_entry_tags (memory_entry_id, tag) VALUES (?, ?)").run(entry.id, "dupe"), + ).toThrow(); + }); +}); + +describe("memory_entry_versions", () => { + let db: Database.Database; + + beforeEach(() => { + db = setup(); + }); + + afterEach(() => { + db.close(); + if (existsSync(TEST_DB_PATH)) rmSync(TEST_DB_PATH); + }); + + it("tracks version history for an entry", () => { + const entryId = "mem_versioned_1"; + insertMemoryEntry(db, { id: entryId, title: "v1 title", body: "v1 body" }); + + // Snapshot version 1 + db.prepare(` + INSERT INTO memory_entry_versions (id, version, title, body, summary, retention_class, review_status, changed_by_type, changed_by_id, changed_at, change_reason) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run(entryId, 1, "v1 title", "v1 body", null, "working", "draft", "user", null, new Date().toISOString(), "Initial"); + + // Snapshot version 2 + db.prepare(` + INSERT INTO memory_entry_versions (id, version, title, body, summary, retention_class, review_status, changed_by_type, changed_by_id, changed_at, change_reason) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run(entryId, 2, "v2 title", "v2 body", null, "working", "reviewed", "user", null, new Date().toISOString(), "Updated content"); + + const versions = db.prepare("SELECT * FROM memory_entry_versions WHERE id = ? ORDER BY version").all(entryId) as Record[]; + expect(versions).toHaveLength(2); + expect(versions[0].version).toBe(1); + expect(versions[1].version).toBe(2); + expect(versions[1].title).toBe("v2 title"); + }); +}); diff --git a/__tests__/search.test.ts b/__tests__/search.test.ts new file mode 100644 index 0000000..da5575c --- /dev/null +++ b/__tests__/search.test.ts @@ -0,0 +1,129 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import Database from "better-sqlite3"; +import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs"; +import path from "node:path"; + +/** + * Tests for FTS5 search and query handling. + * Uses standalone SQLite to avoid polluting the development database. + */ + +const TEST_DIR = path.join(process.cwd(), "storage", "__test__"); +const TEST_DB_PATH = path.join(TEST_DIR, "search-test.sqlite"); +const MIGRATION_PATH = path.join(process.cwd(), "db", "0001_init.sql"); + +function fts5Escape(raw: string): string { + const tokens = raw.match(/\S+/g); + if (!tokens) return '""'; + return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" "); +} + +function setup() { + if (!existsSync(TEST_DIR)) mkdirSync(TEST_DIR, { recursive: true }); + if (existsSync(TEST_DB_PATH)) rmSync(TEST_DB_PATH); + + const db = new Database(TEST_DB_PATH); + db.exec("PRAGMA journal_mode=WAL;"); + db.exec("PRAGMA foreign_keys=ON;"); + db.exec(readFileSync(MIGRATION_PATH, "utf8")); + return db; +} + +function insertSearchChunk(db: Database.Database, sourceKind: string, sourceId: string, title: string, body: string) { + db.prepare("INSERT INTO search_chunks (source_kind, source_id, title, body) VALUES (?, ?, ?, ?)") + .run(sourceKind, sourceId, title, body); +} + +describe("FTS5 search_chunks", () => { + let db: Database.Database; + + beforeEach(() => { + db = setup(); + }); + + afterEach(() => { + db.close(); + if (existsSync(TEST_DB_PATH)) rmSync(TEST_DB_PATH); + }); + + it("indexes and retrieves by keyword match", () => { + insertSearchChunk(db, "trace", "trace:1", "Deploy script", "Deployed the production environment successfully"); + insertSearchChunk(db, "trace", "trace:2", "Bug fix", "Fixed a null pointer exception in the auth module"); + + const results = db.prepare( + "SELECT source_id, title FROM search_chunks WHERE search_chunks MATCH ? LIMIT 10", + ).all(fts5Escape("deploy")) as { source_id: string; title: string }[]; + + expect(results).toHaveLength(1); + expect(results[0].source_id).toBe("trace:1"); + }); + + it("returns empty for no-match queries", () => { + insertSearchChunk(db, "trace", "trace:1", "Test", "Some content here"); + + const results = db.prepare( + "SELECT source_id FROM search_chunks WHERE search_chunks MATCH ? LIMIT 10", + ).all(fts5Escape("zzzznonexistent")); + + expect(results).toHaveLength(0); + }); + + it("matches across title and body", () => { + insertSearchChunk(db, "trace", "trace:1", "Important deployment", "Routine maintenance task"); + + const titleMatch = db.prepare( + "SELECT source_id FROM search_chunks WHERE search_chunks MATCH ? LIMIT 10", + ).all(fts5Escape("deployment")); + + const bodyMatch = db.prepare( + "SELECT source_id FROM search_chunks WHERE search_chunks MATCH ? LIMIT 10", + ).all(fts5Escape("maintenance")); + + expect(titleMatch).toHaveLength(1); + expect(bodyMatch).toHaveLength(1); + }); + + it("supports multi-word queries", () => { + insertSearchChunk(db, "trace", "trace:1", "Auth module", "Updated the login rate limiting logic"); + insertSearchChunk(db, "trace", "trace:2", "UI fix", "Fixed the login button styling"); + + const results = db.prepare( + "SELECT source_id FROM search_chunks WHERE search_chunks MATCH ? LIMIT 10", + ).all(fts5Escape("login rate")) as { source_id: string }[]; + + // Should match trace:1 which has both "login" and "rate" + expect(results.length).toBeGreaterThanOrEqual(1); + expect(results.some((r) => r.source_id === "trace:1")).toBe(true); + }); + + it("respects LIMIT", () => { + for (let i = 0; i < 20; i++) { + insertSearchChunk(db, "trace", `trace:${i}`, `Session ${i}`, `Common keyword appears in session ${i}`); + } + + const results = db.prepare( + "SELECT source_id FROM search_chunks WHERE search_chunks MATCH ? LIMIT 5", + ).all(fts5Escape("common")); + + expect(results).toHaveLength(5); + }); +}); + +describe("fts5Escape", () => { + it("wraps tokens in quotes", () => { + expect(fts5Escape("hello world")).toBe('"hello" "world"'); + }); + + it("escapes quotes within tokens", () => { + expect(fts5Escape('say "hi"')).toBe('"say" """hi"""'); + }); + + it("returns empty-string literal for blank input", () => { + expect(fts5Escape("")).toBe('""'); + expect(fts5Escape(" ")).toBe('""'); + }); + + it("handles single token", () => { + expect(fts5Escape("deploy")).toBe('"deploy"'); + }); +}); diff --git a/__tests__/sql-runner.test.ts b/__tests__/sql-runner.test.ts new file mode 100644 index 0000000..d68143d --- /dev/null +++ b/__tests__/sql-runner.test.ts @@ -0,0 +1,62 @@ +import { describe, it, expect } from "vitest"; + +/** + * Tests for the read-only SQL validation logic. + * We test the validation rules directly without needing a real database. + */ + +const READ_ONLY_PREFIXES = ["select", "with", "pragma table_info", "pragma database_list"]; + +function isReadOnly(sql: string): boolean { + const normalized = sql.trim().toLowerCase(); + return READ_ONLY_PREFIXES.some((prefix) => normalized.startsWith(prefix)); +} + +describe("SQL read-only validation", () => { + it("allows SELECT statements", () => { + expect(isReadOnly("SELECT * FROM sessions")).toBe(true); + expect(isReadOnly(" SELECT id FROM traces LIMIT 5")).toBe(true); + expect(isReadOnly("select count(*) from events")).toBe(true); + }); + + it("allows WITH (CTE) statements", () => { + expect(isReadOnly("WITH cte AS (SELECT 1) SELECT * FROM cte")).toBe(true); + }); + + it("allows PRAGMA table_info", () => { + expect(isReadOnly("PRAGMA table_info(sessions)")).toBe(true); + }); + + it("allows PRAGMA database_list", () => { + expect(isReadOnly("PRAGMA database_list")).toBe(true); + }); + + it("rejects INSERT statements", () => { + expect(isReadOnly("INSERT INTO sessions (id) VALUES ('test')")).toBe(false); + }); + + it("rejects UPDATE statements", () => { + expect(isReadOnly("UPDATE sessions SET status = 'done'")).toBe(false); + }); + + it("rejects DELETE statements", () => { + expect(isReadOnly("DELETE FROM sessions WHERE id = 'test'")).toBe(false); + }); + + it("rejects DROP statements", () => { + expect(isReadOnly("DROP TABLE sessions")).toBe(false); + }); + + it("rejects ALTER statements", () => { + expect(isReadOnly("ALTER TABLE sessions ADD COLUMN foo TEXT")).toBe(false); + }); + + it("rejects CREATE statements", () => { + expect(isReadOnly("CREATE TABLE evil (id TEXT)")).toBe(false); + }); + + it("rejects empty input", () => { + expect(isReadOnly("")).toBe(false); + expect(isReadOnly(" ")).toBe(false); + }); +}); diff --git a/app/(app)/artifacts/error.tsx b/app/(app)/artifacts/error.tsx new file mode 100644 index 0000000..58f3ed9 --- /dev/null +++ b/app/(app)/artifacts/error.tsx @@ -0,0 +1,40 @@ +"use client"; + +import { useEffect } from "react"; + +export default function ArtifactsError({ + error, + reset, +}: { + error: Error & { digest?: string }; + reset: () => void; +}) { + useEffect(() => { + console.error("App error boundary caught:", error); + }, [error]); + + return ( +
+
+

Something went wrong

+

+ This section encountered an error. Your data is safe. +

+
+ + + Back to dashboard + +
+
+
+ ); +} diff --git a/app/(app)/dashboard/error.tsx b/app/(app)/dashboard/error.tsx new file mode 100644 index 0000000..c6a8d8a --- /dev/null +++ b/app/(app)/dashboard/error.tsx @@ -0,0 +1,40 @@ +"use client"; + +import { useEffect } from "react"; + +export default function DashboardError({ + error, + reset, +}: { + error: Error & { digest?: string }; + reset: () => void; +}) { + useEffect(() => { + console.error("App error boundary caught:", error); + }, [error]); + + return ( +
+
+

Something went wrong

+

+ This section encountered an error. Your data is safe. +

+
+ + + Back to dashboard + +
+
+
+ ); +} diff --git a/app/(app)/error.tsx b/app/(app)/error.tsx new file mode 100644 index 0000000..860af52 --- /dev/null +++ b/app/(app)/error.tsx @@ -0,0 +1,40 @@ +"use client"; + +import { useEffect } from "react"; + +export default function AppError({ + error, + reset, +}: { + error: Error & { digest?: string }; + reset: () => void; +}) { + useEffect(() => { + console.error("App error boundary caught:", error); + }, [error]); + + return ( +
+
+

Something went wrong

+

+ This section encountered an error. Your data is safe. +

+
+ + + Back to dashboard + +
+
+
+ ); +} diff --git a/app/(app)/investigations/error.tsx b/app/(app)/investigations/error.tsx new file mode 100644 index 0000000..3ebdb79 --- /dev/null +++ b/app/(app)/investigations/error.tsx @@ -0,0 +1,40 @@ +"use client"; + +import { useEffect } from "react"; + +export default function InvestigationsError({ + error, + reset, +}: { + error: Error & { digest?: string }; + reset: () => void; +}) { + useEffect(() => { + console.error("App error boundary caught:", error); + }, [error]); + + return ( +
+
+

Something went wrong

+

+ This section encountered an error. Your data is safe. +

+
+ + + Back to dashboard + +
+
+
+ ); +} diff --git a/app/(app)/memory/error.tsx b/app/(app)/memory/error.tsx new file mode 100644 index 0000000..f062c40 --- /dev/null +++ b/app/(app)/memory/error.tsx @@ -0,0 +1,40 @@ +"use client"; + +import { useEffect } from "react"; + +export default function MemoryError({ + error, + reset, +}: { + error: Error & { digest?: string }; + reset: () => void; +}) { + useEffect(() => { + console.error("App error boundary caught:", error); + }, [error]); + + return ( +
+
+

Something went wrong

+

+ This section encountered an error. Your data is safe. +

+
+ + + Back to dashboard + +
+
+
+ ); +} diff --git a/app/(app)/traces/error.tsx b/app/(app)/traces/error.tsx new file mode 100644 index 0000000..a7ef39e --- /dev/null +++ b/app/(app)/traces/error.tsx @@ -0,0 +1,40 @@ +"use client"; + +import { useEffect } from "react"; + +export default function TracesError({ + error, + reset, +}: { + error: Error & { digest?: string }; + reset: () => void; +}) { + useEffect(() => { + console.error("App error boundary caught:", error); + }, [error]); + + return ( +
+
+

Something went wrong

+

+ This section encountered an error. Your data is safe. +

+
+ + + Back to dashboard + +
+
+
+ ); +} diff --git a/app/(app)/workflows/error.tsx b/app/(app)/workflows/error.tsx new file mode 100644 index 0000000..e1c43f2 --- /dev/null +++ b/app/(app)/workflows/error.tsx @@ -0,0 +1,40 @@ +"use client"; + +import { useEffect } from "react"; + +export default function WorkflowsError({ + error, + reset, +}: { + error: Error & { digest?: string }; + reset: () => void; +}) { + useEffect(() => { + console.error("App error boundary caught:", error); + }, [error]); + + return ( +
+
+

Something went wrong

+

+ This section encountered an error. Your data is safe. +

+
+ + + Back to dashboard + +
+
+
+ ); +} diff --git a/app/api/auth/login/route.ts b/app/api/auth/login/route.ts index 18aa183..1dcae74 100644 --- a/app/api/auth/login/route.ts +++ b/app/api/auth/login/route.ts @@ -1,4 +1,5 @@ import { NextResponse } from "next/server"; +import { ApiValidationError, fail, ok } from "@/lib/opentrust/api-contract"; import { writeAuthAudit } from "@/lib/opentrust/auth-audit"; import { clearLoginFailures, getLoginRateLimit, recordLoginFailure } from "@/lib/opentrust/auth-rate-limit"; import { verifySameOriginRequest } from "@/lib/opentrust/csrf"; @@ -26,11 +27,11 @@ export async function POST(request: Request) { userAgent: meta.userAgent, detail: `csrf_${csrf.reason}`, }); - return NextResponse.json({ ok: false, error: "Invalid request origin" }, { status: 403 }); + return NextResponse.json(fail(new ApiValidationError("Invalid request origin", { status: 403, code: "csrf_failed" })), { status: 403 }); } if (config.mode === "none") { - return NextResponse.json({ ok: true, mode: config.mode, bypass: true }); + return NextResponse.json(ok({ mode: config.mode, bypass: true })); } if (config.allowLocalhostBypass && isLoopbackHost(await getRequestHostname())) { @@ -38,7 +39,7 @@ export async function POST(request: Request) { if (sessionValue) { clearLoginFailures(meta.ip); writeAuthAudit({ action: "login_success", ip: meta.ip, userAgent: meta.userAgent, detail: "localhost_bypass" }); - const response = NextResponse.json({ ok: true, mode: config.mode, bypass: true }); + const response = NextResponse.json(ok({ mode: config.mode, bypass: true })); response.cookies.set(OPENTRUST_AUTH_COOKIE, sessionValue, { httpOnly: true, sameSite: "strict", @@ -58,13 +59,11 @@ export async function POST(request: Request) { userAgent: meta.userAgent, detail: "rate_limited", }); - return NextResponse.json( - { ok: false, error: "Too many failed attempts. Try again later." }, - { - status: 429, - headers: { "retry-after": String(Math.ceil((rate.resetAt - Date.now()) / 1000)) }, - } - ); + const result = fail(new ApiValidationError("Too many failed attempts. Try again later.", { status: 429, code: "rate_limited" })); + return NextResponse.json(result, { + status: 429, + headers: { "retry-after": String(Math.ceil((rate.resetAt - Date.now()) / 1000)) }, + }); } let body: unknown; @@ -78,7 +77,7 @@ export async function POST(request: Request) { userAgent: meta.userAgent, detail: "invalid_json", }); - return NextResponse.json({ ok: false, error: "Malformed JSON body" }, { status: 400 }); + return NextResponse.json(fail(new ApiValidationError("Malformed JSON body", { status: 400, code: "invalid_json" })), { status: 400 }); } const credential = typeof (body as { credential?: unknown })?.credential === "string" @@ -94,7 +93,7 @@ export async function POST(request: Request) { userAgent: meta.userAgent, detail: failure.limited ? "invalid_credentials_limit_reached" : "invalid_credentials", }); - return NextResponse.json({ ok: false, error: "Invalid credentials" }, { status: 401 }); + return NextResponse.json(fail(new ApiValidationError("Invalid credentials", { status: 401, code: "invalid_credentials" })), { status: 401 }); } const sessionValue = createSessionValue(config); @@ -105,7 +104,7 @@ export async function POST(request: Request) { userAgent: meta.userAgent, detail: "misconfigured_auth", }); - return NextResponse.json({ ok: false, error: "Auth is configured incorrectly" }, { status: 500 }); + return NextResponse.json(fail(new ApiValidationError("Auth is configured incorrectly", { status: 500, code: "misconfigured_auth" })), { status: 500 }); } clearLoginFailures(meta.ip); @@ -118,7 +117,7 @@ export async function POST(request: Request) { const maxAge = rememberMe ? 60 * 60 * 24 * 30 : 60 * 60 * 12; - const response = NextResponse.json({ ok: true, mode: config.mode }); + const response = NextResponse.json(ok({ mode: config.mode })); response.cookies.set(OPENTRUST_AUTH_COOKIE, sessionValue, { httpOnly: true, sameSite: "strict", diff --git a/app/api/auth/logout/route.ts b/app/api/auth/logout/route.ts index de80ff5..1f70f50 100644 --- a/app/api/auth/logout/route.ts +++ b/app/api/auth/logout/route.ts @@ -1,4 +1,5 @@ import { NextResponse } from "next/server"; +import { ApiValidationError, fail, ok } from "@/lib/opentrust/api-contract"; import { writeAuthAudit } from "@/lib/opentrust/auth-audit"; import { verifySameOriginRequest } from "@/lib/opentrust/csrf"; import { getRequestMeta, OPENTRUST_AUTH_COOKIE } from "@/lib/opentrust/auth"; @@ -15,7 +16,7 @@ export async function POST() { userAgent: meta.userAgent, detail: `csrf_${csrf.reason}`, }); - return NextResponse.json({ ok: false, error: "Invalid request origin" }, { status: 403 }); + return NextResponse.json(fail(new ApiValidationError("Invalid request origin", { status: 403, code: "csrf_failed" })), { status: 403 }); } writeAuthAudit({ @@ -25,7 +26,7 @@ export async function POST() { detail: null, }); - const response = NextResponse.json({ ok: true }); + const response = NextResponse.json(ok(null)); response.cookies.set(OPENTRUST_AUTH_COOKIE, "", { httpOnly: true, sameSite: "strict", diff --git a/app/error.tsx b/app/error.tsx new file mode 100644 index 0000000..6a445a3 --- /dev/null +++ b/app/error.tsx @@ -0,0 +1,38 @@ +"use client"; + +import { useEffect } from "react"; + +export default function RootError({ + error, + reset, +}: { + error: Error & { digest?: string }; + reset: () => void; +}) { + useEffect(() => { + console.error("Root error boundary caught:", error); + }, [error]); + + return ( +
+

Something went wrong

+

+ An unexpected error occurred. You can try again or return to the dashboard. +

+
+ + + Go home + +
+
+ ); +} diff --git a/app/not-found.tsx b/app/not-found.tsx new file mode 100644 index 0000000..9983cfe --- /dev/null +++ b/app/not-found.tsx @@ -0,0 +1,16 @@ +import Link from "next/link"; + +export default function NotFound() { + return ( +
+

404

+

The page you're looking for doesn't exist.

+ + Go home + +
+ ); +} diff --git a/docs/IMPROVEMENT-RECOMMENDATIONS.md b/docs/IMPROVEMENT-RECOMMENDATIONS.md new file mode 100644 index 0000000..4b0de9c --- /dev/null +++ b/docs/IMPROVEMENT-RECOMMENDATIONS.md @@ -0,0 +1,205 @@ +# OpenTrust — Improvement Checklist + +Codebase audit conducted 2026-03-27. Covers architecture, code quality, +security, testing, and operational readiness. + +--- + +## Strengths to Preserve + +Before improving, note what's already working well: + +- [x] Timing-safe auth, CSRF protection, rate limiting, audit logging +- [x] Strict TypeScript with minimal `any` usage +- [x] Idempotent upserts, foreign keys, cascading deletes, FTS5 indexes +- [x] 24 documentation files covering architecture through plugin packaging +- [x] Clean module boundaries (DB, ingestion, search, memory, auth, UI) +- [x] Operational scripts (setup, update, harden, review, doctor, reset) +- [x] Local-first SQLite with WAL mode, no external service dependencies + +--- + +## Phase 1 — Data Integrity & Crash Safety + +_Goal: prevent data corruption and UI crashes. Small effort, high impact._ + +### Database Transactions + +- [x] Wrap `import-openclaw.ts` ingestion loop in a `better-sqlite3` transaction +- [x] Wrap `import-cron.ts` workflow ingestion in a transaction +- [x] Wrap memory entry creation (entry + origins + tags) in a transaction +- [x] Wrap artifact extraction writes in a transaction +- [x] Audit all multi-statement write paths for missing transaction boundaries + +### React Error Boundaries + +- [x] Add root-level error boundary in the app layout +- [x] Add `error.tsx` for `(app)/dashboard` +- [x] Add `error.tsx` for `(app)/memory` +- [x] Add `error.tsx` for `(app)/traces` +- [x] Add `error.tsx` for `(app)/investigations` +- [x] Add `error.tsx` for `(app)/workflows` +- [x] Add `error.tsx` for `(app)/artifacts` +- [x] Add global `not-found.tsx` for 404 handling +- [ ] Add graceful fallbacks for API fetch failures in client components + +--- + +## Phase 2 — API Consistency & CI + +_Goal: standardize the API surface and add an automated quality gate._ + +### Standardize API Error Responses + +- [x] Define shared `ApiError` type with `code`, `message`, optional `retryAfter`/`details` +- [x] Create `apiError(code, message, status)` helper function +- [x] Migrate `/api/auth/login` to use shared error helper +- [x] Migrate `/api/auth/logout` to use shared error helper +- [x] Migrate `/api/memory/*` routes to use shared error helper +- [ ] Document error codes in `docs/MEMORY-API-CONTRACT.md` + +### CI/CD Pipeline + +- [x] Create `.github/workflows/ci.yml` with checkout, pnpm install, lint, test, secretlint +- [ ] Verify pipeline passes on current `main` branch +- [ ] Enable branch protection requiring CI to pass before merge + +--- + +## Phase 3 — Test Coverage (Core) + +_Goal: build a safety net for the most critical code paths._ + +### Database & Ingestion Tests + +- [x] Add test for `db.ts` initialization (WAL mode, foreign keys, tables created) +- [ ] Add test for `import-openclaw.ts` — valid session ingestion +- [ ] Add test for `import-openclaw.ts` — malformed JSONL handling +- [ ] Add test for `import-openclaw.ts` — duplicate/idempotent ingestion +- [ ] Add test for `import-openclaw.ts` — partial session recovery +- [ ] Add test for `import-cron.ts` — workflow run ingestion +- [ ] Add test for `import-cron.ts` — duplicate run handling +- [x] Add test for `ingestion-state.ts` — cursor tracking + +### Memory & Search Tests + +- [x] Add test for `memory-entries.ts` — create, read, update, delete +- [x] Add test for `memory-entries.ts` — filtering by tags, retention class +- [x] Add test for `search.ts` — FTS5 search with valid queries +- [x] Add test for `search.ts` — empty/whitespace query handling +- [ ] Add test for `search.ts` — memory → FTS → semantic fallback chain +- [ ] Add test for `semantic.ts` — vector indexing and retrieval + +### Auth Tests + +- [ ] Add test for `auth.ts` — session creation and verification +- [ ] Add test for `auth.ts` — timing-safe credential comparison +- [x] Add test for `auth.ts` — rate limiting behavior +- [ ] Add test for login API route — successful login +- [ ] Add test for login API route — invalid credentials +- [ ] Add test for login API route — rate limit enforcement +- [ ] Add test for logout API route + +### Additional Tests Added + +- [x] Add test for `sql-runner.ts` — read-only SQL validation +- [x] Add test for `api-contract.ts` — ok/fail envelope, ApiValidationError +- [x] Add test for `artifact-extract.ts` — URL, repo, doc extraction + +--- + +## Phase 4 — Runtime Safety & Observability + +_Goal: catch schema drift at runtime and make the system diagnosable._ + +### Runtime Validation of Database Results + +- [ ] Define Zod schemas for core DB row types (sessions, traces, events) +- [ ] Define Zod schemas for memory entry rows +- [ ] Define Zod schemas for search result rows +- [ ] Add Zod `.parse()` validation to ingestion query results +- [ ] Add Zod `.parse()` validation to search query results +- [ ] Add Zod `.parse()` validation to memory CRUD query results +- [ ] Log validation failures with context (query, row shape) before throwing + +### Structured Logging + +- [x] Create `lib/opentrust/logger.ts` with levels: debug, info, warn, error +- [x] Add context fields: module name, operation, duration, session/trace ID +- [x] Output structured JSON for machine parsing +- [x] Replace `console.log` calls in `db.ts` +- [x] Replace `console.log` calls in `import-openclaw.ts` +- [x] Replace `console.log` calls in `import-cron.ts` +- [x] Replace `console.log` calls in `bootstrap.ts` +- [x] Replace `console.log` calls in `auth.ts` +- [x] Replace `console.log`/`console.error` calls in remaining modules +- [ ] Add request-level correlation IDs in API routes + +--- + +## Phase 5 — Performance & Scalability + +_Goal: ensure the system performs well as data volume grows._ + +### Batch SQL Operations + +- [ ] Refactor ingestion loops to use multi-row `INSERT` where record count is known +- [ ] Benchmark session import before/after on a representative large session +- [ ] Profile SQLite query times on datasets with 1k+ sessions + +### Search Scalability + +- [x] Move memory entry filtering from in-memory `.includes()` to SQL LIKE +- [x] Reserve in-memory filtering only for fuzzy/semantic matching +- [ ] Add FTS5 index for memory entries if not already present +- [ ] Add pagination to search results (currently hardcoded `LIMIT 12`) +- [ ] Test search performance with 500+ memory entries + +--- + +## Phase 6 — Test Coverage (Full) & DX + +_Goal: complete test coverage and improve developer experience._ + +### Component & API Tests + +- [ ] Set up Vitest + React Testing Library for component tests +- [ ] Add tests for memory management page +- [ ] Add tests for investigations page +- [ ] Add tests for traces detail page +- [ ] Add tests for dashboard page +- [ ] Achieve test file parity: every `lib/opentrust/` module has a test file + +### JSDoc Documentation + +- [x] Add JSDoc to all exported functions in `db.ts` +- [x] Add JSDoc to all exported functions in `search.ts` +- [x] Add JSDoc to all exported functions in `memory-entries.ts` +- [x] Add JSDoc to all exported functions in `auth.ts` +- [x] Add JSDoc to all exported functions in `import-openclaw.ts` +- [x] Add JSDoc to all exported functions in `import-cron.ts` +- [x] Add JSDoc to all exported functions in remaining `lib/opentrust/` modules + +--- + +## Phase 7 — Polish + +_Goal: refine the edges. Low urgency, ongoing._ + +### UI Polish + +- [ ] Add skeleton screens for traces page during data loading +- [ ] Add skeleton screens for workflows page during data loading +- [ ] Add skeleton screens for memory page during data loading +- [ ] Wire `EmptyState` component into all list views with zero results + +### Type Safety Cleanup + +- [x] Audit `as never` casts — all 3 are in `db.ts` bridging `Record` to `better-sqlite3`'s `BindingOrRecord`. Intentionally kept to avoid leaking the driver type into every caller. +- [ ] Audit for any remaining `as` casts that could be eliminated + +### In-Memory Caching + +- [ ] Add TTL cache for capabilities list (frequently read, rarely changes) +- [ ] Add TTL cache for ingestion state cursors +- [ ] Use simple `Map` with expiry timestamps — no external dependencies diff --git a/lib/opentrust/api-contract.ts b/lib/opentrust/api-contract.ts index 029576d..314219a 100644 --- a/lib/opentrust/api-contract.ts +++ b/lib/opentrust/api-contract.ts @@ -8,6 +8,9 @@ import type { MemorySearchRequest, MemorySourceType, } from "@/lib/types"; +import { createLogger } from "@/lib/opentrust/logger"; + +const log = createLogger("api-contract"); const MEMORY_SOURCE_TYPES: MemorySourceType[] = [ "trace", @@ -39,10 +42,12 @@ export class ApiValidationError extends Error { } } +/** Wrap a successful result in the standard API envelope. */ export function ok(data: T) { return { ok: true as const, data }; } +/** Wrap an error in the standard API envelope, mapping known errors to their status codes. */ export function fail(error: unknown) { if (error instanceof ApiValidationError) { return { @@ -56,7 +61,7 @@ export function fail(error: unknown) { }; } - console.error("Unhandled API error", error); + log.error("Unhandled API error", { error: error instanceof Error ? error.message : String(error) }); return { ok: false as const, @@ -69,6 +74,7 @@ export function fail(error: unknown) { }; } +/** Parse a JSON request body, throwing an ApiValidationError on bad content-type or malformed JSON. */ export async function readJson(request: Request) { const contentType = request.headers.get("content-type") ?? ""; if (!contentType.toLowerCase().includes("application/json")) { @@ -124,6 +130,7 @@ function asEnum( return value as T; } +/** Validate and parse a memory search request from a JSON body. */ export function parseSearchRequest(input: unknown): MemorySearchRequest { const body = (input ?? {}) as Record; const query = asString(body.query, "query", { required: true })!; @@ -152,6 +159,7 @@ export function parseSearchRequest(input: unknown): MemorySearchRequest { }; } +/** Validate and parse a memory inspect request from a JSON body. */ export function parseInspectRequest(input: unknown): MemoryInspectRequest { const body = (input ?? {}) as Record; const ref = (body.ref ?? {}) as Record; @@ -167,6 +175,7 @@ export function parseInspectRequest(input: unknown): MemoryInspectRequest { }; } +/** Validate and parse a memory health request from a JSON body. */ export function parseHealthRequest(input: unknown): MemoryHealthRequest { const body = (input ?? {}) as Record; return { @@ -175,6 +184,7 @@ export function parseHealthRequest(input: unknown): MemoryHealthRequest { }; } +/** Validate and parse a memory promote request from a JSON body. */ export function parsePromoteRequest(input: unknown): MemoryPromoteRequest { const body = (input ?? {}) as Record; const originRefs = body.originRefs; @@ -223,6 +233,7 @@ export function parsePromoteRequest(input: unknown): MemoryPromoteRequest { }; } +/** Parse a memory search request from URL query parameters. */ export function parseSearchQuery(url: URL): MemorySearchRequest { return parseSearchRequest({ query: url.searchParams.get("q"), @@ -233,6 +244,7 @@ export function parseSearchQuery(url: URL): MemorySearchRequest { }); } +/** Parse a memory inspect request from URL query parameters. */ export function parseInspectQuery(url: URL): MemoryInspectRequest { return parseInspectRequest({ ref: { @@ -245,6 +257,7 @@ export function parseInspectQuery(url: URL): MemoryInspectRequest { }); } +/** Parse a memory health request from URL query parameters. */ export function parseHealthQuery(url: URL): MemoryHealthRequest { return parseHealthRequest({ scope: url.searchParams.get("scope") ?? "global", @@ -252,6 +265,7 @@ export function parseHealthQuery(url: URL): MemoryHealthRequest { }); } +/** Filter search results to only include entries matching a specific review status. */ export function filterSearchResultsByReview( results: T[], review?: string, diff --git a/lib/opentrust/artifact-extract.ts b/lib/opentrust/artifact-extract.ts index 1bb04f2..20deda2 100644 --- a/lib/opentrust/artifact-extract.ts +++ b/lib/opentrust/artifact-extract.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { escapeSqlString, runSql } from "@/lib/opentrust/db"; +import { escapeSqlString, runSql, withTransaction } from "@/lib/opentrust/db"; export interface ExtractedArtifact { kind: "url" | "doc" | "repo" | "note"; @@ -15,6 +15,7 @@ function normalize(value: string) { return value.trim(); } +/** Extract URLs, repo references, and doc paths from free-form text. */ export function extractArtifactsFromText(text: string): ExtractedArtifact[] { const artifacts = new Map(); @@ -40,8 +41,10 @@ export function extractArtifactsFromText(text: string): ExtractedArtifact[] { return [...artifacts.values()].slice(0, 24); } +/** Extract artifacts from text and link them to a trace via trace_edges. */ export function upsertArtifactsForTrace(traceId: string, text: string, createdAt: string) { const artifacts = extractArtifactsFromText(text); + return withTransaction(() => { runSql(`DELETE FROM trace_edges WHERE from_kind = 'trace' AND from_id = ${escapeSqlString(traceId)} AND edge_type = 'references' AND to_kind = 'artifact';`); for (const artifact of artifacts) { @@ -75,10 +78,13 @@ export function upsertArtifactsForTrace(traceId: string, text: string, createdAt } return artifacts.length; + }); } +/** Extract artifacts from text and link them to a workflow via run_artifacts. */ export function upsertArtifactsForWorkflow(workflowId: string, text: string, createdAt: string) { const artifacts = extractArtifactsFromText(text); + return withTransaction(() => { runSql(`DELETE FROM run_artifacts WHERE run_id = ${escapeSqlString(workflowId)} AND relation = 'references';`); for (const artifact of artifacts) { @@ -107,4 +113,5 @@ export function upsertArtifactsForWorkflow(workflowId: string, text: string, cre } return artifacts.length; + }); } diff --git a/lib/opentrust/auth-audit.ts b/lib/opentrust/auth-audit.ts index f7ceaa0..a1be648 100644 --- a/lib/opentrust/auth-audit.ts +++ b/lib/opentrust/auth-audit.ts @@ -10,6 +10,7 @@ function auditPath() { return path.join(dir, "auth.log"); } +/** Append a JSON-line auth event to the audit log file (best-effort, swallows FS errors). */ export function writeAuthAudit(event: { action: "login_success" | "login_failure" | "logout"; ip?: string | null; diff --git a/lib/opentrust/auth-rate-limit.ts b/lib/opentrust/auth-rate-limit.ts index 0138276..f042829 100644 --- a/lib/opentrust/auth-rate-limit.ts +++ b/lib/opentrust/auth-rate-limit.ts @@ -6,10 +6,12 @@ function now() { return Date.now(); } +/** Normalize an IP address into a rate-limit key, defaulting to "unknown". */ export function getRateLimitKey(ip: string | null | undefined) { return ip?.trim() || "unknown"; } +/** Return the current rate-limit state for an IP, creating a fresh window if expired. */ export function getLoginRateLimit(ip: string | null | undefined) { const key = getRateLimitKey(ip); const current = entries.get(key); @@ -28,6 +30,7 @@ export function getLoginRateLimit(ip: string | null | undefined) { }; } +/** Increment the failure count for an IP and return whether the limit has been reached. */ export function recordLoginFailure(ip: string | null | undefined) { const state = getLoginRateLimit(ip); const next = { count: state.count + 1, resetAt: state.resetAt }; @@ -39,6 +42,7 @@ export function recordLoginFailure(ip: string | null | undefined) { }; } +/** Reset the failure counter for an IP after a successful login. */ export function clearLoginFailures(ip: string | null | undefined) { entries.delete(getRateLimitKey(ip)); } diff --git a/lib/opentrust/auth.ts b/lib/opentrust/auth.ts index bfb1e79..28d5032 100644 --- a/lib/opentrust/auth.ts +++ b/lib/opentrust/auth.ts @@ -24,6 +24,7 @@ function safeEqual(a: string, b: string) { return timingSafeEqual(left, right); } +/** Read auth configuration from environment variables. */ export function getOpenTrustAuthConfig(): OpenTrustAuthConfig { const mode = (process.env.OPENTRUST_AUTH_MODE ?? "token") as OpenTrustAuthMode; const allowLocalhostBypass = process.env.OPENTRUST_ALLOW_LOCALHOST_BYPASS !== "false"; @@ -36,18 +37,21 @@ export function getOpenTrustAuthConfig(): OpenTrustAuthConfig { }; } +/** Check whether a hostname resolves to a loopback address (localhost/127.0.0.1/::1). */ export function isLoopbackHost(hostname: string | null | undefined) { if (!hostname) return false; const value = hostname.split(":")[0]?.toLowerCase() ?? ""; return value === "localhost" || value === "127.0.0.1" || value === "::1"; } +/** Derive a session cookie value by hashing the configured secret. */ export function createSessionValue(config: OpenTrustAuthConfig) { const secret = config.mode === "password" ? config.password : config.token; if (!secret) return null; return sha256(`opentrust:${config.mode}:${secret}`); } +/** Verify a credential (token or password) against the auth config using timing-safe comparison. */ export function verifyCredential(input: string, config: OpenTrustAuthConfig) { if (config.mode === "none") return true; const expected = config.mode === "password" ? config.password : config.token; @@ -55,6 +59,7 @@ export function verifyCredential(input: string, config: OpenTrustAuthConfig) { return safeEqual(input, expected); } +/** Extract host, IP, and user-agent from the incoming request headers. */ export async function getRequestMeta() { const store = await headers(); const forwardedHost = store.get("x-forwarded-host"); @@ -70,10 +75,12 @@ export async function getRequestMeta() { }; } +/** Return the hostname from the current request (respects x-forwarded-host). */ export async function getRequestHostname() { return (await getRequestMeta()).host; } +/** Check whether the current request is authenticated via cookie or localhost bypass. */ export async function isAuthenticatedRequest() { const config = getOpenTrustAuthConfig(); if (config.mode === "none") return true; @@ -91,6 +98,7 @@ export async function isAuthenticatedRequest() { return !!existing && safeEqual(existing, sessionValue); } +/** Guard for API routes. Returns a 401 response if unauthenticated, or null if OK. */ export async function requireApiAuth() { const ok = await isAuthenticatedRequest(); return ok ? null : NextResponse.json({ ok: false, error: "Unauthorized" }, { status: 401 }); diff --git a/lib/opentrust/bootstrap.ts b/lib/opentrust/bootstrap.ts index df42ecb..b3faa9c 100644 --- a/lib/opentrust/bootstrap.ts +++ b/lib/opentrust/bootstrap.ts @@ -1,6 +1,9 @@ import { existsSync, readdirSync, readFileSync } from "node:fs"; import path from "node:path"; import { ensureMigrated, escapeSqlString, runSql } from "@/lib/opentrust/db"; +import { createLogger } from "@/lib/opentrust/logger"; + +const log = createLogger("bootstrap"); function sqlJson(value: unknown) { return escapeSqlString(JSON.stringify(value)); @@ -77,6 +80,10 @@ function getBuiltinSkillsDirectory() { return null; } +/** + * Run migrations and sync capabilities (skills, plugins, soul, bundle) + * from the local filesystem. Safe to call on every request. + */ export function ensureBootstrapped() { ensureMigrated(); syncSkillsFromDirectory(path.join(process.env.HOME ?? "", ".openclaw", "workspace", "skills"), "workspace-skills"); @@ -85,7 +92,7 @@ export function ensureBootstrapped() { syncSkillsFromDirectory(builtinSkillsDirectory, "builtin-skills"); } else if (process.env.NODE_ENV !== "production" && !warnedAboutMissingBuiltinSkills) { warnedAboutMissingBuiltinSkills = true; - console.warn("OpenTrust could not locate builtin OpenClaw skills; skipping builtin skill sync."); + log.warn("Could not locate builtin OpenClaw skills, skipping builtin skill sync"); } syncPluginsFromConfig(); syncSoulFromIdentity(); diff --git a/lib/opentrust/db.ts b/lib/opentrust/db.ts index 31e455d..3901ef1 100644 --- a/lib/opentrust/db.ts +++ b/lib/opentrust/db.ts @@ -17,11 +17,13 @@ function ensureStorageDir() { } } +/** Return the absolute path to the SQLite database file. */ export function getDatabasePath() { ensureStorageDir(); return dbPath; } +/** Return the shared SQLite database instance, creating it on first access. */ export function getDb() { if (globalForDb.__opentrustDb) return globalForDb.__opentrustDb; @@ -33,30 +35,49 @@ export function getDb() { return db; } +/** Execute a raw SQL string (no parameter binding). */ export function runSql(sql: string) { getDb().exec(sql); } +/** Execute a SQL query and return all matching rows as typed objects. */ export function queryJson(sql: string, params?: Record) { const statement = getDb().prepare(sql); return (params ? statement.all(params as never) : statement.all()) as T[]; } +/** Execute a SQL query and return the first row, or null if none match. */ export function queryOne(sql: string, params?: Record) { const statement = getDb().prepare(sql); const row = (params ? statement.get(params as never) : statement.get()) as T | undefined; return row ?? null; } +/** Execute a SQL statement with optional named parameters and return the run result. */ export function execute(sql: string, params?: Record) { const statement = getDb().prepare(sql); return params ? statement.run(params as never) : statement.run(); } +/** + * Execute a callback inside a SQLite transaction. If the callback throws, + * the transaction is rolled back automatically. Returns the callback's result. + */ +export function withTransaction(fn: () => T): T { + const db = getDb(); + const wrapped = db.transaction(fn); + return wrapped(); +} + +/** Escape a string for safe inclusion in a raw SQL literal (single-quote wrapping). */ export function escapeSqlString(value: string) { return `'${value.replaceAll("'", "''")}'`; } +/** + * Run all database migrations to bring the schema up to date. + * Safe to call repeatedly; uses IF NOT EXISTS and column checks. + */ export function ensureMigrated() { const migrationSql = readFileSync(migrationPath, "utf8"); runSql(migrationSql); diff --git a/lib/opentrust/import-cron.ts b/lib/opentrust/import-cron.ts index e88293b..e5daafc 100644 --- a/lib/opentrust/import-cron.ts +++ b/lib/opentrust/import-cron.ts @@ -1,7 +1,7 @@ import { existsSync, readFileSync } from "node:fs"; import path from "node:path"; import { upsertArtifactsForWorkflow } from "@/lib/opentrust/artifact-extract"; -import { escapeSqlString, runSql } from "@/lib/opentrust/db"; +import { escapeSqlString, runSql, withTransaction } from "@/lib/opentrust/db"; import { getIngestionState, recordIngestionState } from "@/lib/opentrust/ingestion-state"; interface CronJobEntry { @@ -77,6 +77,7 @@ function inferStatus(job: CronJobEntry, records: CronRunRecord[]) { } function upsertWorkflow(job: CronJobEntry, records: CronRunRecord[]) { + withTransaction(() => { const workflowId = `workflow:cron:${job.id}`; const startedAt = msToIso(job.state?.lastRunAtMs ?? records[0]?.runAtMs ?? Date.now()); const updatedAt = msToIso(job.state?.lastRunAtMs ?? records.at(-1)?.ts ?? Date.now()); @@ -145,8 +146,14 @@ function upsertWorkflow(job: CronJobEntry, records: CronRunRecord[]) { importedCount: records.length, metadata: { workflowId, latestStatus: status }, }); + }); } +/** + * Import cron job definitions and their run history as workflow records. + * Skips jobs that have not changed since the last import. + * @returns The number of workflows imported. + */ export function importCronWorkflows(limit = 24) { const jobs = loadJobs().slice(0, limit); let imported = 0; diff --git a/lib/opentrust/import-openclaw.ts b/lib/opentrust/import-openclaw.ts index 6d331dd..b72e0cc 100644 --- a/lib/opentrust/import-openclaw.ts +++ b/lib/opentrust/import-openclaw.ts @@ -1,7 +1,7 @@ import { existsSync, readFileSync } from "node:fs"; import path from "node:path"; import { upsertArtifactsForTrace } from "@/lib/opentrust/artifact-extract"; -import { execute, escapeSqlString, runSql } from "@/lib/opentrust/db"; +import { execute, escapeSqlString, runSql, withTransaction } from "@/lib/opentrust/db"; import { getIngestionState, recordIngestionState } from "@/lib/opentrust/ingestion-state"; import { makeToolCallDraft, maybeBuildToolResultUpdate } from "@/lib/opentrust/tool-results"; @@ -78,6 +78,7 @@ function loadJsonlRecords(file: string): JsonlRecord[] { } function upsertSessionAndTrace(sessionKey: string, entry: SessionIndexEntry, records: JsonlRecord[]) { + withTransaction(() => { const sessionId = entry.sessionId as string; const traceId = `trace:session:${sessionId}`; const startedAt = records.find((record) => record.type === "session")?.timestamp ?? new Date(entry.updatedAt ?? Date.now()).toISOString(); @@ -269,8 +270,14 @@ function upsertSessionAndTrace(sessionKey: string, entry: SessionIndexEntry, rec importedCount: records.length, metadata: { traceId, sessionFile: entry.sessionFile ?? null }, }); + }); } +/** + * Import the most recent OpenClaw sessions into the database. + * Skips sessions that have not changed since the last import. + * @returns The number of sessions imported. + */ export function importRecentOpenClawSessions(limit = 24) { const items = loadSessionIndex().slice(0, limit); let imported = 0; diff --git a/lib/opentrust/ingestion-state.ts b/lib/opentrust/ingestion-state.ts index 755a0c4..f19a307 100644 --- a/lib/opentrust/ingestion-state.ts +++ b/lib/opentrust/ingestion-state.ts @@ -11,6 +11,7 @@ export interface IngestionStateRow { metadata_json: string; } +/** Upsert the ingestion cursor and status for a given source. */ export function recordIngestionState(input: { sourceKey: string; sourceKind: string; @@ -64,6 +65,7 @@ export function recordIngestionState(input: { ); } +/** List all ingestion source states, ordered by most recent run. */ export function getIngestionStates(limit = 12): IngestionStateRow[] { return queryJson( ` @@ -76,6 +78,7 @@ export function getIngestionStates(limit = 12): IngestionStateRow[] { ); } +/** Fetch the ingestion state for a specific source key, or null if not yet tracked. */ export function getIngestionState(sourceKey: string): IngestionStateRow | null { return queryOne( ` diff --git a/lib/opentrust/logger.ts b/lib/opentrust/logger.ts new file mode 100644 index 0000000..36d112d --- /dev/null +++ b/lib/opentrust/logger.ts @@ -0,0 +1,56 @@ +/** + * Lightweight structured logger for OpenTrust. + * Outputs JSON to stdout/stderr for machine parsing. + * No external dependencies. + */ + +export type LogLevel = "debug" | "info" | "warn" | "error"; + +const LOG_LEVELS: Record = { + debug: 0, + info: 1, + warn: 2, + error: 3, +}; + +const currentLevel: LogLevel = (process.env.OPENTRUST_LOG_LEVEL as LogLevel) ?? "info"; + +function shouldLog(level: LogLevel): boolean { + return LOG_LEVELS[level] >= LOG_LEVELS[currentLevel]; +} + +function emit(level: LogLevel, module: string, message: string, ctx?: Record) { + if (!shouldLog(level)) return; + + const entry = { + ts: new Date().toISOString(), + level, + module, + message, + ...ctx, + }; + + const output = JSON.stringify(entry); + + if (level === "error" || level === "warn") { + process.stderr.write(output + "\n"); + } else { + process.stdout.write(output + "\n"); + } +} + +/** + * Create a scoped logger for a specific module. + * + * @example + * const log = createLogger("import-openclaw"); + * log.info("Session imported", { sessionKey, recordCount: 42 }); + */ +export function createLogger(module: string) { + return { + debug: (message: string, ctx?: Record) => emit("debug", module, message, ctx), + info: (message: string, ctx?: Record) => emit("info", module, message, ctx), + warn: (message: string, ctx?: Record) => emit("warn", module, message, ctx), + error: (message: string, ctx?: Record) => emit("error", module, message, ctx), + }; +} diff --git a/lib/opentrust/memory-entries.ts b/lib/opentrust/memory-entries.ts index 692a82d..4703f61 100644 --- a/lib/opentrust/memory-entries.ts +++ b/lib/opentrust/memory-entries.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import { ensureBootstrapped } from "@/lib/opentrust/bootstrap"; -import { execute, queryJson, queryOne } from "@/lib/opentrust/db"; +import { execute, queryJson, queryOne, withTransaction } from "@/lib/opentrust/db"; import { getMemoryConfig } from "@/lib/opentrust/memory-config"; import type { MemoryEntry, @@ -57,9 +57,10 @@ function loadTags(memoryEntryId: string) { ); } +/** Create a new memory entry with origins, tags, and optional review metadata. */ export function createMemoryEntry(input: MemoryPromoteRequest): MemoryPromoteResponse { ensureBootstrapped(); - + return withTransaction(() => { const id = `mem_${randomUUID()}`; const now = nowIso(); const reviewStatus = input.review?.status ?? "draft"; @@ -170,12 +171,15 @@ export function createMemoryEntry(input: MemoryPromoteRequest): MemoryPromoteRes updatedAt: now, }, }; + }); } +/** Alias for {@link createMemoryEntry}. Promotes source material into a memory entry. */ export function promoteToMemory(input: MemoryPromoteRequest) { return createMemoryEntry(input); } +/** Fetch a single memory entry by ID, including its origins and tags. */ export function getMemoryEntry(id: string): MemoryEntryWithOrigins | null { ensureBootstrapped(); @@ -216,6 +220,7 @@ export function getMemoryEntry(id: string): MemoryEntryWithOrigins | null { }; } +/** List memory entries with optional filters for review status and retention class. */ export function listMemoryEntries(filters?: { reviewStatus?: MemoryReviewStatus; retentionClass?: MemoryEntry["retention_class"]; @@ -272,6 +277,7 @@ export function listMemoryEntries(filters?: { })); } +/** Return memory entries in "draft" status that are awaiting human review. */ export function listMemoryReviewQueue(limit = 50): MemoryReviewQueueItem[] { return listMemoryEntries({ reviewStatus: "draft", limit }); } @@ -338,6 +344,7 @@ function snapshotVersion( return version; } +/** List all version snapshots for a memory entry, newest first. */ export function listMemoryEntryVersions(entryId: string): MemoryEntryVersion[] { ensureBootstrapped(); return queryJson( @@ -353,6 +360,7 @@ export function listMemoryEntryVersions(entryId: string): MemoryEntryVersion[] { ); } +/** Fetch a specific version snapshot of a memory entry. */ export function getMemoryEntryVersion(entryId: string, version: number): MemoryEntryVersion | null { ensureBootstrapped(); return queryOne( @@ -373,7 +381,7 @@ export function getMemoryEntryVersion(entryId: string, version: number): MemoryE */ export function rollbackMemoryEntry(entryId: string, toVersion: number): MemoryEntryWithOrigins | null { ensureBootstrapped(); - + return withTransaction(() => { const target = getMemoryEntryVersion(entryId, toVersion); if (!target) return null; @@ -405,12 +413,14 @@ export function rollbackMemoryEntry(entryId: string, toVersion: number): MemoryE ); return getMemoryEntry(entryId); + }); } // --------------------------------------------------------------------------- // Update Operations (with version tracking) // --------------------------------------------------------------------------- +/** Update the review status of a memory entry, snapshotting the previous state. */ export function updateMemoryEntryReview(input: { id: string; reviewStatus: MemoryReviewStatus; @@ -418,7 +428,7 @@ export function updateMemoryEntryReview(input: { reviewNotes?: string | null; }) { ensureBootstrapped(); - + return withTransaction(() => { const existing = getMemoryEntry(input.id); if (existing) { snapshotVersion( @@ -452,8 +462,10 @@ export function updateMemoryEntryReview(input: { ); return getMemoryEntry(input.id); + }); } +/** Update a memory entry's content or metadata, creating a version snapshot first. */ export function updateMemoryEntry(input: { id: string; title?: string; @@ -467,7 +479,7 @@ export function updateMemoryEntry(input: { changeReason?: string; }) { ensureBootstrapped(); - + return withTransaction(() => { const existing = getMemoryEntry(input.id); if (!existing) return null; @@ -508,6 +520,7 @@ export function updateMemoryEntry(input: { ); return getMemoryEntry(input.id); + }); } // --------------------------------------------------------------------------- @@ -630,12 +643,13 @@ function purgeOverflowEntries(): number { */ export function archiveMemory(): ArchiveResult { ensureBootstrapped(); - + return withTransaction(() => { const agedOut = ageOutEntries(); const archived = archiveStaleEntries(); const overflowPurged = purgeOverflowEntries(); return { agedOut, archived, overflowPurged }; + }); } /** diff --git a/lib/opentrust/search.ts b/lib/opentrust/search.ts index 01a791d..2a41f32 100644 --- a/lib/opentrust/search.ts +++ b/lib/opentrust/search.ts @@ -1,6 +1,5 @@ import { ensureBootstrapped } from "@/lib/opentrust/bootstrap"; import { queryJson } from "@/lib/opentrust/db"; -import { listMemoryEntries } from "@/lib/opentrust/memory-entries"; import { searchSemanticFallback } from "@/lib/opentrust/semantic"; export interface InvestigationResult { @@ -17,27 +16,41 @@ function fts5Escape(raw: string): string { return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" "); } +/** + * Search across memory entries and evidence (traces/artifacts) using a + * layered strategy: memory SQL search → FTS5 → semantic fallback. + */ export function searchInvestigations(query: string): InvestigationResult[] { ensureBootstrapped(); const q = query.trim(); if (!q) return []; - const normalized = q.toLowerCase(); - const memoryResults = listMemoryEntries({ limit: 50 }) - .filter((entry) => { - const haystack = [entry.title, entry.body, entry.summary ?? "", ...entry.tags.map((tag) => tag.tag)] - .join(" ") - .toLowerCase(); - return haystack.includes(normalized); - }) - .map((entry) => ({ - source_id: entry.id, - title: entry.title, - snippet: entry.summary ?? entry.body.slice(0, 280), - mode: "memory-entry" as const, - sourceType: "memory" as const, - })); + // Search memory entries at the SQL level instead of loading all into memory. + // Uses LIKE for broad matching across title, body, and summary. + const likePattern = `%${q.replace(/%/g, "\\%").replace(/_/g, "\\_")}%`; + const memoryResults = queryJson<{ id: string; title: string; summary: string | null; body: string }>( + ` + SELECT id, title, summary, body + FROM memory_entries + WHERE archived_at IS NULL + AND review_status != 'rejected' + AND ( + title LIKE :pattern ESCAPE '\\' + OR body LIKE :pattern ESCAPE '\\' + OR summary LIKE :pattern ESCAPE '\\' + ) + ORDER BY updated_at DESC + LIMIT 8; + `, + { pattern: likePattern }, + ).map((entry) => ({ + source_id: entry.id, + title: entry.title, + snippet: entry.summary ?? entry.body.slice(0, 280), + mode: "memory-entry" as const, + sourceType: "memory" as const, + })); const ftsQuery = fts5Escape(q); const ftsResults = queryJson( diff --git a/lib/opentrust/semantic.ts b/lib/opentrust/semantic.ts index e84c484..7db5889 100644 --- a/lib/opentrust/semantic.ts +++ b/lib/opentrust/semantic.ts @@ -81,6 +81,7 @@ function tryLoadSqliteVec() { } } +/** Create the semantic_chunks and semantic_index_state tables if they do not exist. */ export function ensureSemanticTables() { runSql(` CREATE TABLE IF NOT EXISTS semantic_chunks ( @@ -104,6 +105,11 @@ export function ensureSemanticTables() { `); } +/** + * Rebuild the semantic chunk index from scratch by re-chunking all traces + * and artifacts. Loads sqlite-vec for vector search when available. + * @returns The number of chunks inserted. + */ export function rebuildSemanticChunks() { ensureSemanticTables(); const vec = tryLoadSqliteVec(); @@ -215,6 +221,7 @@ export function rebuildSemanticChunks() { return inserted; } +/** Return the current semantic index health: chunk count, vector readiness, and last run time. */ export function getSemanticStatus(): SemanticStatus { ensureSemanticTables(); @@ -236,6 +243,10 @@ export function getSemanticStatus(): SemanticStatus { }; } +/** + * Search semantic chunks using vector similarity when available, + * falling back to a LIKE query on chunk body/title. + */ export function searchSemanticFallback(query: string) { ensureSemanticTables(); const state = getSemanticStatus(); diff --git a/lib/opentrust/sql-runner.ts b/lib/opentrust/sql-runner.ts index cddbf19..d337269 100644 --- a/lib/opentrust/sql-runner.ts +++ b/lib/opentrust/sql-runner.ts @@ -2,6 +2,10 @@ import { getDb } from "@/lib/opentrust/db"; const READ_ONLY_PREFIXES = ["select", "with", "pragma table_info", "pragma database_list"]; +/** + * Execute a read-only SQL statement for investigation previews. + * Rejects anything that is not a SELECT, WITH, or PRAGMA query. + */ export function runReadOnlySql(sql: string) { const normalized = sql.trim().toLowerCase(); if (!READ_ONLY_PREFIXES.some((prefix) => normalized.startsWith(prefix))) { diff --git a/next-env.d.ts b/next-env.d.ts index 9edff1c..c4b7818 100644 --- a/next-env.d.ts +++ b/next-env.d.ts @@ -1,6 +1,6 @@ /// /// -import "./.next/types/routes.d.ts"; +import "./.next/dev/types/routes.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/app/api-reference/config/typescript for more information.