diff --git a/package.json b/package.json index f71c72c8ff..9c0e88a322 100644 --- a/package.json +++ b/package.json @@ -82,7 +82,8 @@ "typecheck:turbo": "turbo run typecheck", "bench": "bun scripts/bench-memory.ts", "bench:ingest": "bun scripts/bench-memory.ts ingest", - "bench:evaluate": "bun scripts/bench-memory.ts --resume run --from-phase search" + "bench:evaluate": "bun scripts/bench-memory.ts --resume run --from-phase search", + "eval:transcript-import": "bun scripts/evals/transcript-import-real-daemon.ts" }, "devDependencies": { "@biomejs/biome": "^2.5.5", diff --git a/platform/core/src/migrations/146-source-transcript-import.ts b/platform/core/src/migrations/146-source-transcript-import.ts new file mode 100644 index 0000000000..fbabf17d20 --- /dev/null +++ b/platform/core/src/migrations/146-source-transcript-import.ts @@ -0,0 +1,67 @@ +import type { MigrationDb } from "./contract"; + +/** Migration 146: durable, agent-scoped transcript import ledgers. */ +export function up(db: MigrationDb): void { + db.exec(` + CREATE TABLE IF NOT EXISTS source_import_jobs ( + id TEXT PRIMARY KEY, kind TEXT NOT NULL CHECK (kind = 'import'), agent_id TEXT NOT NULL, + schema_id TEXT NOT NULL, adapter_version INTEGER NOT NULL CHECK (adapter_version = 1), + state TEXT NOT NULL CHECK (state IN ('staging','inventorying','queued','running','paused','completed','completed_with_rejections','cancelled','failed')), + control_request TEXT CHECK (control_request IN ('pause','resume','retry','cancel') OR control_request IS NULL), + generation INTEGER NOT NULL DEFAULT 0, total INTEGER NOT NULL DEFAULT 0, imported INTEGER NOT NULL DEFAULT 0, + duplicate INTEGER NOT NULL DEFAULT 0, rejected INTEGER NOT NULL DEFAULT 0, pending INTEGER NOT NULL DEFAULT 0, + lease_token TEXT, lease_expires_at TEXT, error TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), started_at TEXT, completed_at TEXT, reconciled_at TEXT + ); + CREATE INDEX IF NOT EXISTS idx_source_import_jobs_agent_state ON source_import_jobs(agent_id, state); + + CREATE TABLE IF NOT EXISTS source_import_files ( + id TEXT PRIMARY KEY, job_id TEXT NOT NULL REFERENCES source_import_jobs(id) ON DELETE CASCADE, + source_id TEXT NOT NULL, agent_id TEXT NOT NULL, ordinal INTEGER NOT NULL, name TEXT NOT NULL, + managed_path TEXT NOT NULL, size_bytes INTEGER NOT NULL DEFAULT 0, content_hash TEXT, + state TEXT NOT NULL CHECK (state IN ('staging','ready','inventorying','completed','failed')), + record_count INTEGER NOT NULL DEFAULT 0, blank_count INTEGER NOT NULL DEFAULT 0, malformed_count INTEGER NOT NULL DEFAULT 0, + inventory_version INTEGER NOT NULL DEFAULT 1, checkpoint_ordinal INTEGER NOT NULL DEFAULT 0, + checkpoint_byte_offset INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL DEFAULT (datetime('now')), updated_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(job_id, ordinal) + ); + CREATE INDEX IF NOT EXISTS idx_source_import_files_job_state ON source_import_files(job_id, state); + + CREATE TABLE IF NOT EXISTS source_import_records ( + id TEXT PRIMARY KEY, job_id TEXT NOT NULL REFERENCES source_import_jobs(id) ON DELETE CASCADE, + file_id TEXT NOT NULL REFERENCES source_import_files(id) ON DELETE CASCADE, source_id TEXT NOT NULL, agent_id TEXT NOT NULL, + ordinal INTEGER NOT NULL, line_number INTEGER NOT NULL, byte_offset INTEGER NOT NULL, byte_length INTEGER NOT NULL, + raw_hash TEXT NOT NULL, external_identity TEXT, conversation_fingerprint TEXT, status TEXT NOT NULL CHECK (status IN ('pending','imported','duplicate','rejected','cancelled')), + canonical_id TEXT, canonical_key TEXT, rejection_code TEXT, attempt_count INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (datetime('now')), updated_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(file_id, ordinal), UNIQUE(file_id, byte_offset) + ); + CREATE INDEX IF NOT EXISTS idx_source_import_records_job_status ON source_import_records(job_id, status); + CREATE INDEX IF NOT EXISTS idx_source_import_records_source_status ON source_import_records(agent_id, source_id, status); + + CREATE TABLE IF NOT EXISTS transcript_import_conversations ( + agent_id TEXT NOT NULL, external_identity TEXT NOT NULL, canonical_key TEXT NOT NULL, + conversation_fingerprint TEXT NOT NULL, canonical_id TEXT NOT NULL, owner_source_id TEXT NOT NULL, + owner_record_id TEXT NOT NULL, state TEXT NOT NULL CHECK (state IN ('committing','committed','removed')), + content_hash TEXT NOT NULL, harness TEXT NOT NULL, timestamp TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), updated_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (agent_id, external_identity), UNIQUE(agent_id, canonical_key) + ); + CREATE INDEX IF NOT EXISTS idx_transcript_import_conversations_source ON transcript_import_conversations(agent_id, owner_source_id); + + CREATE TABLE IF NOT EXISTS source_import_record_attempts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, agent_id TEXT NOT NULL, job_id TEXT NOT NULL, file_id TEXT NOT NULL, + record_id TEXT NOT NULL, generation INTEGER NOT NULL, outcome TEXT NOT NULL, error_code TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE INDEX IF NOT EXISTS idx_source_import_attempts_record ON source_import_record_attempts(agent_id, record_id, created_at); + `); + for (const column of ["source_id", "source_record_id", "source_meta_json"] as const) { + const exists = db + .prepare("SELECT 1 AS found FROM pragma_table_info('session_transcripts') WHERE name = ?") + .get(column); + if (exists == null) db.exec(`ALTER TABLE session_transcripts ADD COLUMN ${column} TEXT`); + } + db.exec( + "CREATE INDEX IF NOT EXISTS idx_session_transcripts_agent_source ON session_transcripts(agent_id, source_id)", + ); +} diff --git a/platform/core/src/migrations/147-source-import-replay-file-slots.ts b/platform/core/src/migrations/147-source-import-replay-file-slots.ts new file mode 100644 index 0000000000..9d3d31cc77 --- /dev/null +++ b/platform/core/src/migrations/147-source-import-replay-file-slots.ts @@ -0,0 +1,27 @@ +import type { MigrationDb } from "./contract"; + +/** Migration 147: one configured Source may own multiple auditable replay file slots. */ +export function up(db: MigrationDb): void { + db.exec(` + CREATE TABLE source_import_files_147 ( + id TEXT PRIMARY KEY, job_id TEXT NOT NULL REFERENCES source_import_jobs(id) ON DELETE CASCADE, + source_id TEXT NOT NULL, agent_id TEXT NOT NULL, ordinal INTEGER NOT NULL, name TEXT NOT NULL, + managed_path TEXT NOT NULL, size_bytes INTEGER NOT NULL DEFAULT 0, content_hash TEXT, + state TEXT NOT NULL CHECK (state IN ('staging','ready','inventorying','completed','failed')), + record_count INTEGER NOT NULL DEFAULT 0, blank_count INTEGER NOT NULL DEFAULT 0, malformed_count INTEGER NOT NULL DEFAULT 0, + inventory_version INTEGER NOT NULL DEFAULT 1, checkpoint_ordinal INTEGER NOT NULL DEFAULT 0, + checkpoint_byte_offset INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL DEFAULT (datetime('now')), updated_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(job_id, ordinal) + ); + INSERT INTO source_import_files_147 ( + id, job_id, source_id, agent_id, ordinal, name, managed_path, size_bytes, content_hash, state, + record_count, blank_count, malformed_count, inventory_version, checkpoint_ordinal, checkpoint_byte_offset, created_at, updated_at + ) + SELECT id, job_id, source_id, agent_id, ordinal, name, managed_path, size_bytes, content_hash, state, + record_count, blank_count, malformed_count, inventory_version, checkpoint_ordinal, checkpoint_byte_offset, created_at, updated_at + FROM source_import_files; + DROP TABLE source_import_files; + ALTER TABLE source_import_files_147 RENAME TO source_import_files; + CREATE INDEX idx_source_import_files_job_state ON source_import_files(job_id, state); + `); +} diff --git a/platform/core/src/migrations/148-source-import-attempt-provenance.ts b/platform/core/src/migrations/148-source-import-attempt-provenance.ts new file mode 100644 index 0000000000..156986ae5d --- /dev/null +++ b/platform/core/src/migrations/148-source-import-attempt-provenance.ts @@ -0,0 +1,9 @@ +import type { MigrationDb } from "./contract"; + +/** Migration 148: retain source identity on append-only import attempt tombstones. */ +export function up(db: MigrationDb): void { + const columns = db.prepare("PRAGMA table_info(source_import_record_attempts)").all() as Array<{ name?: unknown }>; + if (!columns.some((column) => column.name === "source_id")) { + db.exec("ALTER TABLE source_import_record_attempts ADD COLUMN source_id TEXT"); + } +} diff --git a/platform/core/src/migrations/index.ts b/platform/core/src/migrations/index.ts index 4da76f1e3e..3b04ac3534 100644 --- a/platform/core/src/migrations/index.ts +++ b/platform/core/src/migrations/index.ts @@ -153,6 +153,9 @@ import { up as sourceSyncFrontier } from "./142-source-sync-frontier"; import { up as embeddingIndexProgress } from "./143-embedding-index-progress"; import { up as memoryJobLeaseToken } from "./144-memory-job-lease-token"; import { up as dreamingEvidenceReviews } from "./145-dreaming-evidence-reviews"; +import { up as sourceTranscriptImport } from "./146-source-transcript-import"; +import { up as sourceImportReplayFileSlots } from "./147-source-import-replay-file-slots"; +import { up as sourceImportAttemptProvenance } from "./148-source-import-attempt-provenance"; export type { Migration, MigrationArtifacts, MigrationDb } from "./contract"; @@ -1313,6 +1316,37 @@ export const MIGRATIONS: readonly Migration[] = [ up: dreamingEvidenceReviews, artifacts: { tables: ["dreaming_evidence_reviews"] }, }, + { + version: 146, + name: "source-transcript-import", + up: sourceTranscriptImport, + artifacts: { + tables: [ + "source_import_jobs", + "source_import_files", + "source_import_records", + "transcript_import_conversations", + "source_import_record_attempts", + ], + columns: [ + { table: "session_transcripts", column: "source_id" }, + { table: "session_transcripts", column: "source_record_id" }, + { table: "session_transcripts", column: "source_meta_json" }, + ], + }, + }, + { + version: 147, + name: "source-import-replay-file-slots", + up: sourceImportReplayFileSlots, + artifacts: { tables: ["source_import_files"] }, + }, + { + version: 148, + name: "source-import-attempt-provenance", + up: sourceImportAttemptProvenance, + artifacts: { columns: [{ table: "source_import_record_attempts", column: "source_id" }] }, + }, ]; /** Simple checksum for audit trail (hash of migration name + version). */ diff --git a/platform/core/src/migrations/migrations.test.ts b/platform/core/src/migrations/migrations.test.ts index 50639c866f..1c2f368afa 100644 --- a/platform/core/src/migrations/migrations.test.ts +++ b/platform/core/src/migrations/migrations.test.ts @@ -139,7 +139,7 @@ describe("migration framework", () => { runMigrations(db); const applied = db.query("SELECT MAX(version) AS version FROM schema_migrations").get() as { version: number }; - expect(applied.version).toBe(145); + expect(applied.version).toBe(148); expect( (db.query("PRAGMA table_info(memory_jobs)").all() as Array<{ name: string }>).some( (column) => column.name === "lease_token", diff --git a/platform/daemon/src/daemon.ts b/platform/daemon/src/daemon.ts index 7e83a097bf..7521a81e05 100644 --- a/platform/daemon/src/daemon.ts +++ b/platform/daemon/src/daemon.ts @@ -141,7 +141,9 @@ import { startPipeline, stopPipeline, } from "./pipeline"; +import { randomUUID } from "node:crypto"; import { recordDreamingPassTelemetry } from "./pipeline/dreaming"; +import { dbOwnerTransaction } from "./db-owner-runtime"; import { type DreamingWorkerHandle, startDreamingWorker } from "./pipeline/dreaming-worker"; import { retireLegacyExtractionJobsAsync } from "./pipeline/extraction-fallback"; import { invalidateTraversalCache } from "./pipeline/graph-traversal"; @@ -238,6 +240,8 @@ import { } from "./telemetry"; import { type TranscriptCaptureWorkerHandle, startTranscriptCaptureWorker } from "./transcript-capture-worker"; import { type TranscriptRecoveryWorkerHandle, startTranscriptRecoveryWorker } from "./transcript-recovery-worker"; +import { type TranscriptImportWorkerHandle, startTranscriptImportWorker } from "./transcript-import-worker"; +import { createOwnerTranscriptImportStore } from "./transcript-import-store"; import { resolveDaemonRestartMode } from "./daemon-restart"; import { @@ -268,6 +272,7 @@ import { registerGraphiqRoutes } from "./routes/graphiq-routes.js"; import { mountHealthRoutes } from "./routes/health.js"; import { registerHooksRoutes } from "./routes/hooks-routes.js"; import { registerImportRoutes } from "./routes/import-routes.js"; +import { registerTranscriptImportRoutes } from "./routes/transcript-import-routes.js"; import { mountInferenceRoutes } from "./routes/inference.js"; import { registerKnowledgeRoutes } from "./routes/knowledge-routes.js"; import { mountMarketplaceReviewsRoutes } from "./routes/marketplace-reviews.js"; @@ -318,6 +323,7 @@ let embeddingPromotionRestart: Promise | null = null; let skillReconcilerHandle: ReturnType | null = null; let transcriptCaptureWorkerHandle: TranscriptCaptureWorkerHandle | null = null; let transcriptRecoveryWorkerHandle: TranscriptRecoveryWorkerHandle | null = null; +let transcriptImportWorkerHandle: TranscriptImportWorkerHandle | null = null; // These are mirrored into state.ts via setters for read access by // route modules. Only daemon.ts should assign or clear them. let telemetryRef: TelemetryCollector | undefined; @@ -524,6 +530,7 @@ registerSecretRoutes(app); registerSessionRoutes(app, { gitConfig, stopGitSyncTimer, startGitSyncTimer, getGitStatus, gitPull, gitPush, gitSync }); registerSourcesRoutes(app); registerImportRoutes(app); +registerTranscriptImportRoutes(app); registerPipelineRoutes(app); registerReflectionRoutes(app); registerTelemetryRoutes(app); @@ -1584,6 +1591,12 @@ async function stopPipelineRuntime(): Promise { reflectionWorkerHandle = null; } + if (transcriptImportWorkerHandle) { + try { + await transcriptImportWorkerHandle.stop(); + } catch {} + transcriptImportWorkerHandle = null; + } if (transcriptRecoveryWorkerHandle) { try { await transcriptRecoveryWorkerHandle.stop(); @@ -2069,7 +2082,7 @@ async function cleanup() { const renderWorker = getSynthesisRenderWorker(); if (renderWorker !== null) { setSynthesisRenderWorker(null); - renderWorker.terminate().catch((e) => { + await renderWorker.terminate().catch((e) => { logger.debug("daemon", "render worker terminate failed", { error: e instanceof Error ? e.message : String(e), }); @@ -2799,6 +2812,36 @@ async function main() { resolveDaemonAgentId(), ); } + if (!transcriptImportWorkerHandle) { + transcriptImportWorkerHandle = startTranscriptImportWorker({ + store: createOwnerTranscriptImportStore(), + agentId: resolveDaemonAgentId(), + workspaceRoot: AGENTS_DIR, + onBatch: async (_jobId, sourceId) => { + const agentId = resolveDaemonAgentId(); + const subjectRef = `source:${sourceId}`; + const details = JSON.stringify({ sourceId, reason: "transcript-import-committed" }); + await dbOwnerTransaction( + [ + { + sql: `INSERT INTO dreaming_attention + (id, agent_id, kind, subject_ref, details_json, priority) + VALUES (?, ?, 'evidence_requeue', ?, ?, 50) + ON CONFLICT(agent_id, kind, subject_ref) DO UPDATE SET + details_json = excluded.details_json, + priority = MAX(dreaming_attention.priority, excluded.priority), + generation = dreaming_attention.generation + 1, + resolved_at = NULL, + resolved_by_pass_id = NULL`, + params: [randomUUID(), agentId, subjectRef, details], + result: "run", + }, + ], + { operation: "sources.import.dreaming-attention", lane: "write" }, + ); + }, + }); + } checkpointPruneTimer = setInterval(() => { try { @@ -2898,22 +2941,17 @@ async function main() { const createBoundedServer = (...args: Parameters) => { const server = nodeCreateServer(...args); server.on("request", (req, res) => { - let bytes = 0; - let aborted = false; - req.on("data", (chunk: Buffer) => { - if (aborted) return; - bytes += chunk.length; - if (bytes > REQUEST_BODY_LIMIT) { - aborted = true; - logger.warn("http", "Request body exceeded limit", { bytes, limit: REQUEST_BODY_LIMIT }); - if (!res.headersSent) { - res.writeHead(413, { "Content-Type": "application/json" }); - res.end(JSON.stringify({ error: "payload too large" }), () => { - req.socket?.destroy(); - }); - } - } - }); + // Never attach a data listener here. Node switches the request into + // flowing mode and drains the one-shot stream before Hono can consume + // it. Streamed routes (notably transcript imports) enforce their own + // limits while reading the body. Content-Length is the only safe + // pre-read check at this layer; chunked bodies are checked downstream. + const declared = Number(req.headers["content-length"] ?? 0); + if (declared > REQUEST_BODY_LIMIT && !res.headersSent) { + logger.warn("http", "Request body exceeded declared limit", { bytes: declared, limit: REQUEST_BODY_LIMIT }); + res.writeHead(413, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: "payload too large" }), () => req.socket?.destroy()); + } }); return server; }; diff --git a/platform/daemon/src/db-owner-client.test.ts b/platform/daemon/src/db-owner-client.test.ts index d3032efca6..b696971f1c 100644 --- a/platform/daemon/src/db-owner-client.test.ts +++ b/platform/daemon/src/db-owner-client.test.ts @@ -675,6 +675,36 @@ describe("DB owner client", () => { expect(rows).toEqual([{ id: "m1" }]); }); + test("rolls back a transaction when a required precondition changes zero rows", async () => { + const database = makeDb(); + directory = database.directory; + client = createDbOwnerClient({ dbPath: database.path }); + const transaction = client.submit( + { + kind: "transaction", + transaction: { + statements: [ + { + sql: "INSERT INTO memories (id, content) VALUES (?, ?)", + params: ["m2", "rolled back"], + result: "run", + }, + { + sql: "UPDATE memories SET content = ? WHERE id = ?", + params: ["must not persist", "missing"], + result: "run", + requireChanges: true, + }, + ], + }, + }, + { operation: "memory.transaction-precondition", lane: "write", deadlineMs: 1_000 }, + ); + await expect(transaction.result).rejects.toThrow("DB owner transaction precondition changed zero rows"); + const rows = await recallThroughDbOwner<{ id: string }>(client, "SELECT id FROM memories ORDER BY id"); + expect(rows).toEqual([{ id: "m1" }]); + }); + test("retires a broken stdin without starving the queue in a retry loop", async () => { const database = makeDb(); directory = database.directory; diff --git a/platform/daemon/src/db-owner-protocol.ts b/platform/daemon/src/db-owner-protocol.ts index 126aa4aaae..95b43eedd0 100644 --- a/platform/daemon/src/db-owner-protocol.ts +++ b/platform/daemon/src/db-owner-protocol.ts @@ -213,6 +213,31 @@ export interface DbOwnerSourceEvidenceEligibility { readonly legacyObsidianRoot?: string; } +import type { CompletedTranscriptCommit } from "./transcript-import-commit"; + +export interface DbOwnerTranscriptBulkCommit { + readonly agentId: string; + readonly jobId: string; + readonly generation: number; + readonly leaseToken: string; + readonly sourceId: string; + readonly harness: string; + readonly commits: readonly CompletedTranscriptCommit[]; +} +export interface DbOwnerTranscriptImportControl { + readonly agentId: string; + readonly jobId: string; + readonly control: "pause" | "resume" | "retry" | "cancel"; +} +export interface DbOwnerTranscriptImportReconcile { + readonly agentId: string; + readonly jobId: string; +} +export interface DbOwnerTranscriptImportPurge { + readonly agentId: string; + readonly sourceId: string; +} + export type DbOwnerRequest = | { readonly kind: "initialize"; readonly agentsDir?: string } | { readonly kind: "query"; readonly statement: DbOwnerStatement } @@ -243,6 +268,10 @@ export type DbOwnerRequest = | { readonly kind: "source_artifact_upsert"; readonly input: DbOwnerSourceArtifactUpsert } | { readonly kind: "source_artifact_upsert_batch"; readonly input: readonly DbOwnerSourceArtifactUpsert[] } | { readonly kind: "source_evidence_eligibility"; readonly input: DbOwnerSourceEvidenceEligibility } + | { readonly kind: "transcript_bulk_commit"; readonly input: DbOwnerTranscriptBulkCommit } + | { readonly kind: "transcript_import_control"; readonly input: DbOwnerTranscriptImportControl } + | { readonly kind: "transcript_import_reconcile"; readonly input: DbOwnerTranscriptImportReconcile } + | { readonly kind: "transcript_import_purge"; readonly input: DbOwnerTranscriptImportPurge } | { readonly kind: "dreaming_hygiene_attention"; readonly input: DbOwnerDreamingHygieneAttention } | { readonly kind: "dreaming_surprisal_attention"; readonly input: DbOwnerDreamingSurprisalAttention } | { readonly kind: "dreaming_episodic_backlog"; readonly input: DbOwnerDreamingEpisodicBacklog } diff --git a/platform/daemon/src/db-owner-runtime.ts b/platform/daemon/src/db-owner-runtime.ts index e8d768affe..9c22627063 100644 --- a/platform/daemon/src/db-owner-runtime.ts +++ b/platform/daemon/src/db-owner-runtime.ts @@ -13,6 +13,7 @@ import { type DbOwnerSubmitOptions, } from "./db-owner-client"; import { getDbOwnerMaintenance } from "./db-owner-maintenance"; +import { commitCompletedTranscriptBatchInTx } from "./transcript-import-commit"; import type { DbOwnerJob, DbOwnerParameter, @@ -122,9 +123,23 @@ async function executeInlineOwnerRequest(accessor: DbAccessor, request: DbOwnerR return await invokeAccessorAsync(accessor, "withReadDbAsync", (db) => inlineStatement(db, request.statement)); } if (request.kind === "transaction") { - return await invokeAccessorAsync(accessor, "withWriteTxAsync", (db) => - request.transaction.statements.map((statement) => inlineStatement(db, statement)), - ); + return await invokeAccessorAsync(accessor, "withWriteTxAsync", (db) => { + const results = request.transaction.statements.map((statement) => inlineStatement(db, statement)); + for (let i = 0; i < results.length; i++) { + const statement = request.transaction.statements[i]; + const result = results[i]; + if ( + statement?.requireChanges === true && + typeof result === "object" && + result !== null && + "changes" in result && + result.changes === 0 + ) { + throw new Error("DB owner transaction precondition changed zero rows"); + } + } + return results; + }); } if (request.kind === "batch") { return await invokeAccessorAsync(accessor, "withWriteTxAsync", (db) => { @@ -142,6 +157,29 @@ async function executeInlineOwnerRequest(accessor: DbAccessor, request: DbOwnerR return results; }); } + if (request.kind === "transcript_bulk_commit") { + return await invokeAccessorAsync(accessor, "withWriteTxAsync", (db) => + commitCompletedTranscriptBatchInTx(db as never, request.input.commits), + ); + } + if (request.kind === "transcript_import_control") { + return await invokeAccessorAsync(accessor, "withWriteTxAsync", (db) => + db + .prepare( + "UPDATE source_import_jobs SET control_request = ?, updated_at = datetime('now') WHERE id = ? AND agent_id = ?", + ) + .run(request.input.control, request.input.jobId, request.input.agentId), + ); + } + if (request.kind === "transcript_import_reconcile") { + return await invokeAccessorAsync(accessor, "withReadDbAsync", (db) => + db + .prepare( + "SELECT COUNT(*) AS total, SUM(status = 'imported') AS imported, SUM(status = 'duplicate') AS duplicate, SUM(status = 'rejected') AS rejected, SUM(status IN ('pending','cancelled')) AS pending FROM source_import_records WHERE job_id = ? AND agent_id = ?", + ) + .get(request.input.jobId, request.input.agentId), + ); + } if (request.kind === "vector_search") { return invokeAccessor(accessor, "withReadDb", (db) => vectorSearchWithMetadata(db as never, new Float32Array(request.payload.queryEmbedding), request.payload.options), @@ -175,6 +213,7 @@ async function executeInlineOwnerRequest(accessor: DbAccessor, request: DbOwnerR case "vacuum_conversion": case "incremental_vacuum": case "source_evidence_eligibility": + case "transcript_import_purge": case "sleep": throw new Error(`Unsupported inline owner request: ${request.kind}`); default: @@ -478,6 +517,48 @@ export async function dbOwnerSourceSnapshotImport( ); } +export async function dbOwnerTranscriptBulkCommit( + input: import("./db-owner-protocol").DbOwnerTranscriptBulkCommit, + options: DbOwnerSqlOptions, +): Promise { + const owner = await getDbOwner(); + return await submitWithAdmission( + owner, + { kind: "transcript_bulk_commit", input }, + { + ...options, + lane: options.lane ?? "write", + estimatedWorkUnits: options.estimatedWorkUnits ?? input.commits.length, + }, + ); +} +export async function dbOwnerTranscriptImportControl( + input: import("./db-owner-protocol").DbOwnerTranscriptImportControl, + options: DbOwnerSqlOptions, +): Promise { + const owner = await getDbOwner(); + return await submitWithAdmission( + owner, + { kind: "transcript_import_control", input }, + { ...options, lane: options.lane ?? "write" }, + ); +} +export async function dbOwnerTranscriptImportReconcile( + input: import("./db-owner-protocol").DbOwnerTranscriptImportReconcile, + options: DbOwnerSqlOptions, +): Promise { + const owner = await getDbOwner(); + return await submitWithAdmission(owner, { kind: "transcript_import_reconcile", input }, { ...options, lane: "read" }); +} + +export async function dbOwnerTranscriptImportPurge( + input: import("./db-owner-protocol").DbOwnerTranscriptImportPurge, + options: DbOwnerSqlOptions, +): Promise { + const owner = await getDbOwner(); + return await submitWithAdmission(owner, { kind: "transcript_import_purge", input }, { ...options, lane: "write" }); +} + export async function dbOwnerSourceArtifactUpsert( input: DbOwnerSourceArtifactUpsert, options: DbOwnerSqlOptions, diff --git a/platform/daemon/src/db-owner-worker.ts b/platform/daemon/src/db-owner-worker.ts index ddbe5204cb..132b30be69 100644 --- a/platform/daemon/src/db-owner-worker.ts +++ b/platform/daemon/src/db-owner-worker.ts @@ -13,6 +13,7 @@ import { upsertMemoryArtifactInTx, type MemoryArtifactUpsertFields } from "./mem import { upsertMemoryContentSafetyInTx } from "./memory-content-safety"; import { NATIVE_MEMORY_BRIDGE_SOURCE_NODE_ID } from "./native-memory-constants"; import { applySourceSnapshotImportInTx } from "./source-snapshots"; +import { commitCompletedTranscriptBatchInTx } from "./transcript-import-commit"; import { getDreamingHygieneCandidatesInDb } from "./knowledge-graph-hygiene"; import { enqueueDreamingAttentionInTx } from "./pipeline/dreaming-attention"; import { DREAMING_SURPRISAL_SELECTOR_VERSION, selectDreamingSurprisalInDb } from "./pipeline/dreaming-surprisal"; @@ -128,12 +129,16 @@ export function runDbOwnerWorker(): void { } } const parentPid = process.ppid; - const parentWatch = setInterval(() => { - // A test harness can disappear without sending the protocol shutdown - // command. Do not let its owner survive as an orphan indefinitely. - if (process.ppid !== parentPid) process.exit(0); - }, 250); - parentWatch.unref(); + let db = undefined as unknown as SqliteDatabase; + let parentWatch: ReturnType | undefined; + const closeAndExit = (code: number): never => { + if (parentWatch !== undefined) clearInterval(parentWatch); + try { + if (db !== undefined) db.close(); + } finally { + process.exit(code); + } + }; function send(event: DbOwnerEvent): void { process.stdout.write(`${JSON.stringify(event)}\n`); @@ -156,7 +161,6 @@ export function runDbOwnerWorker(): void { } const sqlitePath = process.env.SIGNET_DB_OWNER_SQLITE_PATH ?? process.env.SIGNET_SQLITE_PATH; - let db: SqliteDatabase; try { if (sqlitePath !== undefined && existsSync(sqlitePath) && typeof Database.setCustomSQLite === "function") { Database.setCustomSQLite(sqlitePath); @@ -167,9 +171,15 @@ export function runDbOwnerWorker(): void { if (vecExtension !== null) loadSqliteVecIfAvailable(db, vecExtension); } catch (error) { send({ type: "fatal", error: serializeError(error) }); - process.exit(1); - return; + closeAndExit(1); } + if (db === undefined) closeAndExit(1); + parentWatch = setInterval(() => { + // SIGKILL bypasses daemon cleanup. Close the owner database as soon as + // its control-process parent disappears, before exiting as an orphan. + if (process.ppid !== parentPid) closeAndExit(0); + }, 50); + parentWatch.unref(); const BUSY_RETRIES = 3; const BUSY_BACKOFF_MS = 50; @@ -313,6 +323,19 @@ export function runDbOwnerWorker(): void { } return withBusyRetry(() => { const results = statements.map((statement) => executeStatement({ ...statement, transactional: false }, context)); + for (let i = 0; i < results.length; i++) { + const statement = statements[i]; + const result = results[i]; + if ( + statement?.requireChanges === true && + typeof result === "object" && + result !== null && + "changes" in result && + result.changes === 0 + ) { + throw new Error("DB owner transaction precondition changed zero rows"); + } + } return enforceResultLimit({ sql: "DB_OWNER_TRANSACTION", result: "all" }, results) as readonly unknown[]; }, context); } @@ -353,6 +376,73 @@ export function runDbOwnerWorker(): void { return db.prepare(statement.sql).run(...params); } + function executeTranscriptBulkCommit( + request: Extract, + context: JobExecutionContext, + ): unknown { + if (request.input.commits.length === 0 || request.input.commits.length > 25) + throw new RangeError("invalid transcript commit batch"); + db.exec("BEGIN IMMEDIATE"); + try { + const lease = db + .prepare( + "SELECT 1 FROM source_import_jobs WHERE id = ? AND agent_id = ? AND generation = ? AND lease_token = ? AND state IN ('running','inventorying')", + ) + .get(request.input.jobId, request.input.agentId, request.input.generation, request.input.leaseToken); + if (lease == null) throw new Error("stale import lease"); + const result = commitCompletedTranscriptBatchInTx(db as never, request.input.commits); + commit(context); + return result; + } catch (error) { + try { + db.exec("ROLLBACK"); + } catch { + /* preserve original error */ + } + throw error; + } + } + + function executeTranscriptImportControl( + request: Extract, + context: JobExecutionContext, + ): unknown { + db.exec("BEGIN IMMEDIATE"); + try { + const result = db + .prepare( + "UPDATE source_import_jobs SET control_request = ?, updated_at = datetime('now') WHERE id = ? AND agent_id = ?", + ) + .run(request.input.control, request.input.jobId, request.input.agentId); + commit(context); + return { changed: result.changes > 0 }; + } catch (error) { + try { + db.exec("ROLLBACK"); + } catch { + /* preserve original error */ + } + throw error; + } + } + + function executeTranscriptImportReconcile( + request: Extract, + ): unknown { + const row = db + .prepare( + "SELECT COUNT(*) AS total, SUM(status = 'imported') AS imported, SUM(status = 'duplicate') AS duplicate, SUM(status = 'rejected') AS rejected, SUM(status IN ('pending','cancelled')) AS pending FROM source_import_records WHERE job_id = ? AND agent_id = ?", + ) + .get(request.input.jobId, request.input.agentId) as Record; + return { + total: row.total ?? 0, + imported: row.imported ?? 0, + duplicate: row.duplicate ?? 0, + rejected: row.rejected ?? 0, + pending: row.pending ?? 0, + }; + } + function executeSourceSnapshotImport( job: Extract, context?: JobExecutionContext, @@ -969,6 +1059,39 @@ export function runDbOwnerWorker(): void { if (job.request.kind === "batch") return executeBatch(job.request.statements, job.request.requireChanges === true, context); if (job.request.kind === "source_snapshot_import") return executeSourceSnapshotImport(job.request, context); + if (job.request.kind === "transcript_bulk_commit") return executeTranscriptBulkCommit(job.request, context); + if (job.request.kind === "transcript_import_control") return executeTranscriptImportControl(job.request, context); + if (job.request.kind === "transcript_import_reconcile") return executeTranscriptImportReconcile(job.request); + if (job.request.kind === "transcript_import_purge") { + db.exec("BEGIN IMMEDIATE"); + try { + const source = job.request.input.sourceId; + db.prepare("DELETE FROM session_transcripts WHERE agent_id = ? AND source_id = ?").run( + job.request.input.agentId, + source, + ); + db.prepare( + "UPDATE transcript_import_conversations SET state = 'removed', updated_at = datetime('now') WHERE agent_id = ? AND owner_source_id = ?", + ).run(job.request.input.agentId, source); + db.prepare("DELETE FROM source_import_records WHERE agent_id = ? AND source_id = ?").run( + job.request.input.agentId, + source, + ); + db.prepare("DELETE FROM source_import_files WHERE agent_id = ? AND source_id = ?").run( + job.request.input.agentId, + source, + ); + commit(context); + return { purged: true }; + } catch (error) { + try { + db.exec("ROLLBACK"); + } catch { + /* preserve original error */ + } + throw error; + } + } if (job.request.kind === "dreaming_hygiene_attention") return executeDreamingHygieneAttention(job.request, context); if (job.request.kind === "dreaming_surprisal_attention") return executeDreamingSurprisalAttention(job.request, context); @@ -1137,9 +1260,8 @@ export function runDbOwnerWorker(): void { function handle(command: DbOwnerCommand): void { if (command.type === "shutdown") { - clearInterval(parentWatch); - db.close(); - process.exit(0); + closeAndExit(0); + return; } if (command.type === "cancel") { if (!shouldRecordDbOwnerCancellation(command.jobId, activeJobId, foregroundQueue, maintenanceQueue)) return; @@ -1206,6 +1328,12 @@ export function runDbOwnerWorker(): void { } } }); + process.stdin.on("end", () => { + // The daemon owns this pipe. If it is killed before it can send the + // protocol shutdown command, close SQLite immediately rather than + // leaving an orphan owner holding the eval workspace database lock. + closeAndExit(0); + }); void activeJobId; } diff --git a/platform/daemon/src/episodic-sources.ts b/platform/daemon/src/episodic-sources.ts index 49436cc7ce..c947e1084b 100644 --- a/platform/daemon/src/episodic-sources.ts +++ b/platform/daemon/src/episodic-sources.ts @@ -138,6 +138,16 @@ function readNonEmptyTrimmed(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; } +function managedSourcePath(meta: string | null): string | null { + if (!meta) return null; + try { + const value = JSON.parse(meta) as { managedPath?: unknown }; + return readNonEmptyTrimmed(value.managedPath); + } catch { + return null; + } +} + function tableHasColumn(db: ReadDb, table: string, column: string): boolean { try { const rows = db.prepare(`PRAGMA table_info(${table})`).all() as ReadonlyArray>; @@ -301,11 +311,20 @@ export function readEpisodicTranscript(db: ReadDb, agentId: string, id: string): const completedAt = hasCompleted ? "st.completed_at" : "NULL"; const capturedAt = `COALESCE(${completedAt}, ${updatedAt}, st.created_at)`; const orderBy = `${capturedAt} DESC, st.created_at DESC`; + const hasSourceId = tableHasColumn(db, "session_transcripts", "source_id"); + const hasSourceRecordId = tableHasColumn(db, "session_transcripts", "source_record_id"); + const hasSourceMeta = tableHasColumn(db, "session_transcripts", "source_meta_json"); + const hasContentHash = tableHasColumn(db, "session_transcripts", "content_hash"); + const sourceId = hasSourceId ? "st.source_id" : "NULL"; + const sourceRecordId = hasSourceRecordId ? "st.source_record_id" : "NULL"; + const sourceMeta = hasSourceMeta ? "st.source_meta_json" : "NULL"; + const contentHash = hasContentHash ? "st.content_hash" : "NULL"; const completed = hasCompleted ? "st.completed_at IS NOT NULL" : "0"; const row = db .prepare( `SELECT st.session_key, st.content, st.harness, st.project, st.created_at, ${updatedAt} AS updated_at, - ${completedAt} AS completed_at, ${completed} AS completed + ${completedAt} AS completed_at, ${completed} AS completed, ${sourceId} AS source_id, + ${sourceRecordId} AS source_record_id, ${sourceMeta} AS source_meta_json, ${contentHash} AS content_hash FROM session_transcripts AS st WHERE st.agent_id = ? AND st.session_key IN (${placeholders}) ORDER BY ${orderBy} @@ -321,6 +340,10 @@ export function readEpisodicTranscript(db: ReadDb, agentId: string, id: string): readonly updated_at: string | null; readonly completed_at: string | null; readonly completed: number; + readonly source_id: string | null; + readonly source_record_id: string | null; + readonly source_meta_json: string | null; + readonly content_hash: string | null; } | undefined; if (!row) return null; @@ -339,9 +362,12 @@ export function readEpisodicTranscript(db: ReadDb, agentId: string, id: string): id: row.session_key, content: row.content, sourceKind: "transcript", - sourceId: row.session_key, - sourceEntryId: null, - sourcePath: null, + sourceId: row.source_record_id ?? row.session_key, + sourceEntryId: row.source_id, + sourceRevision: row.source_id + ? (row.content_hash ?? row.completed_at ?? row.updated_at ?? row.created_at) + : (row.completed_at ?? row.updated_at ?? row.created_at), + sourcePath: managedSourcePath(row.source_meta_json), project: row.project, harness: row.harness, capturedAt: row.completed_at ?? row.updated_at ?? row.created_at, @@ -429,6 +455,14 @@ export function readRecentEpisodicSources( const transcriptCompletedAt = transcriptHasCompleted ? "st.completed_at" : "NULL"; const transcriptTime = `COALESCE(${transcriptCompletedAt}, ${transcriptUpdatedAt}, st.created_at)`; const transcriptCompleted = transcriptHasCompleted ? "st.completed_at IS NOT NULL" : "0"; + const transcriptSourceId = tableHasColumn(db, "session_transcripts", "source_id") ? "st.source_id" : "NULL"; + const transcriptSourceRecordId = tableHasColumn(db, "session_transcripts", "source_record_id") + ? "st.source_record_id" + : "NULL"; + const transcriptSourceMeta = tableHasColumn(db, "session_transcripts", "source_meta_json") + ? "st.source_meta_json" + : "NULL"; + const transcriptContentHash = tableHasColumn(db, "session_transcripts", "content_hash") ? "st.content_hash" : "NULL"; const transcriptCursor = cursorPredicate(transcriptTime, "session_key", "transcript", newer, cursor); const summaryCursor = cursorPredicate("latest_at", "id", "summary", newer, cursor); const allowedKinds = kinds ? new Set(kinds) : null; @@ -585,7 +619,9 @@ export function readRecentEpisodicSources( .prepare( `SELECT st.session_key, st.content, st.harness, st.project, st.created_at, ${transcriptUpdatedAt} AS updated_at, ${transcriptCompletedAt} AS completed_at, - ${transcriptTime} AS captured_at, ${transcriptCompleted} AS completed + ${transcriptTime} AS captured_at, ${transcriptCompleted} AS completed, + ${transcriptSourceId} AS source_id, ${transcriptSourceRecordId} AS source_record_id, + ${transcriptSourceMeta} AS source_meta_json, ${transcriptContentHash} AS content_hash FROM session_transcripts AS st WHERE st.agent_id = ? AND ${transcriptCompleted} @@ -605,15 +641,22 @@ export function readRecentEpisodicSources( readonly completed_at: string | null; readonly captured_at: string; readonly completed: number; + readonly source_id: string | null; + readonly source_record_id: string | null; + readonly source_meta_json: string | null; + readonly content_hash: string | null; }; return { kind: "transcript", id: transcript.session_key, content: transcript.content, sourceKind: "transcript", - sourceId: transcript.session_key, - sourceEntryId: null, - sourcePath: null, + sourceId: transcript.source_record_id ?? transcript.session_key, + sourceEntryId: transcript.source_id, + sourceRevision: transcript.source_id + ? (transcript.content_hash ?? transcript.captured_at) + : transcript.captured_at, + sourcePath: managedSourcePath(transcript.source_meta_json), project: transcript.project, harness: transcript.harness, capturedAt: diff --git a/platform/daemon/src/pipeline/dreaming-evidence-consumption.ts b/platform/daemon/src/pipeline/dreaming-evidence-consumption.ts index fb60db146e..b5292cb004 100644 --- a/platform/daemon/src/pipeline/dreaming-evidence-consumption.ts +++ b/platform/daemon/src/pipeline/dreaming-evidence-consumption.ts @@ -264,6 +264,8 @@ export function pendingDreamingEvidenceContinuations( const boundedLimit = Math.min(Math.max(Math.floor(limit), 1), 50); const transcriptUpdatedAt = tableHasColumn(db, "session_transcripts", "updated_at") ? "st.updated_at" : "NULL"; const transcriptCompletedAt = tableHasColumn(db, "session_transcripts", "completed_at") ? "st.completed_at" : "NULL"; + const transcriptSourceId = tableHasColumn(db, "session_transcripts", "source_id") ? "st.source_id" : "NULL"; + const transcriptContentHash = tableHasColumn(db, "session_transcripts", "content_hash") ? "st.content_hash" : "NULL"; const transcriptRevision = `COALESCE(${transcriptCompletedAt}, ${transcriptUpdatedAt}, st.created_at)`; const reviewedPredicate = tableExists(db, "dreaming_evidence_reviews") ? `AND NOT EXISTS ( @@ -311,7 +313,8 @@ export function pendingDreamingEvidenceContinuations( SELECT 1 FROM session_transcripts st WHERE st.agent_id = dec.agent_id AND st.session_key = dec.source_id AND dec.source_captured_at = ${transcriptRevision} - AND dec.source_entry_id = '' AND dec.source_revision = ${transcriptRevision} + AND dec.source_entry_id = COALESCE(${transcriptSourceId}, '') + AND dec.source_revision = CASE WHEN ${transcriptSourceId} IS NULL OR ${transcriptSourceId} = '' THEN ${transcriptRevision} ELSE COALESCE(${transcriptContentHash}, ${transcriptRevision}) END )) OR (dec.source_kind = 'summary' AND EXISTS ( SELECT 1 FROM session_summaries ss @@ -345,57 +348,61 @@ export function pendingDreamingEvidenceContinuations( }); } -/** Narrow truthful completion query for one configured source. */ -export function sourceHasEligibleUnconsumedEvidence( +/** Count current, eligible source-owned evidence using the same delivery frontier and terminal review rules as Dreaming scans. */ +export function countEligibleUnconsumedEvidenceForSource( db: ReadDb, agentId: string, sourceEntryId: string, - legacyObsidianRoot?: string, -): boolean { - if (!tableExists(db, "dreaming_evidence_consumption")) return true; - const legacyRootPrefix = legacyObsidianRoot?.replace(/\\/g, "/").replace(/\/$/, ""); - const rows = db - .prepare( - `SELECT ma.source_path FROM memory_artifacts ma - WHERE ma.agent_id = ? AND COALESCE(ma.is_deleted, 0) = 0 - AND length(ma.content) > 0 - AND ( - ma.source_id = ? - OR ( - ? IS NOT NULL AND ma.harness = 'obsidian' AND ma.source_id IS NULL - AND ma.source_path >= ? AND ma.source_path < ? - ) - ) - AND (ma.source_sha256 IS NULL OR ma.source_sha256 = '' - OR (ma.agent_id, ma.source_path) = ( - SELECT ma2.agent_id, ma2.source_path FROM memory_artifacts ma2 - WHERE ma2.agent_id = ma.agent_id AND COALESCE(ma2.is_deleted, 0) = 0 - AND ma2.source_sha256 = ma.source_sha256 - AND COALESCE(ma2.source_id, '') = COALESCE(ma.source_id, '') - ORDER BY ma2.captured_at DESC, ma2.source_path ASC - LIMIT 1 - )) - ORDER BY ma.source_path ASC`, - ) - .all( - agentId, - sourceEntryId, - legacyRootPrefix ?? null, - legacyRootPrefix ?? "", - `${legacyRootPrefix ?? ""}/\uffff`, - ) as Array<{ source_path: string }>; - return rows.some((row) => { - const source = readEpisodicSource(db, { agentId, from: `artifact:${row.source_path}` }); - if (source === null) return false; + _legacyObsidianRoot?: string, +): number { + if (!tableExists(db, "dreaming_evidence_consumption")) return 1; + const legacyRootPrefix = _legacyObsidianRoot?.replace(/\\/g, "/").replace(/\/$/, "") ?? null; + const candidates: Array<{ kind: EpisodicSourceKind; id: string }> = [ + ...( + db + .prepare( + `SELECT source_path AS id FROM memory_artifacts + WHERE agent_id = ? AND COALESCE(is_deleted, 0) = 0 AND length(content) > 0 + AND (source_id = ? OR (? IS NOT NULL AND harness = 'obsidian' AND source_id IS NULL AND source_path >= ? AND source_path < ?))`, + ) + .all( + agentId, + sourceEntryId, + legacyRootPrefix, + legacyRootPrefix ?? "", + `${legacyRootPrefix ?? ""}/\uffff`, + ) as Array<{ id: string }> + ).map((row) => ({ kind: "artifact" as const, id: row.id })), + ...( + db + .prepare( + "SELECT session_key AS id FROM session_transcripts WHERE agent_id = ? AND source_id = ? AND completed_at IS NOT NULL", + ) + .all(agentId, sourceEntryId) as Array<{ id: string }> + ).map((row) => ({ kind: "transcript" as const, id: row.id })), + ]; + return candidates.reduce((count, candidate) => { + const source = readEpisodicSource(db, { agentId, from: `${candidate.kind}:${candidate.id}` }); + if (source === null) return count; const reviewed = tableExists(db, "dreaming_evidence_reviews") && db .prepare( - `SELECT 1 FROM dreaming_evidence_reviews - WHERE agent_id = ? AND source_kind = 'artifact' AND source_id = ? - AND source_captured_at = ? AND source_entry_id = ? AND source_revision = ?`, + `SELECT 1 FROM dreaming_evidence_reviews WHERE agent_id = ? AND source_kind = ? AND source_id = ? AND source_captured_at = ? AND source_entry_id = ? AND source_revision = ?`, ) - .get(agentId, source.id, source.capturedAt, sourceIdentity(source), sourceRevision(source)) != null; - return !reviewed && deliveredOffsetForSource(db, agentId, source) < renderDreamingEvidence(source).length; - }); + .get(agentId, source.kind, source.id, source.capturedAt, sourceIdentity(source), sourceRevision(source)) != + null; + return reviewed || deliveredOffsetForSource(db, agentId, source) >= renderDreamingEvidence(source).length + ? count + : count + 1; + }, 0); +} + +export function sourceHasEligibleUnconsumedEvidence( + db: ReadDb, + agentId: string, + sourceEntryId: string, + legacyObsidianRoot?: string, +): boolean { + return countEligibleUnconsumedEvidenceForSource(db, agentId, sourceEntryId, legacyObsidianRoot) > 0; } diff --git a/platform/daemon/src/routes/sources-routes.ts b/platform/daemon/src/routes/sources-routes.ts index 807d275825..2c78d4844f 100644 --- a/platform/daemon/src/routes/sources-routes.ts +++ b/platform/daemon/src/routes/sources-routes.ts @@ -63,6 +63,7 @@ import { } from "../source-lifecycle-telemetry"; import { getSourceProvider } from "../source-providers"; import { exportSourceSnapshot, importSourceSnapshot } from "../source-snapshots"; +import { purgeSourceOwnedRows } from "../source-purge"; interface SourceIndexJobInput { readonly source: SignetSourceEntry; @@ -479,7 +480,9 @@ export function registerSourcesRoutes(app: Hono, deps: RegisterSourcesRoutesDeps recordSourceDeletionTombstone(source, sourceAgentId, agentsDir); await removeSourceLifecycleState(source, sourceAgentId); const provider = getSourceProvider(source.kind); - const purged = provider ? await purgeSource(provider, source, sourceAgentId, purgeNativeSource) : 0; + const purged = + (await purgeSourceOwnedRows({ sourceId: source.id, agentId: sourceAgentId })) + + (provider ? await purgeSource(provider, source, sourceAgentId, purgeNativeSource) : 0); const result = removeSourceIfGeneration(sourceId, source.generation, agentsDir); if (result.ok === false) return c.json({ error: result.error }, 500); if (!isSourceIndexInFlight(source.id)) clearSourceDeletionTombstone(source, sourceAgentId, agentsDir); diff --git a/platform/daemon/src/routes/transcript-import-routes.test.ts b/platform/daemon/src/routes/transcript-import-routes.test.ts new file mode 100644 index 0000000000..fdeec34076 --- /dev/null +++ b/platform/daemon/src/routes/transcript-import-routes.test.ts @@ -0,0 +1,112 @@ +import { expect, it, beforeEach, afterEach } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Hono } from "hono"; +import { closeDbAccessor, getDbAccessor, initDbAccessor } from "../db-accessor"; +import { registerTranscriptImportRoutes } from "./transcript-import-routes"; + +let dir = ""; +let oldPath: string | undefined; +let oldAgent: string | undefined; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "signet-transcript-routes-")); + mkdirSync(join(dir, "memory"), { recursive: true }); + oldPath = process.env.SIGNET_PATH; + oldAgent = process.env.SIGNET_AGENT_ID; + process.env.SIGNET_PATH = dir; + process.env.SIGNET_AGENT_ID = "transcript-test-agent"; + closeDbAccessor(); + initDbAccessor(join(dir, "memory", "memories.db")); +}); + +afterEach(() => { + closeDbAccessor(); + if (oldPath === undefined) Reflect.deleteProperty(process.env, "SIGNET_PATH"); + else process.env.SIGNET_PATH = oldPath; + if (oldAgent === undefined) Reflect.deleteProperty(process.env, "SIGNET_AGENT_ID"); + else process.env.SIGNET_AGENT_ID = oldAgent; + rmSync(dir, { recursive: true, force: true }); +}); + +function app(): Hono { + const instance = new Hono(); + registerTranscriptImportRoutes(instance); + return instance; +} + +it("serializes concurrent uploads for one reserved file without leaving a reservation orphan", async () => { + const created = await app().request("/api/sources/imports", { + method: "POST", + body: JSON.stringify({ files: [{ name: "conversation.jsonl" }] }), + headers: { "content-type": "application/json" }, + }); + expect(created.status).toBe(201); + const job = (await created.json()) as { jobId: string; files: Array<{ id: string }> }; + const fileId = job.files[0]?.id as string; + const payload = + '{"source":"signet","id":"one","harness":"h","agent_id":"embedded","session_key":"one","timestamp":"2024-01-01T00:00:00Z","message_count":1,"messages":[{"role":"user","content":"hello"}]}\n'; + const slow = new ReadableStream({ + async start(controller) { + await new Promise((resolve) => setTimeout(resolve, 40)); + controller.enqueue(new TextEncoder().encode(payload)); + controller.close(); + }, + }); + const first = app().request(`/api/sources/imports/${job.jobId}/files/${fileId}`, { + method: "PUT", + body: slow, + } as RequestInit); + await new Promise((resolve) => setTimeout(resolve, 5)); + const second = await app().request(`/api/sources/imports/${job.jobId}/files/${fileId}`, { + method: "PUT", + body: payload, + } as RequestInit); + const firstResponse = await first; + expect([201, 409]).toContain(firstResponse.status); + expect([201, 409]).toContain(second.status); + expect(new Set([firstResponse.status, second.status])).toEqual(new Set([201, 409])); + const rows = getDbAccessor().withReadDb( + (db) => + db.prepare("SELECT source_id, managed_path, state FROM source_import_files WHERE id = ?").all(fileId) as Array<{ + source_id: string; + managed_path: string; + state: string; + }>, + ); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ state: "ready" }); + expect(rows[0]?.source_id).not.toStartWith("reserved:"); + expect(rows[0]?.managed_path).toContain(`imports/transcripts/`); + expect(rows[0]?.managed_path).toContain(fileId); +}); + +it("retry increments generation and returns recoverable rejected records to pending", async () => { + const created = await app().request("/api/sources/imports", { + method: "POST", + body: JSON.stringify({ files: [{ name: "conversation.jsonl" }] }), + headers: { "content-type": "application/json" }, + }); + const job = (await created.json()) as { jobId: string; files: Array<{ id: string }> }; + const fileId = job.files[0]?.id as string; + await getDbAccessor().withWriteTxAsync((db) => { + db.prepare("UPDATE source_import_jobs SET state = 'running', lease_token = 'old-lease' WHERE id = ?").run( + job.jobId, + ); + db.prepare("UPDATE source_import_files SET state = 'completed' WHERE id = ?").run(fileId); + db.prepare( + "INSERT INTO source_import_records (id, job_id, file_id, source_id, agent_id, ordinal, line_number, byte_offset, byte_length, raw_hash, status, rejection_code) VALUES ('retry-record', ?, ?, 'source', 'transcript-test-agent', 1, 1, 0, 1, 'hash', 'rejected', 'provider_error')", + ).run(job.jobId, fileId); + }); + const response = await app().request(`/api/sources/imports/${job.jobId}/retry`, { method: "POST" }); + expect(response.status).toBe(200); + const row = await getDbAccessor().withReadDbAsync((db) => + db.prepare("SELECT generation, lease_token, control_request FROM source_import_jobs WHERE id = ?").get(job.jobId), + ); + const record = await getDbAccessor().withReadDbAsync((db) => + db.prepare("SELECT status, rejection_code FROM source_import_records WHERE id = 'retry-record'").get(), + ); + expect(row).toEqual({ generation: 1, lease_token: null, control_request: null }); + expect(record).toEqual({ status: "pending", rejection_code: null }); +}); diff --git a/platform/daemon/src/routes/transcript-import-routes.ts b/platform/daemon/src/routes/transcript-import-routes.ts new file mode 100644 index 0000000000..44e9bc95cd --- /dev/null +++ b/platform/daemon/src/routes/transcript-import-routes.ts @@ -0,0 +1,283 @@ +import { randomUUID } from "node:crypto"; +import { addImportedSource } from "@signet/core"; +import type { Context, Hono } from "hono"; +import { resolveDaemonAgentId } from "../agent-id"; +import { authConfig } from "./state"; +import { requirePermission } from "../auth"; +import { dbOwnerQuery, dbOwnerTransaction } from "../db-owner-runtime"; +import { stageTranscriptStream } from "../transcript-import-staging"; +import type { StagedTranscriptFile } from "../transcript-import-staging"; + +const MAX_FILE_BYTES = 512 * 1024 * 1024; +const now = (): string => new Date().toISOString(); +const uploadInFlight = new Map>(); +function agent(c: Context): string | null { + const resolved = resolveDaemonAgentId(); + const requested = c.req.query("agentId") ?? c.req.query("agent_id"); + return requested === undefined || requested === resolved ? resolved : null; +} +function bodyStream(request: Request): AsyncIterable { + // Bun's server Request clone does not preserve the streamed upload body here: + // the clone's reader reaches EOF even though the original body is non-empty. + // Consume the one-shot server stream directly; staging remains byte-streamed. + const body = request.body; + if (body === null) throw new Error("request body is required"); + return (async function* (): AsyncGenerator { + const reader = body.getReader(); + if (reader === undefined) throw new Error("request body is required"); + let total = 0; + try { + while (true) { + const item = await reader.read(); + if (item.done) return; + total += item.value.byteLength; + if (total > MAX_FILE_BYTES) throw new Error("file exceeds limit"); + yield item.value; + } + } finally { + reader.releaseLock(); + } + })(); +} +function permission(name: "modify" | "recall" | "admin") { + return requirePermission(name, authConfig); +} + +export function registerTranscriptImportRoutes(app: Hono): void { + app.use("/api/sources/imports", permission("modify")); + app.use("/api/sources/imports/*", permission("modify")); + app.post("/api/sources/imports", async (c) => { + const agentId = agent(c); + if (agentId === null) return c.json({ error: "agent scope denied" }, 403); + const input = (await c.req.json().catch(() => ({}))) as { schemaId?: string; files?: readonly { name?: string }[] }; + if (input.schemaId !== undefined && input.schemaId !== "signet-export") + return c.json({ error: "unsupported schema" }, 400); + const jobId = randomUUID(); + const requestedFiles = (input.files ?? []).map((file) => ({ + id: randomUUID(), + name: file.name?.trim() || "source.jsonl", + })); + const statements = [ + { + sql: "INSERT INTO source_import_jobs (id,kind,agent_id,schema_id,adapter_version,state,generation,created_at,updated_at) VALUES (?,?,?,?,1,'staging',0,?,?)", + params: [jobId, "import", agentId, "signet-export", now(), now()], + result: "run" as const, + }, + ...requestedFiles.map((file, ordinal) => ({ + sql: "INSERT INTO source_import_files (id,job_id,source_id,agent_id,ordinal,name,managed_path,state) VALUES (?,?,?,?,?,?,?,'staging')", + params: [ + file.id, + jobId, + `reserved:${jobId}:${file.id}`, + agentId, + ordinal, + file.name, + `imports/transcripts/${jobId}/${file.id}/source.jsonl`, + ], + result: "run" as const, + })), + ]; + await dbOwnerTransaction(statements, { operation: "sources.import.create", lane: "write" }); + return c.json({ id: jobId, jobId, agentId, state: "staging", files: requestedFiles }, 201); + }); + app.get("/api/sources/imports", async (c) => { + const agentId = agent(c); + if (agentId === null) return c.json({ error: "agent scope denied" }, 403); + const rows = await dbOwnerQuery( + { + sql: "SELECT * FROM source_import_jobs WHERE agent_id = ? ORDER BY created_at DESC LIMIT 100", + params: [agentId], + result: "all", + readonly: true, + }, + { operation: "sources.import.list", lane: "read" }, + ); + return c.json({ imports: rows }); + }); + app.get("/api/sources/imports/:jobId", async (c) => { + const agentId = agent(c); + if (agentId === null) return c.json({ error: "agent scope denied" }, 403); + const jobId = c.req.param("jobId"); + const job = await dbOwnerQuery( + { + sql: "SELECT * FROM source_import_jobs WHERE id = ? AND agent_id = ?", + params: [jobId, agentId], + result: "get", + readonly: true, + }, + { operation: "sources.import.status", lane: "read" }, + ); + if (job == null) return c.json({ error: "import not found" }, 404); + const files = await dbOwnerQuery( + { + sql: "SELECT * FROM source_import_files WHERE job_id = ? AND agent_id = ? ORDER BY ordinal", + params: [jobId, agentId], + result: "all", + readonly: true, + }, + { operation: "sources.import.files", lane: "read" }, + ); + return c.json({ job, files }); + }); + app.put("/api/sources/imports/:jobId/files/:fileId", async (c) => { + const agentId = agent(c); + if (agentId === null) return c.json({ error: "agent scope denied" }, 403); + const jobId = c.req.param("jobId"), + fileId = c.req.param("fileId"); + const job = await dbOwnerQuery<{ id: string }>( + { + sql: "SELECT id FROM source_import_jobs WHERE id = ? AND agent_id = ? AND state = 'staging'", + params: [jobId, agentId], + result: "get", + readonly: true, + }, + { operation: "sources.import.upload.scope", lane: "read" }, + ); + if (job == null) return c.json({ error: "import not found or not staging" }, 404); + const file = await dbOwnerQuery<{ id: string; name: string; ordinal: number; state: string }>( + { + sql: "SELECT id,name,ordinal,state FROM source_import_files WHERE id = ? AND job_id = ? AND agent_id = ? AND state = 'staging'", + params: [fileId, jobId, agentId], + result: "get", + readonly: true, + }, + { operation: "sources.import.upload.file", lane: "read" }, + ); + if (file == null) return c.json({ error: "file not found or already uploaded" }, 404); + const previous = uploadInFlight.get(fileId) ?? Promise.resolve(); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + uploadInFlight.set(fileId, gate); + await previous; + try { + const stillStaging = await dbOwnerQuery<{ id: string }>( + { + sql: "SELECT id FROM source_import_files WHERE id = ? AND job_id = ? AND agent_id = ? AND state = 'staging'", + params: [fileId, jobId, agentId], + result: "get", + readonly: true, + }, + { operation: "sources.import.upload.claim", lane: "read" }, + ); + if (stillStaging == null) return c.json({ error: "file not found or already uploaded" }, 409); + const sourceId = `import:${jobId}:${fileId}`; + let staged: StagedTranscriptFile; + try { + staged = await stageTranscriptStream( + process.env.SIGNET_PATH ?? `${process.env.HOME}/.agents`, + `${sourceId.replaceAll(":", "-")}-${jobId}-${fileId}`, + bodyStream(c.req.raw), + ); + } catch (error) { + return c.json({ error: error instanceof Error ? error.message : "upload failed" }, 413); + } + const added = addImportedSource( + { fileName: "source.jsonl", contentHash: staged.contentHash, format: "jsonl", agentId }, + process.env.SIGNET_PATH, + ); + if (!added.ok) return c.json({ error: added.error }, 400); + await dbOwnerTransaction( + [ + { + sql: "UPDATE source_import_files SET source_id = ?, name = ?, managed_path = ?, size_bytes = ?, content_hash = ?, state = 'ready', updated_at = ? WHERE id = ? AND job_id = ? AND agent_id = ? AND state = 'staging'", + params: [ + added.source.id, + c.req.header("x-file-name") ?? file.name, + staged.managedPath, + staged.sizeBytes, + staged.contentHash, + now(), + fileId, + jobId, + agentId, + ], + result: "run" as const, + requireChanges: true, + }, + { + sql: "UPDATE source_import_jobs SET updated_at = ? WHERE id = ? AND agent_id = ?", + params: [now(), jobId, agentId], + result: "run" as const, + requireChanges: true, + }, + ], + { operation: "sources.import.upload", lane: "write" }, + ); + return c.json( + { + fileId, + sourceId: added.source.id, + managedPath: staged.managedPath, + sizeBytes: staged.sizeBytes, + contentHash: staged.contentHash, + }, + 201, + ); + } finally { + release(); + if (uploadInFlight.get(fileId) === gate) uploadInFlight.delete(fileId); + } + }); + for (const control of ["start", "pause", "resume", "retry", "cancel"] as const) { + app.post(`/api/sources/imports/:jobId/${control}`, async (c) => { + const agentId = agent(c); + if (agentId === null) return c.json({ error: "agent scope denied" }, 403); + const jobId = c.req.param("jobId"); + const transition = control === "start" ? "queued" : control; + const result = await dbOwnerTransaction( + [ + { + sql: "UPDATE source_import_jobs SET state = CASE WHEN ? = 'queued' THEN 'queued' ELSE state END, control_request = CASE WHEN ? = 'queued' THEN NULL WHEN ? = 'retry' THEN NULL ELSE ? END, generation = CASE WHEN ? = 'retry' THEN generation + 1 ELSE generation END, lease_token = CASE WHEN ? = 'retry' THEN NULL ELSE lease_token END, lease_expires_at = CASE WHEN ? = 'retry' THEN NULL ELSE lease_expires_at END, updated_at = ? WHERE id = ? AND agent_id = ? AND state NOT IN ('completed','completed_with_rejections','cancelled')", + params: [ + transition, + transition, + transition, + transition, + transition, + transition, + transition, + now(), + jobId, + agentId, + ], + result: "run" as const, + }, + ...(control === "retry" + ? [ + { + sql: "UPDATE source_import_records SET status = 'pending', rejection_code = NULL, updated_at = datetime('now') WHERE job_id = ? AND agent_id = ? AND status = 'rejected' AND rejection_code NOT IN ('schema_invalid','malformed')", + params: [jobId, agentId], + result: "run" as const, + }, + ] + : []), + ], + { operation: `sources.import.${control}`, lane: "write" }, + ); + const first = result[0] as { readonly changes?: number } | undefined; + return c.json({ jobId, control, changed: (first?.changes ?? 0) > 0 }); + }); + } + for (const suffix of ["rejections", "reconciliation"] as const) { + app.get(`/api/sources/imports/:jobId/${suffix}`, async (c) => { + const agentId = agent(c); + if (agentId === null) return c.json({ error: "agent scope denied" }, 403); + const jobId = c.req.param("jobId"); + const rows = await dbOwnerQuery( + { + sql: + suffix === "rejections" + ? "SELECT * FROM source_import_records WHERE job_id = ? AND agent_id = ? AND status = 'rejected' ORDER BY ordinal" + : "SELECT status, COUNT(*) AS count FROM source_import_records WHERE job_id = ? AND agent_id = ? GROUP BY status", + params: [jobId, agentId], + result: "all", + readonly: true, + }, + { operation: `sources.import.${suffix}`, lane: "read" }, + ); + return c.json({ jobId, [suffix]: rows }); + }); + } +} diff --git a/platform/daemon/src/source-artifact-graph.test.ts b/platform/daemon/src/source-artifact-graph.test.ts index 91355b3be5..48f60784bb 100644 --- a/platform/daemon/src/source-artifact-graph.test.ts +++ b/platform/daemon/src/source-artifact-graph.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it } from "bun:test"; -import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { DreamingConfig } from "@signet/core"; @@ -13,6 +13,12 @@ import { import { purgeSourceOwnedRows } from "./source-purge"; import { txIngestEnvelope } from "./transactions"; +it("routes transcript source purge through the DB owner boundary", () => { + const source = readFileSync(new URL("./source-purge.ts", import.meta.url), "utf8"); + expect(source).toContain("dbOwnerTranscriptImportPurge"); + expect(source).not.toContain("withReadDbAsync"); +}); + const DREAMING_CONFIG: DreamingConfig = { tokenThreshold: 1, maxInterval: 6 * 60 * 60 * 1_000, diff --git a/platform/daemon/src/source-purge.ts b/platform/daemon/src/source-purge.ts index 724ceb0305..b77c8c1434 100644 --- a/platform/daemon/src/source-purge.ts +++ b/platform/daemon/src/source-purge.ts @@ -1,7 +1,9 @@ import { SOURCE_CHUNK_SOURCE_TYPE } from "@signet/core"; import { getDbAccessor, runWriteTxAsync, type WriteDb } from "./db-accessor"; import { countChanges, syncVecDeleteByEmbeddingIds, tableExists } from "./db-helpers"; +import { dbOwnerQuery, dbOwnerTranscriptImportPurge, runDbOwnerDomainOperation } from "./db-owner-runtime"; import { reconcileOntologyContradictionsInTx } from "./ontology-contradictions"; +import { purgeTranscriptImportFilesystem } from "./transcript-import-worker"; import { purgeAttributeMemoryProjectionsInTx } from "./semantic-memory-projection"; interface PurgeSourceOwnedRowsInput { @@ -17,6 +19,54 @@ const SOURCE_OWNED_GRAPH_TABLES = [ ] as const; export async function purgeSourceOwnedRows(input: PurgeSourceOwnedRowsInput): Promise { + const sourceId = input.sourceId.trim(); + if (!sourceId) return 0; + let managedPaths: string[] = []; + try { + managedPaths = await runDbOwnerDomainOperation(getDbAccessor(), { + runWithOwner: async () => + ( + await dbOwnerQuery>( + { + sql: `SELECT managed_path FROM source_import_files WHERE ${input.agentId ? "agent_id = ? AND " : ""}source_id = ?`, + params: input.agentId ? [input.agentId, sourceId] : [sourceId], + result: "all", + readonly: true, + }, + { operation: "sources.import.purge-paths", lane: "read" }, + ) + ).map((row) => row.managed_path), + runInline: ({ read }) => + read((db) => + ( + db + .prepare( + `SELECT managed_path FROM source_import_files WHERE ${input.agentId ? "agent_id = ? AND " : ""}source_id = ?`, + ) + .all(...(input.agentId ? [input.agentId] : []), sourceId) as Array<{ managed_path: string }> + ).map((row) => row.managed_path), + ), + }); + } catch { + // Older databases do not have import ledgers. + } + await purgeTranscriptImportFilesystem( + process.env.SIGNET_PATH ?? `${process.env.HOME ?? "."}/.agents`, + sourceId, + input.agentId, + managedPaths, + ); + await runDbOwnerDomainOperation(getDbAccessor(), { + runWithOwner: async () => { + await dbOwnerTranscriptImportPurge( + { agentId: input.agentId ?? "default", sourceId }, + { operation: "sources.import.purge", lane: "write" }, + ); + }, + runInline: ({ write }) => { + write((db) => purgeSourceOwnedRowsInTx(db, input)); + }, + }); return await runWriteTxAsync(getDbAccessor(), (db) => purgeSourceOwnedRowsInTx(db, input)); } @@ -45,6 +95,7 @@ export function purgeSourceOwnedRowsInTx(db: WriteDb, input: PurgeSourceOwnedRow throw new Error("failed to reconcile vec_embeddings before source purge"); } let purged = embeddingIds.length; + if (embeddingIds.length > 0) { const stmt = db.prepare("DELETE FROM embeddings WHERE id = ?"); for (const id of embeddingIds) stmt.run(id); diff --git a/platform/daemon/src/transcript-import-adapter.test.ts b/platform/daemon/src/transcript-import-adapter.test.ts new file mode 100644 index 0000000000..0221ef4269 --- /dev/null +++ b/platform/daemon/src/transcript-import-adapter.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, test } from "bun:test"; +import { + validateSignetExport, + conversationFingerprint, + canonicalTranscriptIdentity, +} from "./transcript-import-adapter"; + +describe("signet-export-v1 adapter", () => { + const record = { + id: "r1", + source: "signet", + harness: "hermes", + agent_id: "a", + session_key: "s", + project: null, + timestamp: "2026-01-01T00:00:00.000Z", + message_count: 2, + messages: [ + { role: "user", content: " hello\nworld " }, + { role: "tool", content: "" }, + ], + }; + test("preserves lossless message content and deterministic identities", () => { + const parsed = validateSignetExport(record); + expect(parsed.messages[0]?.content).toBe(" hello\nworld "); + expect(conversationFingerprint(parsed)).toBe(conversationFingerprint(validateSignetExport({ ...record }))); + expect(canonicalTranscriptIdentity(parsed).canonicalId).toMatch(/^import:[0-9a-f]{64}$/); + }); + test("rejects unknown roles and count mismatches", () => { + expect(() => validateSignetExport({ ...record, messages: [{ role: "bogus", content: "x" }] })).toThrow(); + expect(() => validateSignetExport({ ...record, message_count: 1 })).toThrow(); + }); +}); diff --git a/platform/daemon/src/transcript-import-adapter.ts b/platform/daemon/src/transcript-import-adapter.ts new file mode 100644 index 0000000000..488272aa47 --- /dev/null +++ b/platform/daemon/src/transcript-import-adapter.ts @@ -0,0 +1,114 @@ +import { createHash } from "node:crypto"; + +export const TRANSCRIPT_IMPORT_LIMITS = { + maxRecordsPerBatch: 25, + maxCanonicalBatchBytes: 8 * 1024 * 1024, + maxRecordBytes: 16 * 1024 * 1024, + maxMessageBytes: 4 * 1024 * 1024, + maxMessages: 50_000, +} as const; + +export const TRANSCRIPT_ROLES = ["user", "assistant", "system", "tool", "unknown"] as const; +export type TranscriptRole = (typeof TRANSCRIPT_ROLES)[number]; +export interface CompletedTranscriptMessage { + readonly role: TranscriptRole; + readonly content: string; +} +export interface SignetExportRecord { + readonly id: string; + readonly source: "signet"; + readonly harness: string; + readonly agent_id: string; + readonly session_key: string; + readonly project: string | null; + readonly timestamp: string; + readonly message_count: number; + readonly messages: readonly CompletedTranscriptMessage[]; +} +export interface TranscriptAdapter { + readonly id: "signet-export"; + readonly version: 1; + parse(value: unknown): SignetExportRecord; +} + +function isRole(value: unknown): value is TranscriptRole { + return typeof value === "string" && (TRANSCRIPT_ROLES as readonly string[]).includes(value); +} +function nonempty(value: unknown): value is string { + return typeof value === "string" && value.length > 0; +} +export function validateSignetExport(value: unknown): SignetExportRecord { + if (value == null || typeof value !== "object" || Array.isArray(value)) throw new Error("record must be an object"); + const r = value as Record; + for (const field of ["id", "harness", "agent_id", "session_key"] as const) + if (!nonempty(r[field])) throw new Error(`${field} must be nonempty`); + if (r.source !== "signet") throw new Error("source must be signet"); + if (typeof r.project !== "string" && r.project !== null) throw new Error("project must be string or null"); + if (typeof r.timestamp !== "string" || Number.isNaN(Date.parse(r.timestamp))) throw new Error("timestamp is invalid"); + if (!Array.isArray(r.messages) || r.messages.length > TRANSCRIPT_IMPORT_LIMITS.maxMessages) + throw new Error("messages is invalid or oversized"); + const messages: CompletedTranscriptMessage[] = []; + for (const message of r.messages) { + if (message == null || typeof message !== "object" || Array.isArray(message)) throw new Error("message is invalid"); + const m = message as Record; + if (!isRole(m.role) || typeof m.content !== "string") throw new Error("message role/content is invalid"); + if (Buffer.byteLength(m.content, "utf8") > TRANSCRIPT_IMPORT_LIMITS.maxMessageBytes) + throw new Error("message is oversized"); + messages.push({ role: m.role, content: m.content }); + } + if (r.message_count !== messages.length || typeof r.message_count !== "number") + throw new Error("message_count mismatch"); + if (!messages.some((m) => m.content.length > 0)) throw new Error("record has no nonempty message"); + return { + id: r.id as string, + source: "signet", + harness: r.harness as string, + agent_id: r.agent_id as string, + session_key: r.session_key as string, + project: r.project as string | null, + timestamp: r.timestamp as string, + message_count: r.message_count, + messages, + }; +} +export const signetExportV1Adapter: TranscriptAdapter = { + id: "signet-export", + version: 1, + parse: validateSignetExport, +}; + +function fixed(fields: readonly string[]): string { + return fields.map((field) => `${field.length}:${field}`).join("|"); +} +export function externalIdentityFingerprint( + record: Pick, +): string { + return createHash("sha256") + .update(fixed([record.agent_id, record.harness, record.session_key])) + .digest("hex"); +} +export function conversationFingerprint(record: SignetExportRecord): string { + return createHash("sha256") + .update( + fixed([ + record.agent_id, + record.harness, + record.session_key, + record.project ?? "", + record.timestamp, + ...record.messages.flatMap((m) => [m.role, m.content]), + ]), + ) + .digest("hex"); +} +export function canonicalTranscriptIdentity(record: SignetExportRecord): { + readonly canonicalId: string; + readonly canonicalKey: string; + readonly contentHash: string; +} { + const contentHash = conversationFingerprint(record); + const digest = createHash("sha256") + .update(fixed([externalIdentityFingerprint(record), contentHash])) + .digest("hex"); + return { canonicalId: `import:${digest}`, canonicalKey: `import:${digest}`, contentHash }; +} diff --git a/platform/daemon/src/transcript-import-commit.test.ts b/platform/daemon/src/transcript-import-commit.test.ts new file mode 100644 index 0000000000..87e95f471f --- /dev/null +++ b/platform/daemon/src/transcript-import-commit.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, test } from "bun:test"; +import { + buildCompletedTranscriptCommit, + canonicalTranscriptLine, + serializeCompletedTranscriptMessages, +} from "./transcript-import-commit"; +import { signetExportV1Adapter } from "./transcript-import-adapter"; + +const record = (agent_id: string, content = " exact\n\ntext ") => + signetExportV1Adapter.parse({ + source: "signet", + id: "conversation-1", + harness: "claude", + agent_id, + session_key: "same-session", + project: null, + timestamp: "2024-01-01T00:00:00.000Z", + message_count: 3, + messages: [ + { role: "system", content: "system\nline" }, + { role: "tool", content: '{"x":1}' }, + { role: "user", content }, + ], + }); + +describe("transcript import invariants", () => { + test("lossless roles and whitespace are retained without role-prefixed text", () => { + const commit = buildCompletedTranscriptCommit(record("agent-a"), { + agentId: "agent-a", + sourceId: "source-a", + sourceRecordId: "record-a", + }); + expect(commit.messages[0]).toEqual({ role: "system", content: "system\nline" }); + expect(serializeCompletedTranscriptMessages(commit.messages)).toContain(" exact\\n\\ntext "); + expect(canonicalTranscriptLine(commit)).toContain('"role":"tool"'); + }); + test("agent scope is part of deterministic identity", () => { + const a = buildCompletedTranscriptCommit(record("agent-a"), { + agentId: "agent-a", + sourceId: "source-a", + sourceRecordId: "record-a", + }); + const b = buildCompletedTranscriptCommit(record("agent-b"), { + agentId: "agent-b", + sourceId: "source-a", + sourceRecordId: "record-a", + }); + expect(a.recordId).not.toBe(b.recordId); + expect(a.canonicalId).not.toBe(b.canonicalId); + }); + test("replay uses the same record id and canonical revision", () => { + const input = { agentId: "agent-a", sourceId: "source-a", sourceRecordId: "record-a" } as const; + expect(buildCompletedTranscriptCommit(record("agent-a"), input)).toEqual( + buildCompletedTranscriptCommit(record("agent-a"), input), + ); + }); +}); diff --git a/platform/daemon/src/transcript-import-commit.ts b/platform/daemon/src/transcript-import-commit.ts new file mode 100644 index 0000000000..5b0c7c8807 --- /dev/null +++ b/platform/daemon/src/transcript-import-commit.ts @@ -0,0 +1,200 @@ +import { createHash } from "node:crypto"; +import type { WriteDb } from "./db-accessor"; +import { + canonicalTranscriptIdentity, + externalIdentityFingerprint, + type SignetExportRecord, +} from "./transcript-import-adapter"; + +export interface CompletedTranscriptEvidence { + readonly recordId: string; + readonly sourceId: string; + readonly sourceRecordId: string; + readonly agentId: string; + readonly harness: string; + readonly project: string | null; + readonly sessionKey: string; + readonly capturedAt: string; + readonly messages: readonly { + readonly role: "user" | "assistant" | "system" | "tool" | "unknown"; + readonly content: string; + }[]; + readonly contentHash: string; + readonly sourceMetaJson: string | null; + /** Managed relative path of the staged import file, when imported. */ + readonly sourcePath: string | null; +} + +export interface CompletedTranscriptCommit extends CompletedTranscriptEvidence { + readonly externalIdentity: string; + readonly canonicalId: string; + readonly canonicalKey: string; +} + +export type TranscriptCommitOutcome = "imported" | "duplicate" | "conversation_identity_conflict"; +export interface TranscriptCommitResult { + readonly outcome: TranscriptCommitOutcome; + readonly canonicalId: string; + readonly sessionKey: string; +} + +const fixed = (fields: readonly string[]) => fields.map((field) => `${field.length}:${field}`).join("|"); + +export function buildCompletedTranscriptCommit( + record: SignetExportRecord, + input: { + readonly agentId: string; + readonly sourceId: string; + readonly sourceRecordId: string; + readonly sourceMetaJson?: string | null; + readonly sourcePath?: string | null; + }, +): CompletedTranscriptCommit { + const identity = canonicalTranscriptIdentity(record); + const externalIdentity = externalIdentityFingerprint(record); + const recordId = createHash("sha256") + .update(fixed([input.agentId, input.sourceId, input.sourceRecordId, record.harness, identity.contentHash])) + .digest("hex"); + return { + recordId, + sourceId: input.sourceId, + sourceRecordId: input.sourceRecordId, + agentId: input.agentId, + harness: record.harness, + project: record.project, + sessionKey: identity.canonicalKey, + capturedAt: record.timestamp, + messages: record.messages, + contentHash: identity.contentHash, + sourceMetaJson: input.sourceMetaJson ?? null, + sourcePath: input.sourcePath ?? null, + externalIdentity, + canonicalId: identity.canonicalId, + canonicalKey: identity.canonicalKey, + }; +} + +/** Lossless storage representation. It deliberately does not normalize whitespace or prefix roles. */ +export function serializeCompletedTranscriptMessages(messages: CompletedTranscriptEvidence["messages"]): string { + return JSON.stringify(messages); +} + +export function canonicalTranscriptLine(commit: CompletedTranscriptCommit): string { + return `${JSON.stringify({ + id: commit.recordId, + session_id: commit.canonicalId, + session_key: commit.canonicalKey, + agent_id: commit.agentId, + harness: commit.harness, + project: commit.project, + captured_at: commit.capturedAt, + source_id: commit.sourceId, + source_record_id: commit.sourceRecordId, + content_hash: commit.contentHash, + source_path: commit.sourcePath, + source_meta_json: + commit.sourceMetaJson ?? (commit.sourcePath === null ? null : JSON.stringify({ managedPath: commit.sourcePath })), + messages: commit.messages, + })}\n`; +} + +/** Insert the durable transcript exactly once when recovering a committing claim. */ +function insertSessionTranscriptIfMissing(db: WriteDb, commit: CompletedTranscriptCommit): void { + const existing = db + .prepare("SELECT 1 FROM session_transcripts WHERE session_key = ? AND agent_id = ? LIMIT 1") + .get(commit.canonicalKey, commit.agentId); + if (existing) return; + const content = serializeCompletedTranscriptMessages(commit.messages); + const columns = db.prepare("PRAGMA table_info(session_transcripts)").all() as Array<{ name: string }>; + const names = new Set(columns.map((column) => column.name)); + const insertColumns = ["session_key", "content", "harness", "project", "agent_id", "created_at"]; + const values: unknown[] = [ + commit.canonicalKey, + content, + commit.harness, + commit.project, + commit.agentId, + commit.capturedAt, + ]; + if (names.has("updated_at")) { + insertColumns.push("updated_at"); + values.push(commit.capturedAt); + } + if (names.has("completed_at")) { + insertColumns.push("completed_at"); + values.push(commit.capturedAt); + } + if (names.has("content_hash")) { + insertColumns.push("content_hash"); + values.push(commit.contentHash); + } + if (names.has("source_id")) { + insertColumns.push("source_id"); + values.push(commit.sourceId); + } + if (names.has("source_record_id")) { + insertColumns.push("source_record_id"); + values.push(commit.sourceRecordId); + } + if (names.has("source_meta_json")) { + insertColumns.push("source_meta_json"); + values.push(commit.sourceMetaJson); + } + db.prepare( + `INSERT INTO session_transcripts (${insertColumns.join(",")}) VALUES (${insertColumns.map(() => "?").join(",")})`, + ).run(...values); +} + +/** Owner-side atomic DB prepare/finalize operation. Filesystem finalization is separate and replay-safe by recordId. */ +export function commitCompletedTranscriptBatchInTx( + db: WriteDb, + commits: readonly CompletedTranscriptCommit[], +): TranscriptCommitResult[] { + const results: TranscriptCommitResult[] = []; + for (const commit of commits) { + const existing = db + .prepare( + "SELECT conversation_fingerprint, canonical_id, canonical_key, state FROM transcript_import_conversations WHERE agent_id = ? AND external_identity = ?", + ) + .get(commit.agentId, commit.externalIdentity) as + | { conversation_fingerprint: string; canonical_id: string; canonical_key: string; state: string } + | undefined; + if (existing) { + if (existing.conversation_fingerprint !== commit.contentHash) { + results.push({ + outcome: "conversation_identity_conflict", + canonicalId: existing.canonical_id, + sessionKey: existing.canonical_key, + }); + continue; + } + if (existing.state === "committing") { + insertSessionTranscriptIfMissing(db, commit); + db.prepare( + "UPDATE transcript_import_conversations SET state = 'committed', updated_at = datetime('now') WHERE agent_id = ? AND external_identity = ? AND state = 'committing'", + ).run(commit.agentId, commit.externalIdentity); + results.push({ outcome: "imported", canonicalId: existing.canonical_id, sessionKey: existing.canonical_key }); + } else { + results.push({ outcome: "duplicate", canonicalId: existing.canonical_id, sessionKey: existing.canonical_key }); + } + continue; + } + insertSessionTranscriptIfMissing(db, commit); + db.prepare(`INSERT INTO transcript_import_conversations + (agent_id, external_identity, canonical_key, conversation_fingerprint, canonical_id, owner_source_id, owner_record_id, state, content_hash, harness, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, 'committed', ?, ?, ?)`).run( + commit.agentId, + commit.externalIdentity, + commit.canonicalKey, + commit.contentHash, + commit.canonicalId, + commit.sourceId, + commit.sourceRecordId, + commit.contentHash, + commit.harness, + commit.capturedAt, + ); + results.push({ outcome: "imported", canonicalId: commit.canonicalId, sessionKey: commit.canonicalKey }); + } + return results; +} diff --git a/platform/daemon/src/transcript-import-inventory.test.ts b/platform/daemon/src/transcript-import-inventory.test.ts new file mode 100644 index 0000000000..67cc19ae9f --- /dev/null +++ b/platform/daemon/src/transcript-import-inventory.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { inventoryTranscriptFile } from "./transcript-import-inventory"; + +const line = (id: string) => + JSON.stringify({ + id, + source: "signet", + harness: "h", + agent_id: "a", + session_key: id, + project: null, + timestamp: "2026-01-01T00:00:00Z", + message_count: 1, + messages: [{ role: "user", content: ` ${id} ` }], + }); + +describe("transcript import inventory invariants", () => { + test("tracks CRLF, LF, final line, blanks and byte restart checkpoints", async () => { + const dir = await mkdtemp(join(tmpdir(), "signet-import-")); + const path = join(dir, "source.jsonl"); + await writeFile(path, `${line("one")}\r\n\n${line("two")}\nnot-json`); + try { + const batches: number[] = []; + const result = await inventoryTranscriptFile(path, undefined, 25, async (batch) => batches.push(batch.length)); + expect(result.records).toHaveLength(3); + expect(result.blankLines).toBe(1); + expect(result.malformedLines).toBe(1); + expect(result.complete).toBe(true); + expect(result.nextByteOffset).toBe(Buffer.byteLength(await Bun.file(path).text())); + expect(batches).toEqual([3]); + const checkpoint = result.records[1]; + if (checkpoint == null) throw new Error("missing checkpoint record"); + const resumed = await inventoryTranscriptFile(path, { byteOffset: checkpoint.byteOffset, ordinal: 1 }); + expect(resumed.records[0]?.value?.id).toBe("two"); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/platform/daemon/src/transcript-import-inventory.ts b/platform/daemon/src/transcript-import-inventory.ts new file mode 100644 index 0000000000..097542084d --- /dev/null +++ b/platform/daemon/src/transcript-import-inventory.ts @@ -0,0 +1,149 @@ +import { open, stat } from "node:fs/promises"; +import { TRANSCRIPT_IMPORT_LIMITS, signetExportV1Adapter, type SignetExportRecord } from "./transcript-import-adapter"; + +export type InventoryStatus = "pending" | "rejected"; +export interface InventoryRecord { + readonly ordinal: number; + readonly lineNumber: number; + readonly byteOffset: number; + readonly byteLength: number; + readonly rawHash: string; + readonly status: InventoryStatus; + readonly value?: SignetExportRecord; + readonly rejectionCode?: string; +} +export interface InventoryResult { + readonly records: readonly InventoryRecord[]; + readonly blankLines: number; + readonly malformedLines: number; + readonly nextByteOffset: number; + readonly nextOrdinal: number; + readonly bytes: number; + readonly complete: boolean; +} + +/** Scan JSONL incrementally. checkpoint is a durable byte/ordinal pair, never a line number. */ +export async function inventoryTranscriptFile( + path: string, + checkpoint: { readonly byteOffset: number; readonly ordinal: number } = { byteOffset: 0, ordinal: 0 }, + batchSize = TRANSCRIPT_IMPORT_LIMITS.maxRecordsPerBatch, + onBatch?: (records: readonly InventoryRecord[], checkpoint: { byteOffset: number; ordinal: number }) => Promise, +): Promise { + if (batchSize < 1 || batchSize > TRANSCRIPT_IMPORT_LIMITS.maxRecordsPerBatch) + throw new Error("batchSize exceeds bounded import limit"); + const handle = await open(path, "r"); + const records: InventoryRecord[] = []; + let buffer = Buffer.alloc(0); + let offset = checkpoint.byteOffset; + let ordinal = checkpoint.ordinal; + let lineNumber = checkpoint.ordinal; + let blankLines = 0; + let malformedLines = 0; + let eof = false; + let batch: InventoryRecord[] = []; + const decoder = new TextDecoder("utf-8", { fatal: true }); + try { + const info = await stat(path); + if (checkpoint.byteOffset > info.size) throw new Error("checkpoint is beyond file size"); + while (!eof) { + const chunk = Buffer.allocUnsafe(64 * 1024); + const read = await handle.read(chunk, 0, chunk.length, offset + buffer.length); + eof = read.bytesRead === 0; + if (read.bytesRead > 0) buffer = Buffer.concat([buffer, chunk.subarray(0, read.bytesRead)]); + let newline = buffer.indexOf(0x0a); + while (newline >= 0) { + const raw = buffer.subarray(0, newline); + buffer = buffer.subarray(newline + 1); + const start = offset; + offset += newline + 1; + lineNumber++; + const line = raw.length > 0 && raw[raw.length - 1] === 0x0d ? raw.subarray(0, raw.length - 1) : raw; + if (line.length === 0 || /^\s*$/.test(line.toString("utf8"))) { + blankLines++; + newline = buffer.indexOf(0x0a); + continue; + } + ordinal++; + const rawHash = await sha256(raw); + let item: InventoryRecord; + try { + if (raw.length > TRANSCRIPT_IMPORT_LIMITS.maxRecordBytes) throw new Error("oversized_record"); + const value = signetExportV1Adapter.parse(JSON.parse(decoder.decode(line))); + item = { ordinal, lineNumber, byteOffset: start, byteLength: newline + 1, rawHash, status: "pending", value }; + } catch (error) { + malformedLines++; + item = { + ordinal, + lineNumber, + byteOffset: start, + byteLength: newline + 1, + rawHash, + status: "rejected", + rejectionCode: error instanceof Error ? error.message : "malformed_record", + }; + } + records.push(item); + batch.push(item); + if (batch.length === batchSize) { + await onBatch?.(batch, { byteOffset: offset, ordinal }); + batch = []; + } + newline = buffer.indexOf(0x0a); + } + if (eof && buffer.length > 0) { + const raw = buffer; + buffer = Buffer.alloc(0); + const start = offset; + offset += raw.length; + lineNumber++; + if (!/^\s*$/.test(raw.toString("utf8"))) { + ordinal++; + const rawHash = await sha256(raw); + let item: InventoryRecord; + try { + if (raw.length > TRANSCRIPT_IMPORT_LIMITS.maxRecordBytes) throw new Error("oversized_record"); + const value = signetExportV1Adapter.parse(JSON.parse(decoder.decode(raw))); + item = { + ordinal, + lineNumber, + byteOffset: start, + byteLength: raw.length, + rawHash, + status: "pending", + value, + }; + } catch (error) { + malformedLines++; + item = { + ordinal, + lineNumber, + byteOffset: start, + byteLength: raw.length, + rawHash, + status: "rejected", + rejectionCode: error instanceof Error ? error.message : "malformed_record", + }; + } + records.push(item); + batch.push(item); + } else blankLines++; + } + } + if (batch.length) await onBatch?.(batch, { byteOffset: offset, ordinal }); + return { + records, + blankLines, + malformedLines, + nextByteOffset: offset, + nextOrdinal: ordinal, + bytes: info.size, + complete: offset === info.size, + }; + } finally { + await handle.close(); + } +} +async function sha256(value: Uint8Array): Promise { + const { createHash } = await import("node:crypto"); + return createHash("sha256").update(value).digest("hex"); +} diff --git a/platform/daemon/src/transcript-import-replay-regression.test.ts b/platform/daemon/src/transcript-import-replay-regression.test.ts new file mode 100644 index 0000000000..07c21b4ac1 --- /dev/null +++ b/platform/daemon/src/transcript-import-replay-regression.test.ts @@ -0,0 +1,43 @@ +import { expect, test } from "bun:test"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { initDbAccessor, closeDbAccessor, getDbAccessor } from "./db-accessor"; +import { createJob, createOwnerTranscriptImportStore } from "./transcript-import-store"; + +test("replay keeps one configured source identity while allowing a second file slot", async () => { + const root = await mkdtemp(join(tmpdir(), "signet-import-replay-schema-")); + const oldOwner = process.env.SIGNET_DB_OWNER_WORKER; + process.env.SIGNET_DB_OWNER_WORKER = "1"; + try { + await mkdir(join(root, "memory"), { recursive: true }); + closeDbAccessor(); + initDbAccessor(join(root, "memory", "memories.db"), { agentsDir: root }); + const store = createOwnerTranscriptImportStore(); + await createJob(store, { jobId: "job-a", agentId: "agent-a" }); + await createJob(store, { jobId: "job-b", agentId: "agent-a" }); + await getDbAccessor().withWriteTxAsync((db) => { + for (const job of ["job-a", "job-b"]) + db.prepare("UPDATE source_import_jobs SET state = 'queued' WHERE id = ?").run(job); + for (const [job, file] of [ + ["job-a", "file-a"], + ["job-b", "file-b"], + ]) + db.prepare( + "INSERT INTO source_import_files (id,job_id,source_id,agent_id,ordinal,name,managed_path,state) VALUES (?,?,?,?,0,?,?, 'ready')", + ).run(file, job, "import:same-source", "agent-a", "source.jsonl", `imports/transcripts/${file}/source.jsonl`); + }); + const rows = await getDbAccessor().withReadDbAsync( + (db) => + db + .prepare("SELECT source_id, COUNT(*) AS count FROM source_import_files WHERE agent_id = ? GROUP BY source_id") + .all("agent-a") as Array<{ source_id: string; count: number }>, + ); + expect(rows).toEqual([{ source_id: "import:same-source", count: 2 }]); + } finally { + closeDbAccessor(); + if (oldOwner === undefined) Reflect.deleteProperty(process.env, "SIGNET_DB_OWNER_WORKER"); + else process.env.SIGNET_DB_OWNER_WORKER = oldOwner; + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/platform/daemon/src/transcript-import-staging.test.ts b/platform/daemon/src/transcript-import-staging.test.ts new file mode 100644 index 0000000000..2c15a065c3 --- /dev/null +++ b/platform/daemon/src/transcript-import-staging.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtemp, readFile, readdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { stageTranscriptStream } from "./transcript-import-staging"; + +describe("managed transcript staging", () => { + test("publishes only after streaming, fsync and atomic rename", async () => { + const root = await mkdtemp(join(tmpdir(), "signet-stage-")); + try { + const result = await stageTranscriptStream( + root, + "source-1", + (async function* () { + yield new TextEncoder().encode("one\n"); + yield new TextEncoder().encode("two\n"); + })(), + ); + expect(result.managedPath).toBe("imports/transcripts/source-1/source.jsonl"); + expect(result.sizeBytes).toBe(8); + expect(await readFile(join(root, result.managedPath), "utf8")).toBe("one\ntwo\n"); + expect(await readdir(join(root, "imports/transcripts/source-1"))).toEqual(["source.jsonl"]); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); diff --git a/platform/daemon/src/transcript-import-staging.ts b/platform/daemon/src/transcript-import-staging.ts new file mode 100644 index 0000000000..be153dcb77 --- /dev/null +++ b/platform/daemon/src/transcript-import-staging.ts @@ -0,0 +1,84 @@ +import { createHash } from "node:crypto"; +import { createWriteStream } from "node:fs"; +import { mkdir, rename, stat } from "node:fs/promises"; +import { dirname, join, relative, resolve } from "node:path"; + +export interface StagedTranscriptFile { + readonly managedPath: string; + readonly sizeBytes: number; + readonly contentHash: string; +} + +/** Stream raw bytes into a managed path, then fsync and atomically publish it. */ +export async function stageTranscriptStream( + root: string, + sourceId: string, + body: AsyncIterable, +): Promise { + if (!/^[A-Za-z0-9_-]+$/.test(sourceId)) throw new Error("invalid source id"); + const managedRelativePath = join("imports", "transcripts", sourceId, "source.jsonl"); + const destination = resolve(root, managedRelativePath); + const rootResolved = resolve(root); + if (!relative(rootResolved, destination) || relative(rootResolved, destination).startsWith("..")) + throw new Error("managed path escapes root"); + await mkdir(dirname(destination), { recursive: true }); + const partial = `${destination}.partial`; + const hash = createHash("sha256"); + let sizeBytes = 0; + const stream = createWriteStream(partial, { flags: "w" }); + try { + for await (const chunk of body) { + if (!(chunk instanceof Uint8Array)) throw new Error("stream chunk must be bytes"); + sizeBytes += chunk.byteLength; + hash.update(chunk); + if (!stream.write(chunk)) { + await new Promise((resolvePromise, reject) => { + const cleanup = (): void => { + stream.removeListener("drain", onDrain); + stream.removeListener("error", onError); + }; + const onDrain = (): void => { + cleanup(); + resolvePromise(); + }; + const onError = (error: Error): void => { + cleanup(); + reject(error); + }; + stream.once("drain", onDrain); + stream.once("error", onError); + }); + } + } + await new Promise((resolvePromise, reject) => { + const cleanup = (): void => { + stream.removeListener("finish", onFinish); + stream.removeListener("error", onError); + }; + const onFinish = (): void => { + cleanup(); + resolvePromise(); + }; + const onError = (error: Error): void => { + cleanup(); + reject(error); + }; + stream.once("finish", onFinish); + stream.once("error", onError); + stream.end(); + }); + const fd = await import("node:fs/promises").then((fs) => fs.open(partial, "r+")); + try { + await fd.sync(); + } finally { + await fd.close(); + } + const actual = await stat(partial); + if (actual.size !== sizeBytes) throw new Error("staged size verification failed"); + await rename(partial, destination); + return { managedPath: managedRelativePath, sizeBytes, contentHash: hash.digest("hex") }; + } catch (error) { + stream.destroy(); + throw error; + } +} diff --git a/platform/daemon/src/transcript-import-store.ts b/platform/daemon/src/transcript-import-store.ts new file mode 100644 index 0000000000..e8e4ffa637 --- /dev/null +++ b/platform/daemon/src/transcript-import-store.ts @@ -0,0 +1,392 @@ +import type { InventoryRecord } from "./transcript-import-inventory"; +import { TRANSCRIPT_IMPORT_LIMITS } from "./transcript-import-adapter"; +import { dbOwnerQuery, dbOwnerTransaction, dbOwnerTranscriptBulkCommit } from "./db-owner-runtime"; + +export type ImportJobState = + | "staging" + | "inventorying" + | "queued" + | "running" + | "paused" + | "completed" + | "completed_with_rejections" + | "cancelled" + | "failed"; +export interface ImportStoreOperation { + readonly kind: "source_import"; + readonly operation: + | "create_job" + | "checkpoint" + | "record_batch" + | "reconcile" + | "lease" + | "control" + | "list" + | "file_complete" + | "finalize" + | "reject" + | "commit" + | "recover"; + readonly agentId: string; + readonly jobId: string; + readonly payload: Readonly>; +} +export interface ImportStore { + readonly run: (operation: ImportStoreOperation) => Promise; +} + +/** Repository facade: production callers provide the DB-owner runner, never a SQLite writer. */ +export function createTranscriptImportStore(run: ImportStore["run"]): ImportStore { + return { run }; +} + +/** Production facade: import ledger SQL is always submitted to the DB owner. */ +export function createOwnerTranscriptImportStore(): ImportStore { + return createTranscriptImportStore(async (operation: ImportStoreOperation): Promise => { + const p = operation.payload; + if (operation.operation === "list") + return (await dbOwnerQuery( + { + sql: + p.view === "files" + ? "SELECT * FROM source_import_files WHERE job_id = ? AND agent_id = ? ORDER BY ordinal" + : p.view === "pending" + ? "SELECT id,job_id,file_id,source_id,agent_id,ordinal,line_number,byte_offset AS byteOffset,byte_length AS byteLength,raw_hash AS rawHash,status,rejection_code AS rejectionCode FROM source_import_records WHERE job_id = ? AND agent_id = ? AND file_id = ? AND status = 'pending' ORDER BY ordinal LIMIT ?" + : p.view === "status" + ? "SELECT * FROM source_import_jobs WHERE id = ? AND agent_id = ?" + : p.view === "recovery" + ? "SELECT * FROM source_import_jobs WHERE agent_id = ? AND state IN ('running','inventorying') ORDER BY created_at ASC LIMIT ?" + : "SELECT * FROM source_import_jobs WHERE agent_id = ? AND state IN ('queued','running','inventorying') ORDER BY created_at ASC LIMIT ?", + params: + p.view === "files" + ? [operation.jobId, operation.agentId] + : p.view === "pending" + ? [operation.jobId, operation.agentId, p.fileId as string, Number(p.limit ?? 25)] + : p.view === "status" + ? [operation.jobId, operation.agentId] + : [operation.agentId, Number(p.limit ?? 100)], + result: "all", + readonly: true, + }, + { operation: "sources.import.store.list", lane: "read" }, + )) as Result; + if (operation.operation === "reconcile") + return (await dbOwnerQuery( + { + sql: "SELECT status, COUNT(*) AS count FROM source_import_records WHERE job_id = ? AND agent_id = ? GROUP BY status", + params: [operation.jobId, operation.agentId], + result: "all", + readonly: true, + }, + { operation: "sources.import.store.reconcile", lane: "read" }, + )) as Result; + if (operation.operation === "lease") + return (await dbOwnerQuery( + { + sql: "UPDATE source_import_jobs SET state = 'running', lease_token = ?, lease_expires_at = datetime('now','+5 minutes'), started_at = COALESCE(started_at, datetime('now')), updated_at = datetime('now') WHERE id = ? AND agent_id = ? AND state = 'queued' AND generation = ? AND control_request IS NULL RETURNING *", + params: [p.token as string, operation.jobId, operation.agentId, p.generation as number], + result: "get", + readonly: false, + }, + { operation: "sources.import.store.lease", lane: "write" }, + )) as Result; + if (operation.operation === "record_batch") { + const records = p.records as InventoryRecord[]; + const statements = [ + { + sql: "UPDATE source_import_jobs SET updated_at = datetime('now') WHERE id = ? AND agent_id = ? AND generation = ? AND lease_token = ? AND state IN ('running','inventorying')", + params: [operation.jobId, operation.agentId, p.generation as number, p.leaseToken as string], + result: "run" as const, + requireChanges: true, + }, + ...records.map((r) => ({ + sql: "INSERT OR IGNORE INTO source_import_records (id,job_id,file_id,source_id,agent_id,ordinal,line_number,byte_offset,byte_length,raw_hash,status,rejection_code) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + params: [ + `${operation.jobId}:${p.fileId}:${r.ordinal}`, + operation.jobId, + p.fileId as string, + p.sourceId as string, + operation.agentId, + r.ordinal, + r.lineNumber, + r.byteOffset, + r.byteLength, + r.rawHash, + r.status, + r.rejectionCode ?? null, + ], + result: "run" as const, + })), + ]; + statements.push({ + sql: "UPDATE source_import_files SET checkpoint_byte_offset = MAX(checkpoint_byte_offset, ?), checkpoint_ordinal = MAX(checkpoint_ordinal, ?), record_count = (SELECT COUNT(*) FROM source_import_records WHERE file_id = ? AND agent_id = ?), malformed_count = (SELECT COUNT(*) FROM source_import_records WHERE file_id = ? AND agent_id = ? AND status = 'rejected'), state = 'inventorying', updated_at = datetime('now') WHERE id = ? AND job_id = ? AND agent_id = ? AND checkpoint_ordinal <= ? AND checkpoint_byte_offset <= ?", + params: [ + (p.checkpoint as { byteOffset: number }).byteOffset, + (p.checkpoint as { ordinal: number }).ordinal, + p.fileId as string, + operation.agentId, + p.fileId as string, + operation.agentId, + p.fileId as string, + operation.jobId, + operation.agentId, + (p.checkpoint as { ordinal: number }).ordinal, + (p.checkpoint as { byteOffset: number }).byteOffset, + ], + result: "run" as const, + }); + statements.push({ + sql: "UPDATE source_import_jobs SET total = (SELECT COUNT(*) FROM source_import_records WHERE job_id = ? AND agent_id = ?), updated_at = datetime('now') WHERE id = ? AND agent_id = ?", + params: [operation.jobId, operation.agentId, operation.jobId, operation.agentId], + result: "run" as const, + }); + return (await dbOwnerTransaction(statements, { + operation: "sources.import.store.record-batch", + lane: "write", + })) as Result; + } + if (operation.operation === "file_complete") + return (await dbOwnerTransaction( + [ + { + sql: "UPDATE source_import_files SET state = 'completed', updated_at = datetime('now') WHERE id = ? AND job_id = ? AND agent_id = ? AND EXISTS (SELECT 1 FROM source_import_jobs WHERE id = ? AND agent_id = ? AND generation = ? AND lease_token = ?)", + params: [ + p.fileId as string, + operation.jobId, + operation.agentId, + operation.jobId, + operation.agentId, + p.generation as number, + p.leaseToken as string, + ], + result: "run", + }, + ], + { operation: "sources.import.store.file-complete", lane: "write" }, + )) as Result; + if (operation.operation === "reject") + return (await dbOwnerTransaction( + [ + { + sql: "UPDATE source_import_records SET status = 'rejected', rejection_code = ?, updated_at = datetime('now') WHERE id = ? AND job_id = ? AND agent_id = ? AND status = 'pending'", + params: [p.code as string, p.recordId as string, operation.jobId, operation.agentId], + result: "run", + }, + { + sql: "INSERT INTO source_import_record_attempts (agent_id,job_id,file_id,record_id,generation,outcome,error_code,source_id) SELECT agent_id,job_id,file_id,id,0,'rejected',?,source_id FROM source_import_records WHERE id = ? AND job_id = ? AND agent_id = ?", + params: [p.code as string, p.recordId as string, operation.jobId, operation.agentId], + result: "run", + }, + ], + { operation: "sources.import.store.reject", lane: "write" }, + )) as Result; + if (operation.operation === "commit") { + const commits = p.commits as import("./transcript-import-commit").CompletedTranscriptCommit[]; + const results = await dbOwnerTranscriptBulkCommit( + { + agentId: operation.agentId, + jobId: operation.jobId, + generation: p.generation as number, + leaseToken: p.leaseToken as string, + sourceId: commits[0]?.sourceId ?? "", + harness: commits[0]?.harness ?? "", + commits, + }, + { operation: "sources.import.store.commit", lane: "write" }, + ); + for (let i = 0; i < commits.length; i++) + await dbOwnerTransaction( + [ + { + sql: "UPDATE source_import_records SET status = ?, canonical_id = ?, canonical_key = ?, external_identity = ?, conversation_fingerprint = ?, updated_at = datetime('now') WHERE id = ? AND job_id = ? AND agent_id = ? AND EXISTS (SELECT 1 FROM source_import_jobs WHERE id = ? AND agent_id = ? AND generation = ? AND lease_token = ?)", + params: [ + (results[i] as { outcome: string }).outcome === "conversation_identity_conflict" + ? "rejected" + : (results[i] as { outcome: string }).outcome, + commits[i]?.canonicalId ?? null, + commits[i]?.canonicalKey ?? null, + commits[i]?.externalIdentity ?? null, + commits[i]?.contentHash ?? null, + commits[i]?.sourceRecordId ?? "", + operation.jobId, + operation.agentId, + operation.jobId, + operation.agentId, + p.generation as number, + p.leaseToken as string, + ], + result: "run", + }, + { + sql: "INSERT INTO source_import_record_attempts (agent_id,job_id,file_id,record_id,generation,outcome,error_code,source_id) SELECT agent_id,job_id,file_id,id,0,?,?,source_id FROM source_import_records WHERE id = ? AND job_id = ? AND agent_id = ?", + params: [ + (results[i] as { outcome: string }).outcome, + (results[i] as { outcome: string }).outcome === "conversation_identity_conflict" + ? "conversation_identity_conflict" + : null, + commits[i]?.sourceRecordId ?? "", + operation.jobId, + operation.agentId, + ], + result: "run", + }, + ], + { operation: "sources.import.store.commit-record", lane: "write" }, + ); + return results as Result; + } + if (operation.operation === "finalize") + return (await dbOwnerTransaction( + [ + { + sql: "UPDATE source_import_jobs SET state = CASE WHEN (SELECT COUNT(*) FROM source_import_records WHERE job_id = ? AND agent_id = ? AND status = 'rejected') > 0 THEN 'completed_with_rejections' ELSE 'completed' END, imported = (SELECT COUNT(*) FROM source_import_records WHERE job_id = ? AND agent_id = ? AND status = 'imported'), duplicate = (SELECT COUNT(*) FROM source_import_records WHERE job_id = ? AND agent_id = ? AND status = 'duplicate'), rejected = (SELECT COUNT(*) FROM source_import_records WHERE job_id = ? AND agent_id = ? AND status = 'rejected'), pending = (SELECT COUNT(*) FROM source_import_records WHERE job_id = ? AND agent_id = ? AND status = 'pending'), lease_token = NULL, lease_expires_at = NULL, completed_at = datetime('now'), reconciled_at = datetime('now'), updated_at = datetime('now') WHERE id = ? AND agent_id = ? AND generation = ? AND lease_token = ?", + params: [ + operation.jobId, + operation.agentId, + operation.jobId, + operation.agentId, + operation.jobId, + operation.agentId, + operation.jobId, + operation.agentId, + operation.jobId, + operation.agentId, + operation.jobId, + operation.agentId, + p.generation as number, + p.leaseToken as string, + ], + result: "run", + }, + ], + { operation: "sources.import.store.finalize", lane: "write" }, + )) as Result; + if (operation.operation === "recover") + return (await dbOwnerTransaction( + [ + { + sql: "UPDATE source_import_jobs SET lease_token = NULL, lease_expires_at = NULL, state = CASE WHEN state IN ('running','inventorying') THEN 'queued' ELSE state END, updated_at = datetime('now') WHERE agent_id = ? AND (? = '*' OR id = ?)", + params: [operation.agentId, operation.jobId, operation.jobId], + result: "run", + }, + ], + { operation: "sources.import.store.recover", lane: "write" }, + )) as Result; + if (operation.operation === "control") + return (await dbOwnerTransaction( + [ + { + sql: + p.apply === true + ? "UPDATE source_import_jobs SET state = CASE WHEN control_request = 'pause' THEN 'paused' WHEN control_request = 'cancel' THEN 'cancelled' ELSE state END, lease_token = CASE WHEN control_request IN ('pause','cancel') THEN NULL ELSE lease_token END, control_request = CASE WHEN control_request IN ('pause','cancel') THEN NULL ELSE control_request END, updated_at = datetime('now') WHERE id = ? AND agent_id = ? AND generation = ? AND lease_token = ?" + : "UPDATE source_import_jobs SET control_request = ?, updated_at = datetime('now') WHERE id = ? AND agent_id = ?", + params: + p.apply === true + ? [operation.jobId, operation.agentId, p.generation as number, p.leaseToken as string] + : [p.control as string, operation.jobId, operation.agentId], + result: "run", + }, + ...(p.apply === true + ? [ + { + sql: "UPDATE source_import_records SET status = 'cancelled', rejection_code = 'cancelled_by_user', updated_at = datetime('now') WHERE job_id = ? AND agent_id = ? AND status = 'pending'", + params: [operation.jobId, operation.agentId], + result: "run" as const, + }, + ] + : []), + ], + { operation: "sources.import.store.control", lane: "write" }, + )) as Result; + if (operation.operation === "checkpoint") + return (await dbOwnerTransaction( + [ + { + sql: "UPDATE source_import_files SET checkpoint_byte_offset = MAX(checkpoint_byte_offset, ?), checkpoint_ordinal = MAX(checkpoint_ordinal, ?), updated_at = datetime('now') WHERE id = ? AND job_id = ? AND agent_id = ?", + params: [ + p.byteOffset as number, + p.ordinal as number, + p.fileId as string, + operation.jobId, + operation.agentId, + ], + result: "run", + requireChanges: true, + }, + ], + { operation: "sources.import.store.checkpoint", lane: "write" }, + )) as Result; + if (operation.operation === "create_job") + return (await dbOwnerTransaction( + [ + { + sql: "INSERT INTO source_import_jobs (id,kind,agent_id,schema_id,adapter_version,state,generation) VALUES (?, 'import', ?, ?, 1, 'staging', 0)", + params: [operation.jobId, operation.agentId, p.schemaId as string], + result: "run", + }, + ], + { operation: "sources.import.store.create", lane: "write" }, + )) as Result; + throw new Error(`Unsupported transcript import store operation: ${operation.operation}`); + }); +} +export function createJob( + store: ImportStore, + input: { jobId: string; agentId: string; schemaId?: string }, +): Promise { + return store.run({ + kind: "source_import", + operation: "create_job", + agentId: input.agentId, + jobId: input.jobId, + payload: { schemaId: input.schemaId ?? "signet-export", adapterVersion: 1, state: "staging", generation: 0 }, + }); +} +export function checkpointInventory( + store: ImportStore, + input: { jobId: string; agentId: string; fileId: string; byteOffset: number; ordinal: number }, +): Promise { + if (input.byteOffset < 0 || input.ordinal < 0) throw new Error("checkpoint must be nonnegative"); + return store.run({ + kind: "source_import", + operation: "checkpoint", + agentId: input.agentId, + jobId: input.jobId, + payload: { fileId: input.fileId, byteOffset: input.byteOffset, ordinal: input.ordinal, inventoryVersion: 1 }, + }); +} +export function recordInventoryBatch( + store: ImportStore, + input: { jobId: string; agentId: string; fileId?: string; sourceId?: string; records: readonly InventoryRecord[] }, +): Promise { + if (input.records.length > TRANSCRIPT_IMPORT_LIMITS.maxRecordsPerBatch) + throw new Error("inventory batch exceeds limit"); + return store.run({ + kind: "source_import", + operation: "record_batch", + agentId: input.agentId, + jobId: input.jobId, + payload: { fileId: input.fileId, sourceId: input.sourceId, records: input.records }, + }); +} +export function reconcileImport(store: ImportStore, input: { jobId: string; agentId: string }): Promise { + return store.run({ + kind: "source_import", + operation: "reconcile", + agentId: input.agentId, + jobId: input.jobId, + payload: {}, + }); +} +export function controlImport( + store: ImportStore, + input: { jobId: string; agentId: string; control: "pause" | "resume" | "retry" | "cancel" }, +): Promise { + return store.run({ + kind: "source_import", + operation: "control", + agentId: input.agentId, + jobId: input.jobId, + payload: { control: input.control }, + }); +} diff --git a/platform/daemon/src/transcript-import-worker.test.ts b/platform/daemon/src/transcript-import-worker.test.ts new file mode 100644 index 0000000000..a49e828d6a --- /dev/null +++ b/platform/daemon/src/transcript-import-worker.test.ts @@ -0,0 +1,423 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, rm, mkdir, writeFile, readdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + appendCanonical, + startTranscriptImportWorker, + purgeTranscriptImportFilesystem, +} from "./transcript-import-worker"; +import { closeDbAccessor, getDbAccessor, initDbAccessor } from "./db-accessor"; +import { createJob, createOwnerTranscriptImportStore } from "./transcript-import-store"; +import type { ImportStore, ImportStoreOperation } from "./transcript-import-store"; +import type { InventoryRecord } from "./transcript-import-inventory"; + +type TestRow = { id: string; source_id: string; status: string; [key: string]: unknown }; +type TestCommit = { sourceId: string; canonicalId: string; canonicalKey: string }; + +const valid = (id: string) => + JSON.stringify({ + source: "signet", + id, + harness: "h", + agent_id: "embedded", + session_key: id, + project: null, + timestamp: "2024-01-01T00:00:00Z", + message_count: 2, + messages: [ + { role: "system", content: " exact\n" }, + { role: "user", content: ` ${id} ` }, + ], + }); + +test("worker inventories, rereads, appends canonical evidence, and is replay-safe", async () => { + const root = await mkdtemp(join(tmpdir(), "signet-import-worker-")); + const source = "source-a"; + await mkdir(join(root, "imports", "transcripts", source), { recursive: true }); + await writeFile( + join(root, "imports", "transcripts", source, "source.jsonl"), + `${valid("one")}\nnot-json\n${valid("two")}\n`, + ); + const records: TestRow[] = []; + let commitAttempts = 0; + const store: ImportStore = { + run: async (op: ImportStoreOperation) => { + if (op.operation === "list" && op.payload.view === "work") return [{ id: "job", state: "queued" }] as T; + if (op.operation === "lease") return { id: "job", state: "running" } as T; + if (op.operation === "list" && op.payload.view === "files") + return [ + { + id: "file", + job_id: "job", + source_id: source, + managed_path: `imports/transcripts/${source}/source.jsonl`, + checkpoint_byte_offset: 0, + checkpoint_ordinal: 0, + state: records.length ? "completed" : "ready", + }, + ] as T; + if (op.operation === "record_batch") { + for (const r of op.payload.records as InventoryRecord[]) + records.push({ + ...r, + id: `record-${r.ordinal}`, + file_id: "file", + job_id: "job", + source_id: source, + status: r.status, + }); + return undefined as T; + } + if (op.operation === "list" && op.payload.view === "pending") + return records.filter((r) => r.status === "pending") as T; + if (op.operation === "commit") { + commitAttempts++; + if (commitAttempts === 1) throw new Error("injected crash after filesystem append"); + for (const c of op.payload.commits as TestCommit[]) { + const r = records.find((x) => x.source_id === c.sourceId && x.status === "pending"); + if (r) r.status = "imported"; + } + return (op.payload.commits as TestCommit[]).map((c) => ({ + outcome: "imported", + canonicalId: c.canonicalId, + sessionKey: c.canonicalKey, + })) as T; + } + if (op.operation === "file_complete" || op.operation === "finalize" || op.operation === "recover") + return undefined as T; + if (op.operation === "reject") { + const r = records.find((x) => x.id === op.payload.recordId); + if (r) r.status = "rejected"; + return undefined as T; + } + return [] as T; + }, + }; + try { + const worker = startTranscriptImportWorker({ store, agentId: "agent-a", workspaceRoot: root, pollMs: 1 }); + await new Promise((resolve) => setTimeout(resolve, 40)); + await worker.stop(); + expect(records.filter((r) => r.status === "pending")).toHaveLength(0); + expect(records.filter((r) => r.status === "rejected")).toHaveLength(1); + const canonicalFiles = await readdir(join(root, "transcripts")); + expect(canonicalFiles.filter((name) => name.endsWith(".jsonl"))).toHaveLength(1); + const canonical = await (await import("node:fs/promises")).readFile( + join(root, "transcripts", canonicalFiles.find((name) => name.endsWith(".jsonl")) as string), + "utf8", + ); + expect(canonical.trim().split("\n")).toHaveLength(2); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("canonical append removes a lock whose recorded owner process is gone", async () => { + const root = await mkdtemp(join(tmpdir(), "signet-import-stale-lock-")); + try { + const { signetExportV1Adapter } = await import("./transcript-import-adapter"); + const { buildCompletedTranscriptCommit } = await import("./transcript-import-commit"); + const commit = buildCompletedTranscriptCommit(signetExportV1Adapter.parse(JSON.parse(valid("stale-lock"))), { + agentId: "agent-a", + sourceId: "source-a", + sourceRecordId: "record-a", + }); + const digest = (await import("node:crypto")).createHash("sha256").update("agent-a\0h").digest("hex").slice(0, 24); + const lock = join(root, "transcripts", `${digest}.jsonl.lock`); + await mkdir(lock, { recursive: true }); + await writeFile(join(lock, "owner"), "99999999\\n"); + await appendCanonical(root, "agent-a", "h", [commit]); + expect(await readdir(join(root, "transcripts"))).toContain(`${digest}.jsonl`); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("owner store persists inventory and terminal counters in the migrated database", async () => { + const root = await mkdtemp(join(tmpdir(), "signet-import-owner-")); + const dbPath = join(root, "memory", "memories.db"); + await mkdir(join(root, "memory"), { recursive: true }); + const oldOwner = process.env.SIGNET_DB_OWNER_WORKER; + process.env.SIGNET_DB_OWNER_WORKER = "1"; + closeDbAccessor(); + initDbAccessor(dbPath, { agentsDir: root }); + const agentId = "owner-agent"; + const jobId = "owner-job"; + const source = "owner-source"; + try { + const store = createOwnerTranscriptImportStore(); + await createJob(store, { jobId, agentId }); + await getDbAccessor().withWriteTxAsync((db) => { + db.prepare("UPDATE source_import_jobs SET state = 'queued' WHERE id = ? AND agent_id = ?").run(jobId, agentId); + }); + await getDbAccessor().withWriteTxAsync((db) => { + db.prepare( + "INSERT INTO source_import_files (id, job_id, source_id, agent_id, ordinal, name, managed_path, state) VALUES (?, ?, ?, ?, 0, 'source.jsonl', ?, 'ready')", + ).run("owner-file", jobId, source, agentId, `imports/transcripts/${source}/source.jsonl`); + }); + await mkdir(join(root, "imports", "transcripts", source), { recursive: true }); + await writeFile(join(root, "imports", "transcripts", source, "source.jsonl"), `${valid("owner")}\n`); + const worker = startTranscriptImportWorker({ store, agentId, workspaceRoot: root, pollMs: 1 }); + await new Promise((resolve) => setTimeout(resolve, 100)); + await worker.stop(); + const state = await getDbAccessor().withReadDbAsync( + (db) => + db + .prepare("SELECT state, imported, pending FROM source_import_jobs WHERE id = ? AND agent_id = ?") + .get(jobId, agentId) as { state: string; imported: number; pending: number }, + ); + const checkpoint = await getDbAccessor().withReadDbAsync( + (db) => + db + .prepare("SELECT checkpoint_ordinal, record_count FROM source_import_files WHERE id = ? AND agent_id = ?") + .get("owner-file", agentId) as { checkpoint_ordinal: number; record_count: number }, + ); + expect(state).toEqual({ state: "completed", imported: 1, pending: 0 }); + expect(checkpoint).toEqual({ checkpoint_ordinal: 1, record_count: 1 }); + } finally { + closeDbAccessor(); + if (oldOwner === undefined) Reflect.deleteProperty(process.env, "SIGNET_DB_OWNER_WORKER"); + else process.env.SIGNET_DB_OWNER_WORKER = oldOwner; + await rm(root, { recursive: true, force: true }); + } +}); + +test("migrated DB crash recovery finalizes pending records after filesystem write without duplicate canonical evidence", async () => { + const root = await mkdtemp(join(tmpdir(), "signet-import-fs-before-db-recovery-")); + const dbPath = join(root, "memory", "memories.db"); + const oldOwner = process.env.SIGNET_DB_OWNER_WORKER; + process.env.SIGNET_DB_OWNER_WORKER = "1"; + closeDbAccessor(); + await mkdir(join(root, "memory"), { recursive: true }); + initDbAccessor(dbPath, { agentsDir: root }); + const store = createOwnerTranscriptImportStore(); + const jobId = "crash-job"; + const fileId = "crash-file"; + const sourceId = "import:crash-source"; + const agentId = "crash-agent"; + const raw = valid("crash"); + try { + await createJob(store, { jobId, agentId }); + await getDbAccessor().withWriteTxAsync((db) => { + db.prepare( + "UPDATE source_import_jobs SET state = 'running', total = 1, pending = 0, lease_token = 'stale' WHERE id = ?", + ).run(jobId); + db.prepare( + "INSERT INTO source_import_files (id,job_id,source_id,agent_id,ordinal,name,managed_path,state,record_count,checkpoint_ordinal,checkpoint_byte_offset) VALUES (?,?,?,?,0,'crash.jsonl',?,'completed',1,1,?)", + ).run( + fileId, + jobId, + sourceId, + agentId, + `imports/transcripts/${sourceId}/source.jsonl`, + Buffer.byteLength(`${raw}\n`), + ); + db.prepare( + "INSERT INTO source_import_records (id,job_id,file_id,source_id,agent_id,ordinal,line_number,byte_offset,byte_length,raw_hash,status) VALUES (?,?,?,?,?,?,?,?,?,?, 'pending')", + ).run(`${jobId}:${fileId}:1`, jobId, fileId, sourceId, agentId, 1, 1, 0, Buffer.byteLength(raw), "raw-hash"); + }); + await mkdir(join(root, "imports", "transcripts", sourceId), { recursive: true }); + await writeFile(join(root, "imports", "transcripts", sourceId, "source.jsonl"), `${raw}\n`); + const parsed = JSON.parse(raw); + const { signetExportV1Adapter } = await import("./transcript-import-adapter"); + const { buildCompletedTranscriptCommit, canonicalTranscriptLine } = await import("./transcript-import-commit"); + const commit = buildCompletedTranscriptCommit(signetExportV1Adapter.parse(parsed), { + agentId, + sourceId, + sourceRecordId: `${jobId}:${fileId}:1`, + sourcePath: `imports/transcripts/${sourceId}/source.jsonl`, + }); + await mkdir(join(root, "transcripts"), { recursive: true }); + await writeFile(join(root, "transcripts", "canonical.jsonl"), canonicalTranscriptLine(commit)); + const worker = startTranscriptImportWorker({ store, agentId, workspaceRoot: root, pollMs: 1 }); + await new Promise((resolve) => setTimeout(resolve, 100)); + await worker.stop(); + const state = await getDbAccessor().withReadDbAsync( + (db) => + db.prepare("SELECT state, imported, pending, lease_token FROM source_import_jobs WHERE id = ?").get(jobId) as { + state: string; + imported: number; + pending: number; + lease_token: string | null; + }, + ); + const counts = await getDbAccessor().withReadDbAsync( + (db) => + db + .prepare("SELECT status, COUNT(*) AS count FROM source_import_records WHERE job_id = ? GROUP BY status") + .all(jobId) as Array<{ status: string; count: number }>, + ); + const canonical = await (await import("node:fs/promises")).readFile( + join(root, "transcripts", "canonical.jsonl"), + "utf8", + ); + expect(state).toEqual({ state: "completed", imported: 1, pending: 0, lease_token: null }); + expect(counts).toEqual([{ status: "imported", count: 1 }]); + expect(canonical.trim().split("\n")).toHaveLength(1); + } finally { + closeDbAccessor(); + if (oldOwner === undefined) Reflect.deleteProperty(process.env, "SIGNET_DB_OWNER_WORKER"); + else process.env.SIGNET_DB_OWNER_WORKER = oldOwner; + await rm(root, { recursive: true, force: true }); + } +}); +test("filesystem purge removes only the selected agent source and staged data", async () => { + const root = await mkdtemp(join(tmpdir(), "signet-import-purge-")); + try { + await mkdir(join(root, "imports", "transcripts", "source-a"), { recursive: true }); + await writeFile(join(root, "imports", "transcripts", "source-a", "source.jsonl"), "staged"); + await mkdir(join(root, "transcripts"), { recursive: true }); + await writeFile( + join(root, "transcripts", "aggregate.jsonl"), + `${[ + JSON.stringify({ id: "a", agent_id: "agent-a", source_id: "source-a" }), + JSON.stringify({ id: "b", agent_id: "agent-b", source_id: "source-a" }), + JSON.stringify({ id: "c", agent_id: "agent-a", source_id: "source-b" }), + ].join("\n")}\n`, + ); + await purgeTranscriptImportFilesystem(root, "source-a", "agent-a"); + expect(await readdir(join(root, "imports", "transcripts"))).toEqual([]); + const aggregate = await (await import("node:fs/promises")).readFile( + join(root, "transcripts", "aggregate.jsonl"), + "utf8", + ); + expect(aggregate).toContain('"id":"b"'); + expect(aggregate).not.toContain('"id":"a"'); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("filesystem purge removes the ledger-reserved managed upload path", async () => { + const root = await mkdtemp(join(tmpdir(), "signet-import-managed-purge-")); + try { + const managedPath = "imports/transcripts/job/file/source.jsonl"; + await mkdir(join(root, "imports", "transcripts", "job", "file"), { recursive: true }); + await writeFile(join(root, managedPath), "staged"); + await purgeTranscriptImportFilesystem(root, "import:job:file", "agent-a", [managedPath]); + expect(await Bun.file(join(root, managedPath)).exists()).toBe(false); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("active pause and cancel controls clear the lease at the checkpoint", async () => { + const root = await mkdtemp(join(tmpdir(), "signet-import-controls-")); + const oldOwner = process.env.SIGNET_DB_OWNER_WORKER; + process.env.SIGNET_DB_OWNER_WORKER = "1"; + closeDbAccessor(); + await mkdir(join(root, "memory"), { recursive: true }); + initDbAccessor(join(root, "memory", "memories.db"), { agentsDir: root }); + const store = createOwnerTranscriptImportStore(); + try { + for (const [jobId, control, expectedState] of [ + ["pause-job", "pause", "paused"], + ["cancel-job", "cancel", "cancelled"], + ] as const) { + await createJob(store, { jobId, agentId: "agent-a" }); + await getDbAccessor().withWriteTxAsync((db) => + db + .prepare( + "UPDATE source_import_jobs SET state = 'running', lease_token = 'lease-a', control_request = ? WHERE id = ?", + ) + .run(control, jobId), + ); + await store.run({ + kind: "source_import", + operation: "control", + agentId: "agent-a", + jobId, + payload: { apply: true, generation: 0, leaseToken: "lease-a" }, + }); + const row = await getDbAccessor().withReadDbAsync((db) => + db.prepare("SELECT state, lease_token FROM source_import_jobs WHERE id = ?").get(jobId), + ); + expect(row).toEqual({ state: expectedState, lease_token: null }); + } + } finally { + closeDbAccessor(); + if (oldOwner === undefined) Reflect.deleteProperty(process.env, "SIGNET_DB_OWNER_WORKER"); + else process.env.SIGNET_DB_OWNER_WORKER = oldOwner; + await rm(root, { recursive: true, force: true }); + } +}); + +test("stale finalize cannot complete a job after its lease generation changes", async () => { + const root = await mkdtemp(join(tmpdir(), "signet-import-stale-finalize-")); + const oldOwner = process.env.SIGNET_DB_OWNER_WORKER; + process.env.SIGNET_DB_OWNER_WORKER = "1"; + closeDbAccessor(); + await mkdir(join(root, "memory"), { recursive: true }); + initDbAccessor(join(root, "memory", "memories.db"), { agentsDir: root }); + const store = createOwnerTranscriptImportStore(); + try { + await createJob(store, { jobId: "stale-job", agentId: "agent-a" }); + await getDbAccessor().withWriteTxAsync((db) => + db + .prepare( + "UPDATE source_import_jobs SET state = 'running', generation = 1, lease_token = 'new-lease' WHERE id = 'stale-job'", + ) + .run(), + ); + await store.run({ + kind: "source_import", + operation: "finalize", + agentId: "agent-a", + jobId: "stale-job", + payload: { generation: 0, leaseToken: "old-lease" }, + }); + const row = await getDbAccessor().withReadDbAsync((db) => + db.prepare("SELECT state, generation, lease_token FROM source_import_jobs WHERE id = 'stale-job'").get(), + ); + expect(row).toEqual({ state: "running", generation: 1, lease_token: "new-lease" }); + } finally { + closeDbAccessor(); + if (oldOwner === undefined) Reflect.deleteProperty(process.env, "SIGNET_DB_OWNER_WORKER"); + else process.env.SIGNET_DB_OWNER_WORKER = oldOwner; + await rm(root, { recursive: true, force: true }); + } +}); + +test("source purge audit attempts retain source identity after import ledgers are removed", async () => { + const root = await mkdtemp(join(tmpdir(), "signet-import-audit-tombstone-")); + const dbPath = join(root, "memory", "memories.db"); + const oldOwner = process.env.SIGNET_DB_OWNER_WORKER; + process.env.SIGNET_DB_OWNER_WORKER = "1"; + closeDbAccessor(); + await mkdir(join(root, "memory"), { recursive: true }); + initDbAccessor(dbPath, { agentsDir: root }); + const store = createOwnerTranscriptImportStore(); + try { + await createJob(store, { jobId: "audit-job", agentId: "audit-agent" }); + await getDbAccessor().withWriteTxAsync((db) => { + db.prepare( + "INSERT INTO source_import_files (id,job_id,source_id,agent_id,ordinal,name,managed_path,state) VALUES ('audit-file','audit-job','audit-source','audit-agent',0,'source.jsonl','imports/transcripts/audit-source/source.jsonl','completed')", + ).run(); + db.prepare( + "INSERT INTO source_import_records (id,job_id,file_id,source_id,agent_id,ordinal,line_number,byte_offset,byte_length,raw_hash,status) VALUES ('audit-record','audit-job','audit-file','audit-source','audit-agent',1,1,0,1,'hash','pending')", + ).run(); + }); + await store.run({ + kind: "source_import", + operation: "reject", + agentId: "audit-agent", + jobId: "audit-job", + payload: { recordId: "audit-record", code: "malformed" }, + }); + const attempt = await getDbAccessor().withReadDbAsync( + (db) => + db + .prepare("SELECT source_id, outcome FROM source_import_record_attempts WHERE record_id = ?") + .get("audit-record") as { + source_id: string; + outcome: string; + }, + ); + expect(attempt).toEqual({ source_id: "audit-source", outcome: "rejected" }); + } finally { + closeDbAccessor(); + if (oldOwner === undefined) Reflect.deleteProperty(process.env, "SIGNET_DB_OWNER_WORKER"); + else process.env.SIGNET_DB_OWNER_WORKER = oldOwner; + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/platform/daemon/src/transcript-import-worker.ts b/platform/daemon/src/transcript-import-worker.ts new file mode 100644 index 0000000000..6f4aba51d2 --- /dev/null +++ b/platform/daemon/src/transcript-import-worker.ts @@ -0,0 +1,470 @@ +import { createHash, randomUUID } from "node:crypto"; +import { open, mkdir, rm, writeFile, readdir, rename } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { TRANSCRIPT_IMPORT_LIMITS, signetExportV1Adapter } from "./transcript-import-adapter"; +import { buildCompletedTranscriptCommit, canonicalTranscriptLine } from "./transcript-import-commit"; +import { inventoryTranscriptFile, type InventoryRecord } from "./transcript-import-inventory"; +import type { ImportJobState, ImportStore, ImportStoreOperation } from "./transcript-import-store"; +import { controlImport, reconcileImport } from "./transcript-import-store"; + +export interface TranscriptImportWorkerHandle { + readonly running: boolean; + stop(): Promise; + nudge(): void; +} +export interface TranscriptImportWorkerOptions { + readonly store: ImportStore; + readonly agentId: string; + readonly workspaceRoot?: string; + readonly pressure?: () => boolean; + readonly yield?: () => Promise; + readonly onBatch?: (jobId: string, sourceId: string) => Promise; + readonly pollMs?: number; +} +type Job = { + id: string; + state: ImportJobState; + generation?: number; + control_request?: string | null; + lease_token?: string | null; +}; +type File = { + id: string; + job_id: string; + source_id: string; + managed_path: string; + checkpoint_byte_offset: number; + checkpoint_ordinal: number; + state: string; +}; +type Row = InventoryRecord & { + id: string; + file_id: string; + source_id: string; + job_id: string; + status: string; + value?: never; +}; + +/** Executes imports one job/file at a time. SQLite mutations are delegated to ImportStore. */ +export function startTranscriptImportWorker(options: TranscriptImportWorkerOptions): TranscriptImportWorkerHandle { + let active = true; + let wake: (() => void) | undefined; + let loopPromise: Promise; + const wait = () => + new Promise((resolve) => { + wake = resolve; + }); + const root = options.workspaceRoot ?? process.env.SIGNET_PATH ?? join(process.env.HOME ?? ".", ".agents"); + const run = async (): Promise => { + // Recovery can race DB-owner startup. Keep retrying the durable reset until + // it is accepted; proceeding after one swallowed failure strands jobs with + // an old lease and prevents the post-filesystem replay path from running. + while (active) { + try { + await options.store.run({ + kind: "source_import", + operation: "recover", + agentId: options.agentId, + jobId: "*", + payload: {}, + }); + break; + } catch { + await new Promise((resolve) => setTimeout(resolve, 250)); + } + } + while (active) { + if (options.pressure?.()) { + await wait(); + continue; + } + try { + const jobs = await options.store.run({ + kind: "source_import", + operation: "list", + agentId: options.agentId, + jobId: "*", + payload: { limit: 1, view: "work" }, + }); + const job = jobs[0]; + if (job && ["queued", "running", "inventorying"].includes(job.state)) await processJob(job); + } catch { + /* retain durable state; the next nudge retries infrastructure work */ + } + if (active) { + await options.yield?.(); + if ((options.pollMs ?? 250) > 0) + await Promise.race([wait(), new Promise((r) => setTimeout(r, options.pollMs ?? 250))]); + } + } + }; + const processJob = async (job: Job): Promise => { + const leaseToken = randomUUID(); + const leased = await options.store.run<(Job & { lease_token: string }) | null>({ + kind: "source_import", + operation: "lease", + agentId: options.agentId, + jobId: job.id, + payload: { token: leaseToken, generation: job.generation ?? 0 }, + }); + if (!leased) return; + const guard = async (): Promise => { + const rows = await options.store.run({ + kind: "source_import", + operation: "list", + agentId: options.agentId, + jobId: job.id, + payload: { view: "status" }, + }); + const current = rows[0]; + if (!current) return; + if (current.generation !== leased.generation || current.lease_token !== leaseToken) + throw new Error("stale import lease"); + if (current.control_request === "pause" || current.control_request === "cancel") { + await options.store.run({ + kind: "source_import", + operation: "control", + agentId: options.agentId, + jobId: job.id, + payload: { apply: true, generation: leased.generation, leaseToken }, + }); + throw new Error(`import ${current.control_request}d at checkpoint`); + } + }; + const guarded = async (operation: ImportStoreOperation): Promise => { + await guard(); + return options.store.run(operation); + }; + await guard(); + const files = await options.store.run({ + kind: "source_import", + operation: "list", + agentId: options.agentId, + jobId: job.id, + payload: { view: "files" }, + }); + const committedSources = new Set(); + for (const file of files) { + if (!active) return; + const path = join(root, file.managed_path); + if (file.state !== "completed") { + await inventoryTranscriptFile( + path, + { byteOffset: file.checkpoint_byte_offset, ordinal: file.checkpoint_ordinal }, + TRANSCRIPT_IMPORT_LIMITS.maxRecordsPerBatch, + async (records, checkpoint) => { + await guarded({ + kind: "source_import", + operation: "record_batch", + agentId: options.agentId, + jobId: job.id, + payload: { + fileId: file.id, + sourceId: file.source_id, + records, + checkpoint, + generation: leased.generation, + leaseToken, + }, + }); + await maybeCrashDuringInventory(root); + }, + ); + await guarded({ + kind: "source_import", + operation: "file_complete", + agentId: options.agentId, + jobId: job.id, + payload: { fileId: file.id, generation: leased.generation, leaseToken }, + }); + } + if (await commitPending(job.id, file, path, guarded, { generation: leased.generation ?? 0, leaseToken })) + committedSources.add(file.source_id); + } + await guarded({ + kind: "source_import", + operation: "finalize", + agentId: options.agentId, + jobId: job.id, + payload: { generation: leased.generation, leaseToken }, + }); + for (const sourceId of committedSources) await options.onBatch?.(job.id, sourceId); + }; + const commitPending = async ( + jobId: string, + file: File, + path: string, + guarded: (operation: ImportStoreOperation) => Promise, + lease: { generation: number; leaseToken: string }, + ): Promise => { + let committed = false; + for (;;) { + const rows = await options.store.run({ + kind: "source_import", + operation: "list", + agentId: options.agentId, + jobId, + payload: { view: "pending", fileId: file.id, limit: TRANSCRIPT_IMPORT_LIMITS.maxRecordsPerBatch }, + }); + if (!rows.length) return committed; + const commits = []; + for (const row of rows) { + try { + const handle = await open(path, "r"); + const raw = Buffer.alloc(row.byteLength); + try { + const got = await handle.read(raw, 0, row.byteLength, row.byteOffset); + if (got.bytesRead !== row.byteLength) throw new Error("source_changed"); + const bytes = raw.subarray(0, got.bytesRead); + const value = signetExportV1Adapter.parse( + JSON.parse( + new TextDecoder("utf-8", { fatal: true }).decode( + bytes[bytes.length - 1] === 0x0a ? bytes.subarray(0, bytes.length - 1) : bytes, + ), + ), + ); + commits.push( + buildCompletedTranscriptCommit(value, { + agentId: options.agentId, + sourceId: row.source_id, + sourceRecordId: row.id, + sourcePath: file.managed_path, + }), + ); + } finally { + await handle.close(); + } + } catch (error) { + await options.store.run({ + kind: "source_import", + operation: "reject", + agentId: options.agentId, + jobId, + payload: { recordId: row.id, code: error instanceof Error ? error.message : "source_changed" }, + }); + } + } + if (commits.length) { + const byHarness = new Map(); + for (const commit of commits) { + let group = byHarness.get(commit.harness); + if (group === undefined) { + group = []; + byHarness.set(commit.harness, group); + } + group.push(commit); + } + for (const [harness, batch] of byHarness) { + await appendCanonical(root, options.agentId, harness, batch); + await guarded({ + kind: "source_import", + operation: "commit", + agentId: options.agentId, + jobId, + payload: { commits: batch, generation: lease.generation, leaseToken: lease.leaseToken }, + }); + committed = true; + } + } + } + }; + loopPromise = run(); + return { + get running() { + return active; + }, + stop: async () => { + active = false; + wake?.(); + await loopPromise; + }, + nudge: () => wake?.(), + }; +} + +export async function appendCanonical( + root: string, + agentId: string, + harness: string, + commits: readonly Parameters[0][], +): Promise { + const path = join(root, "transcripts", `${hash(`${agentId}\0${harness}`)}.jsonl`); + await mkdir(dirname(path), { recursive: true }); + const lock = `${path}.lock`; + for (;;) { + try { + await mkdir(lock); + await writeFile(join(lock, "owner"), `${process.pid}\n`, "utf8"); + break; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + try { + const owner = (await import("node:fs/promises")).readFile(join(lock, "owner"), "utf8"); + const ownerPid = Number((await owner).trim()); + if (Number.isInteger(ownerPid) && ownerPid > 0) { + try { + process.kill(ownerPid, 0); + await new Promise((resolve) => setTimeout(resolve, 5)); + continue; + } catch (probeError) { + if ((probeError as NodeJS.ErrnoException).code !== "ESRCH") throw probeError; + } + } + await rm(lock, { recursive: true, force: true }); + } catch { + // A process killed after mkdir or while holding the lock leaves a + // stale directory. Only this worker writes these per-harness locks. + await rm(lock, { recursive: true, force: true }); + } + } + } + try { + const existing = await readText(path); + const ids = new Set(); + for (const line of existing.split("\n")) { + if (!line) continue; + try { + const id = (JSON.parse(line) as { id?: unknown }).id; + if (typeof id === "string") ids.add(id); + } catch { + throw new Error("canonical_transcript_corrupt"); + } + } + const missing = commits.filter((commit) => !ids.has(commit.recordId)); + if (!missing.length) return; + const temp = `${path}.append-${randomUUID()}`; + const payload = missing.map(canonicalTranscriptLine).join(""); + const staged = await open(temp, "wx"); + try { + await staged.writeFile(payload); + await staged.sync(); + } finally { + await staged.close(); + } + const source = await open(path, "a+"); + try { + await source.writeFile(payload); + await source.sync(); + } finally { + await source.close(); + } + await maybeCrashAfterCanonicalWrite(root); + await rm(temp, { force: true }); + const directory = await open(dirname(path), "r"); + try { + await directory.sync(); + } finally { + await directory.close(); + } + } finally { + await rm(lock, { recursive: true, force: true }); + } +} +async function maybeCrashDuringInventory(root: string): Promise { + if (process.env.SIGNET_TRANSCRIPT_IMPORT_FAILPOINT !== "inventory") return; + const marker = join(root, ".daemon", "transcript-import-inventory-failpoint-fired"); + try { + await open(marker, "wx").then((handle) => handle.close()); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") return; + throw error; + } + process.stderr.write("transcript-import failpoint: inventory\\n"); + process.exit(87); +} + +async function maybeCrashAfterCanonicalWrite(root: string): Promise { + if (process.env.SIGNET_TRANSCRIPT_IMPORT_FAILPOINT !== "after-fs-before-db") return; + const marker = join(root, ".daemon", "transcript-import-failpoint-fired"); + try { + await open(marker, "wx").then((handle) => handle.close()); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") return; + throw error; + } + process.stderr.write("transcript-import failpoint: after-fs-before-db\\n"); + process.exit(86); +} + +async function _recover(options: TranscriptImportWorkerOptions): Promise { + const jobs = await options.store.run({ + kind: "source_import", + operation: "list", + agentId: options.agentId, + jobId: "*", + payload: { limit: 100, view: "recovery" }, + }); + for (const job of jobs) + if (["running", "inventorying"].includes(job.state)) + await options.store.run({ + kind: "source_import", + operation: "recover", + agentId: options.agentId, + jobId: job.id, + payload: {}, + }); +} +async function readText(path: string): Promise { + try { + return await (await import("node:fs/promises")).readFile(path, "utf8"); + } catch { + return ""; + } +} +function hash(value: string): string { + return createHash("sha256").update(value).digest("hex").slice(0, 24); +} + +/** Remove staged source data and rewrite each canonical harness stream once. */ +export async function purgeTranscriptImportFilesystem( + root: string, + sourceId: string, + agentId?: string, + managedPaths: readonly string[] = [], +): Promise { + const safePaths = managedPaths.filter( + (path) => path.startsWith("imports/transcripts/") && !path.includes("..") && !path.startsWith("/"), + ); + for (const managedPath of safePaths) await rm(join(root, managedPath), { force: true }); + await rm(join(root, "imports", "transcripts", sourceId), { recursive: true, force: true }); + const dir = join(root, "transcripts"); + let names: string[] = []; + try { + names = await readdir(dir); + } catch { + return; + } + for (const name of names.filter((entry) => entry.endsWith(".jsonl"))) { + const path = join(dir, name); + const input = await readText(path); + const kept = input + .split(/(?<=\n)/u) + .filter((line) => { + try { + const value = JSON.parse(line) as { source_id?: unknown; agent_id?: unknown }; + return !(value.source_id === sourceId && (agentId === undefined || value.agent_id === agentId)); + } catch { + return true; + } + }) + .join(""); + if (kept === input) continue; + const temp = `${path}.purge-${randomUUID()}`; + await writeFile(temp, kept, { flag: "w" }); + const fd = await open(temp, "r+"); + try { + await fd.sync(); + } finally { + await fd.close(); + } + await rename(temp, path); + } +} +export function importWorkerBatchSize(): number { + return TRANSCRIPT_IMPORT_LIMITS.maxRecordsPerBatch; +} +export function newImportLeaseToken(): string { + return randomUUID(); +} +export { controlImport, reconcileImport }; diff --git a/platform/daemon/src/transcript-recovery-supervisor.ts b/platform/daemon/src/transcript-recovery-supervisor.ts index fcab0b1b4b..b35e6e4e33 100644 --- a/platform/daemon/src/transcript-recovery-supervisor.ts +++ b/platform/daemon/src/transcript-recovery-supervisor.ts @@ -5,6 +5,7 @@ const TRANSCRIPT_RECOVERY_CHILD_GRACE_MS = 4_000; async function main(): Promise { const childPath = process.env.SIGNET_TRANSCRIPT_RECOVERY_CHILD_PATH; if (childPath === undefined) throw new Error("Transcript recovery supervisor child path is missing"); + const parentPid = process.ppid; const child = spawn(process.execPath, [childPath], { env: process.env, @@ -16,6 +17,7 @@ async function main(): Promise { if (child.pid !== undefined) process.stdout.write(`${JSON.stringify({ type: "started", pid: child.pid })}\n`); let killTimer: ReturnType | undefined; + let parentWatch: ReturnType | undefined; const killTarget = (): void => { if (child.pid === undefined) return; try { @@ -46,6 +48,18 @@ async function main(): Promise { killTimer = setTimeout(killTarget, TRANSCRIPT_RECOVERY_CHILD_GRACE_MS); } }; + const closeAndExit = (): never => { + if (parentWatch !== undefined) clearInterval(parentWatch); + killTarget(); + process.exit(0); + }; + parentWatch = setInterval(() => { + // SIGKILL bypasses daemon cleanup; the supervisor owns the detached child group. + if (process.ppid !== parentPid) closeAndExit(); + }, 50); + parentWatch.unref(); + process.stdin.once("end", closeAndExit); + process.stdin.once("close", closeAndExit); process.once("SIGTERM", () => forward("SIGTERM")); process.once("SIGINT", () => forward("SIGINT")); diff --git a/platform/daemon/src/transcript-recovery-worker.test.ts b/platform/daemon/src/transcript-recovery-worker.test.ts index 770144611a..6827e86e5e 100644 --- a/platform/daemon/src/transcript-recovery-worker.test.ts +++ b/platform/daemon/src/transcript-recovery-worker.test.ts @@ -1,3 +1,5 @@ +import { Database } from "bun:sqlite"; +import { spawn } from "node:child_process"; import { afterEach, beforeEach, describe, expect, it } from "bun:test"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -786,4 +788,71 @@ describe("transcript recovery worker", () => { ) as Array<{ transcript_path: string }>; expect(jobs.map((job) => job.transcript_path)).toEqual([firstPath, secondPath]); }); + + it("closes the recovery database when its daemon parent is killed", async () => { + const databasePath = join(dir, "recovery-lock.db"); + const childPath = join(dir, "locking-recovery-child.ts"); + const harnessPath = join(dir, "recovery-parent-harness.ts"); + const childPidPath = join(dir, "locking-child.pid"); + const supervisorPidPath = join(dir, "recovery-supervisor.pid"); + writeFileSync( + childPath, + [ + 'import { Database } from "bun:sqlite";', + 'import { writeFileSync } from "node:fs";', + `const db = new Database(${JSON.stringify(databasePath)});`, + 'db.exec("CREATE TABLE IF NOT EXISTS lifecycle_lock (value INTEGER)");', + 'db.exec("BEGIN IMMEDIATE");', + 'db.prepare("INSERT INTO lifecycle_lock (value) VALUES (1)").run();', + `writeFileSync(${JSON.stringify(childPidPath)}, String(process.pid));`, + "setInterval(() => {}, 1_000);", + ].join("\n"), + ); + const supervisorPath = join(dirname(fileURLToPath(import.meta.url)), "transcript-recovery-supervisor.ts"); + writeFileSync( + harnessPath, + [ + 'import { spawn } from "node:child_process";', + `const supervisor = spawn(process.execPath, [${JSON.stringify(supervisorPath)}], {`, + `env: { ...process.env, SIGNET_TRANSCRIPT_RECOVERY_CHILD_PATH: ${JSON.stringify(childPath)} },`, + 'stdio: ["pipe", "ignore", "ignore"],', + "});", + `require("node:fs").writeFileSync(${JSON.stringify(supervisorPidPath)}, String(supervisor.pid));`, + "setInterval(() => {}, 1_000);", + ].join("\n"), + ); + const parent = spawn(process.execPath, [harnessPath], { stdio: "ignore" }); + try { + for (let attempt = 0; attempt < 200 && !existsSync(childPidPath); attempt++) await Bun.sleep(5); + expect(existsSync(childPidPath)).toBe(true); + parent.kill("SIGKILL"); + let childAlive = true; + for (let attempt = 0; attempt < 200 && childAlive; attempt++) { + try { + process.kill(Number(readFileSync(childPidPath, "utf8")), 0); + } catch { + childAlive = false; + } + if (childAlive) await Bun.sleep(5); + } + expect(childAlive).toBe(false); + let supervisorAlive = true; + try { + process.kill(Number(readFileSync(supervisorPidPath, "utf8")), 0); + } catch { + supervisorAlive = false; + } + expect(supervisorAlive).toBe(false); + const database = new Database(databasePath); + try { + database.exec("PRAGMA busy_timeout = 1000"); + database.exec("BEGIN IMMEDIATE"); + database.exec("ROLLBACK"); + } finally { + database.close(); + } + } finally { + parent.kill("SIGKILL"); + } + }, 15_000); }); diff --git a/platform/daemon/src/transcript-recovery-worker.ts b/platform/daemon/src/transcript-recovery-worker.ts index 5de65561a7..dd9ee0f827 100644 --- a/platform/daemon/src/transcript-recovery-worker.ts +++ b/platform/daemon/src/transcript-recovery-worker.ts @@ -697,7 +697,7 @@ export function startTranscriptRecoveryWorker( SIGNET_TRANSCRIPT_RECOVERY_CHILD_PATH: childPath, SIGNET_TRANSCRIPT_RECOVERY_INPUT: JSON.stringify({ basePath, agentId, options: scanOptions }), }, - stdio: ["ignore", "pipe", "pipe"], + stdio: ["pipe", "pipe", "pipe"], }); activeChild = child; return await new Promise((resolve, reject) => { @@ -785,6 +785,7 @@ export function startTranscriptRecoveryWorker( timer = null; const scanToStop = activeScan; if (scanToStop === null) return; + activeChild?.stdin?.end(); signalActiveProcesses("SIGTERM"); let graceTimer: ReturnType | undefined; const stoppedGracefully = await Promise.race([ diff --git a/scripts/evals/transcript-import-real-daemon.ts b/scripts/evals/transcript-import-real-daemon.ts new file mode 100644 index 0000000000..bc88430f53 --- /dev/null +++ b/scripts/evals/transcript-import-real-daemon.ts @@ -0,0 +1,368 @@ +#!/usr/bin/env bun +/** Real-daemon acceptance eval for transcript import (#1814). */ +import { spawn, type ChildProcess } from "node:child_process"; +import { access, mkdir, mkdtemp, readdir, readlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { Database } from "bun:sqlite"; +import { ensureUnifiedSchema } from "../../platform/core/src/migration"; +import { runMigrations } from "../../platform/core/src/migrations/index"; + +const root = await mkdtemp(join("/mnt/work/hermes-scratch/", "1814-import-eval-")); +const port = 43000 + Math.floor(Math.random() * 1000); +const origin = `http://127.0.0.1:${port}`; +const agent = "eval-target-agent"; +const foreignAgent = "embedded-foreign-agent"; +const daemonScript = join(import.meta.dir, "../../platform/daemon/src/daemon.ts"); +let daemon: ChildProcess | undefined; +const stdout: string[] = [], + stderr: string[] = []; +const checks: Record = {}; +const details: Record = {}; + +function record(ok: boolean, name: string, detail?: unknown) { + checks[name] = ok; + if (detail !== undefined) details[name] = detail; + if (!ok) throw new Error(name); +} +function line( + input: Partial> & { + id: string; + session_key: string; + agent_id?: string; + messages?: unknown[]; + }, +) { + const messages = input.messages ?? [ + { role: "user", content: ` exact ${input.id}\nmultiline ` }, + { role: "assistant", content: "reply" }, + ]; + return JSON.stringify({ + source: "signet", + harness: input.harness ?? "claude", + agent_id: input.agent_id ?? foreignAgent, + session_key: input.session_key, + project: input.project ?? "project-a", + timestamp: input.timestamp ?? "2020-01-01T00:00:00.000Z", + message_count: input.message_count ?? messages.length, + messages, + id: input.id, + }); +} +function corpus(count: number, prefix: string) { + const rows: string[] = []; + for (let i = 0; i < count; i++) + rows.push( + line({ + id: `${prefix}-${i}`, + session_key: `${prefix}-session-${i}`, + harness: i % 2 ? "codex" : "claude", + project: i % 3 ? "project-a" : null, + agent_id: i % 5 === 0 ? foreignAgent : `${foreignAgent}-${i}`, + }), + ); + return `${rows.join("\n")}\n`; +} +async function req(path: string, init?: RequestInit) { + const r = await fetch(origin + path, { ...init, signal: AbortSignal.timeout(5000) }); + const text = await r.text(); + // biome-ignore lint/suspicious/noExplicitAny: dynamic daemon JSON envelope + let body: any; + try { + body = JSON.parse(text); + } catch { + body = text; + } + return { status: r.status, body }; +} +async function waitLive(child: ChildProcess) { + for (let i = 0; i < 300; i++) { + if (child.exitCode !== null) throw new Error(`daemon exited ${child.exitCode}: ${stderr.slice(-10).join("")}`); + try { + if ((await req("/health/live")).status === 200) return; + } catch {} + await Bun.sleep(100); + } + throw new Error("daemon did not become live"); +} +async function start(env: Record = {}) { + daemon = spawn(process.execPath, [daemonScript], { + cwd: join(import.meta.dir, "../.."), + env: { + ...process.env, + SIGNET_PATH: root, + SIGNET_PORT: String(port), + SIGNET_HOST: "127.0.0.1", + SIGNET_BIND: "127.0.0.1", + SIGNET_TELEMETRY_OPTOUT: "1", + SIGNET_DAEMON_ENTRYPOINT: "1", + SIGNET_AGENT_ID: agent, + // Keep native watcher discovery inside this eval workspace. The daemon's + // production defaults intentionally inspect the user's configured homes; + // inheriting the evaluator's HOME would contaminate agent/source counts. + HOME: root, + HERMES_HOME: join(root, ".hermes"), + XDG_CONFIG_HOME: join(root, ".config"), + XDG_DATA_HOME: join(root, ".local", "share"), + XDG_STATE_HOME: join(root, ".local", "state"), + ...env, + }, + stdio: ["ignore", "pipe", "pipe"], + }); + daemon.stdout?.on("data", (b) => stdout.push(String(b))); + daemon.stderr?.on("data", (b) => stderr.push(String(b))); + await waitLive(daemon); +} +async function stop(signal: NodeJS.Signals = "SIGKILL") { + if (daemon && daemon.exitCode === null) { + daemon.kill(signal); + for (let i = 0; i < 100 && daemon.exitCode === null; i++) await Bun.sleep(50); + } +} +async function waitForFailpoint(marker: string, expectedExit: number): Promise { + for (let i = 0; i < 1200; i++) { + let seen = false; + try { + await access(join(root, ".daemon", marker)); + seen = true; + } catch { + // The process may exit immediately after creating the durable marker. + } + if (seen && daemon?.exitCode === expectedExit) return; + if (daemon?.exitCode !== null && daemon?.exitCode !== expectedExit) + throw new Error(`failpoint exited before marker: ${daemon?.exitCode}`); + await Bun.sleep(100); + } + throw new Error(`failpoint marker/readback not observed: ${marker}`); +} +async function status(jobId: string) { + return (await req(`/api/sources/imports/${jobId}?agentId=${agent}`)).body; +} +async function importFile(text: string, name: string) { + const job = ( + await req(`/api/sources/imports?agentId=${agent}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ schemaId: "signet-export", files: [{ name }] }), + }) + ).body; + const fileId = (job.files?.[0] as { id: string } | undefined)?.id; + if (!fileId) throw new Error("import job did not reserve an upload file"); + const uploadPath = join(root, `.upload-${fileId}.jsonl`); + await writeFile(uploadPath, text, "utf8"); + const uploadProcess = Bun.spawnSync([ + "curl", + "-sS", + "--upload-file", + uploadPath, + "-H", + "content-type: application/jsonl", + "-H", + `x-file-name: ${name}`, + `${origin}/api/sources/imports/${job.jobId}/files/${fileId}?agentId=${agent}`, + ]); + const uploadText = new TextDecoder().decode(uploadProcess.stdout); + let uploadBody: { sourceId: string; [key: string]: unknown }; + try { + uploadBody = JSON.parse(uploadText) as { sourceId: string; [key: string]: unknown }; + } catch { + throw new Error( + `upload response was not JSON (${uploadProcess.exitCode}): ${uploadText || new TextDecoder().decode(uploadProcess.stderr)}`, + ); + } + const uploaded = { + status: uploadProcess.exitCode === 0 ? 201 : 599, + body: uploadBody, + }; + record(uploaded.status === 201, `upload-${name}`, uploaded.body); + record( + (await req(`/api/sources/imports/${job.jobId}/start?agentId=${agent}`, { method: "POST" })).status === 200, + `start-${name}`, + ); + return { jobId: job.jobId, sourceId: uploaded.body.sourceId }; +} +function dbRows(sql: string, params: unknown[] = []) { + const db = new Database(join(root, "memory", "memories.db"), { readonly: true }); + try { + // biome-ignore lint/suspicious/noExplicitAny: bun sqlite returns dynamic rows + return db.query(sql).all(...params) as any[]; + } finally { + db.close(); + } +} +function dbOne(sql: string, params: unknown[] = []) { + return dbRows(sql, params)[0] ?? {}; +} +async function waitCompleted(jobId: string) { + for (let i = 0; i < 600; i++) { + const j = await status(jobId); + if (["completed", "completed_with_rejections", "cancelled"].includes(j.job.state)) return j; + await Bun.sleep(100); + } + throw new Error(`job ${jobId} did not finish`); +} +async function waitForDatabaseOwnershipRelease(): Promise { + const dbPath = join(root, "memory", "memories.db"); + for (let i = 0; i < 200; i++) { + const holders: string[] = []; + for (const entry of await readdir("/proc")) { + if (!/^\d+$/.test(entry)) continue; + try { + let ownsByOpenFile = false; + for (const fd of await readdir(`/proc/${entry}/fd`)) { + try { + const target = await readlink(`/proc/${entry}/fd/${fd}`); + if (target === dbPath || target === `${dbPath} (deleted)`) { + ownsByOpenFile = true; + break; + } + } catch { + // The descriptor can close between enumeration and readback. + } + } + if (ownsByOpenFile) holders.push(entry); + } catch { + // The process can exit between /proc enumeration and readback. + } + } + if (holders.length === 0) return; + await Bun.sleep(50); + } + throw new Error("database ownership was not released within 10 seconds"); +} + +try { + await mkdir(join(root, ".daemon/logs"), { recursive: true }); + await mkdir(join(root, "memory"), { recursive: true }); + await writeFile(join(root, "agent.yaml"), "embedding:\n provider: none\n"); + const setupDb = new Database(join(root, "memory", "memories.db")); + try { + ensureUnifiedSchema(setupDb as unknown as Parameters[0]); + runMigrations(setupDb as unknown as Parameters[0]); + } finally { + setupDb.close(); + } + await start({ SIGNET_TRANSCRIPT_IMPORT_FAILPOINT: "inventory" }); + const mixed = corpus(2200, "large"); + const first = await importFile(mixed, "large.jsonl"); + await waitForFailpoint("transcript-import-inventory-failpoint-fired", 87); + record(daemon?.exitCode === 87, "kill-during-inventory", { exit: daemon?.exitCode }); + await waitForDatabaseOwnershipRelease(); + await start(); + const firstDone = await waitCompleted(first.jobId); + record(firstDone.job.imported === 2200, "inventory-restart-imported", firstDone.job); + record(firstDone.job.pending === 0, "inventory-restart-zero-pending"); + + const replay = await importFile(mixed, "replay.jsonl"); + const replayDone = await waitCompleted(replay.jobId); + record(replayDone.job.duplicate === 2200, "exact-replay-duplicates", replayDone.job); + const invalid = `${[ + line({ id: "unknown-role", session_key: "bad-1", messages: [{ role: "wat", content: "x" }], message_count: 1 }), + line({ id: "count-mismatch", session_key: "bad-2", message_count: 9 }), + "{not-json}", + " ", + "", + line({ id: "blank-content", session_key: "bad-3", messages: [{ role: "user", content: "" }], message_count: 1 }), + line({ + id: "oversize", + session_key: "bad-4", + messages: [{ role: "user", content: "x".repeat(4 * 1024 * 1024 + 1) }], + message_count: 1, + }), + ].join("\n")}\n`; + const rejected = await importFile(invalid, "rejections.jsonl"); + const rejectedDone = await waitCompleted(rejected.jobId); + record(rejectedDone.job.rejected === 5, "exact-rejection-count", rejectedDone.job); + record(rejectedDone.job.pending === 0, "rejection-zero-pending", rejectedDone.job); + + await stop(); + await waitForDatabaseOwnershipRelease(); + await start({ SIGNET_TRANSCRIPT_IMPORT_FAILPOINT: "after-fs-before-db" }); + const crash = await importFile(corpus(40, "crash"), "crash.jsonl"); + await waitForFailpoint("transcript-import-failpoint-fired", 86); + record(daemon?.exitCode === 86, "fs-before-db-failpoint", { exit: daemon?.exitCode, stderr: stderr.slice(-5) }); + await waitForDatabaseOwnershipRelease(); + await start(); + const crashDone = await waitCompleted(crash.jobId); + record(crashDone.job.imported === 40, "fs-replay-imported", crashDone.job); + const importedSessions = dbOne( + "SELECT COUNT(*) count FROM session_transcripts WHERE agent_id = ? AND source_id IS NOT NULL", + [agent], + ).count; + const importedIds = dbOne( + "SELECT COUNT(DISTINCT session_key) count FROM session_transcripts WHERE agent_id = ? AND source_id IS NOT NULL", + [agent], + ).count; + record(importedSessions === importedIds, "no-duplicate-session-record-ids", { importedSessions, importedIds }); + const foreign = dbOne("SELECT COUNT(*) count FROM session_transcripts WHERE agent_id = ?", [foreignAgent]).count; + record(foreign === 0, "embedded-agent-does-not-escape-scope", foreign); + const old = dbOne("SELECT COUNT(*) count FROM session_transcripts WHERE agent_id = ? AND created_at LIKE '2020-%'", [ + agent, + ]).count; + record(old > 0, "historical-timestamps-preserved", old); + const canonicalFiles = (await readdir(join(root, "transcripts"))).filter((n) => n.endsWith(".jsonl")); + record(canonicalFiles.length >= 2, "canonical-harness-files", canonicalFiles); + + const reconciliation = await req(`/api/sources/imports/${first.jobId}/reconciliation?agentId=${agent}`); + record(reconciliation.status === 200, "reconciliation-route", reconciliation.body); + const reconRows = reconciliation.body.reconciliation ?? []; + const totals = dbOne("SELECT COUNT(*) count FROM source_import_records WHERE job_id = ?", [first.jobId]).count; + const terminal = dbOne( + "SELECT COUNT(*) count FROM source_import_records WHERE job_id = ? AND status IN ('imported','duplicate','rejected')", + [first.jobId], + ).count; + record(totals === terminal, "exact-reconciliation-equation", { totals, terminal }); + const healthTimes: number[] = []; + for (let i = 0; i < 20; i++) { + const t = performance.now(); + record((await req("/health/live")).status === 200, `health-${i}`); + healthTimes.push(performance.now() - t); + } + healthTimes.sort((a, b) => a - b); + details.healthP95Ms = Math.round(healthTimes[Math.floor(healthTimes.length * 0.95)] ?? 0); + record((details.healthP95Ms as number) < 500, "health-bounded", details.healthP95Ms); + const pendingBefore = dbOne( + "SELECT COUNT(*) count FROM source_import_records WHERE agent_id = ? AND status = 'pending'", + [agent], + ).count; + record(pendingBefore === 0, "zero-pending", pendingBefore); + const sourceRowsBefore = dbOne( + "SELECT COUNT(*) count FROM session_transcripts WHERE agent_id = ? AND source_id = ?", + [agent, first.sourceId], + ).count; + record(sourceRowsBefore > 0, "source-owned-session-rows", sourceRowsBefore); + const removed = await req(`/api/sources/${encodeURIComponent(first.sourceId)}?agentId=${agent}`, { + method: "DELETE", + }); + record(removed.status === 200, "source-remove", removed.body); + const after = dbOne("SELECT COUNT(*) count FROM session_transcripts WHERE agent_id = ? AND source_id = ?", [ + agent, + first.sourceId, + ]).count; + record(after === 0, "source-purge-session-rows", after); + const tombstone = dbOne( + "SELECT COUNT(*) count FROM source_import_record_attempts WHERE agent_id = ? AND source_id = ?", + [agent, first.sourceId], + ).count; + record(tombstone > 0, "audit-tombstones-retained", tombstone); + details.jobs = [firstDone.job, replayDone.job, rejectedDone.job, crashDone.job]; + details.reconciliation = reconRows; + const result = { verdict: "pass", checks, details, workspace: root, port }; + await writeFile("/mnt/work/hermes-scratch/1814-eval-result.json", JSON.stringify(result, null, 2)); + console.log(JSON.stringify(result, null, 2)); +} catch (error) { + const result = { + verdict: "fail", + error: error instanceof Error ? error.message : String(error), + checks, + details, + workspace: root, + port, + stderr: stderr.slice(-20), + stdout: stdout.slice(-20), + }; + await writeFile("/mnt/work/hermes-scratch/1814-eval-result.json", JSON.stringify(result, null, 2)); + console.log(JSON.stringify(result, null, 2)); + process.exitCode = 1; +} finally { + await stop("SIGTERM"); +} diff --git a/scripts/event-loop-contract-baseline.json b/scripts/event-loop-contract-baseline.json index 1db75f73bd..6657057266 100644 --- a/scripts/event-loop-contract-baseline.json +++ b/scripts/event-loop-contract-baseline.json @@ -333,168 +333,168 @@ }, { "path": "daemon.ts", - "line": 604, + "line": 611, "api": "existsSync", "source": "if (!existsSync(agentsMdPath)) return;", "category": "hot-path" }, { "path": "daemon.ts", - "line": 621, + "line": 628, "api": "existsSync", "source": "const existingFiles = files.filter((f) => existsSync(join(AGENTS_DIR, f.name)));", "category": "hot-path" }, { "path": "daemon.ts", - "line": 649, + "line": 656, "api": "existsSync", "source": "if (!existsSync(identityPath)) return \"\";", "category": "hot-path" }, { "path": "daemon.ts", - "line": 668, + "line": 675, "api": "existsSync", "source": "if (activeHarnesses.has(\"opencode\") && existsSync(opencodeDir)) {", "category": "hot-path" }, { "path": "daemon.ts", - "line": 700, + "line": 707, "api": "existsSync", "source": "if (activeHarnesses.has(\"opencode\") && existsSync(opencodeDir)) {", "category": "hot-path" }, { "path": "daemon.ts", - "line": 707, + "line": 714, "api": "existsSync", "source": "if (existsSync(geminiDir)) {", "category": "hot-path" }, { "path": "daemon.ts", - "line": 712, + "line": 719, "api": "existsSync", "source": "if (existsSync(settingsPath)) {", "category": "hot-path" }, { "path": "daemon.ts", - "line": 713, + "line": 720, "api": "readFileSync", "source": "const parsed = JSON.parse(readFileSync(settingsPath, \"utf8\"));", "category": "hot-path" }, { "path": "daemon.ts", - "line": 733, + "line": 740, "api": "existsSync", "source": "if (!existsSync(targetPath)) continue;", "category": "hot-path" }, { "path": "daemon.ts", - "line": 1298, + "line": 1305, "api": "existsSync", "source": "if (!existsSync(memoryDir)) {", "category": "hot-path" }, { "path": "daemon.ts", - "line": 1656, + "line": 1669, "api": "existsSync", "source": "if (!existsSync(p)) continue;", "category": "hot-path" }, { "path": "daemon.ts", - "line": 1658, + "line": 1671, "api": "readFileSync", "source": "const yaml = parseSimpleYaml(readFileSync(p, \"utf-8\")) as Record;", "category": "hot-path" }, { "path": "daemon.ts", - "line": 1702, + "line": 1715, "api": "existsSync", "source": "if (!existsSync(path)) continue;", "category": "hot-path" }, { "path": "daemon.ts", - "line": 1704, + "line": 1717, "api": "readFileSync", "source": "const routing = parseRoutingConfig(parseSimpleYaml(readFileSync(path, \"utf-8\")));", "category": "hot-path" }, { "path": "daemon.ts", - "line": 2093, + "line": 2106, "api": "existsSync", "source": "if (existsSync(PID_FILE)) {", "category": "hot-path" }, { "path": "daemon.ts", - "line": 2095, + "line": 2108, "api": "unlinkSync", "source": "unlinkSync(PID_FILE);", "category": "hot-path" }, { "path": "daemon.ts", - "line": 2239, + "line": 2252, "api": "mkdirSync", "source": "mkdirSync(DAEMON_DIR, { recursive: true });", "category": "hot-path" }, { "path": "daemon.ts", - "line": 2240, + "line": 2253, "api": "mkdirSync", "source": "mkdirSync(LOG_DIR, { recursive: true });", "category": "hot-path" }, { "path": "daemon.ts", - "line": 2241, + "line": 2254, "api": "mkdirSync", "source": "mkdirSync(dirname(MEMORY_DB), { recursive: true });", "category": "hot-path" }, { "path": "daemon.ts", - "line": 2403, + "line": 2416, "api": "existsSync", "source": "const workerPath = existsSync(bundled)", "category": "hot-path" }, { "path": "daemon.ts", - "line": 2478, + "line": 2491, "api": "writeFileSync", "source": "writeFileSync(PID_FILE, process.pid.toString());", "category": "hot-path" }, { "path": "daemon.ts", - "line": 3212, + "line": 3250, "api": "existsSync", "source": "if (existsSync(healthStampPath)) {", "category": "hot-path" }, { "path": "daemon.ts", - "line": 3213, + "line": 3251, "api": "readFileSync", "source": "const prev = JSON.parse(readFileSync(healthStampPath, \"utf-8\"));", "category": "hot-path" }, { "path": "daemon.ts", - "line": 3216, + "line": 3254, "api": "writeFileSync", "source": "writeFileSync(", "category": "hot-path" @@ -844,42 +844,42 @@ }, { "path": "db-owner-worker.ts", - "line": 125, + "line": 126, "api": "readFileSync", "source": "return readFileSync(cancellationRegistryPath, \"utf8\").includes(`${jobId}\\n`);", "category": "hot-path" }, { "path": "db-owner-worker.ts", - "line": 153, + "line": 158, "api": "writeFileSync", "source": "writeFileSync(startupStarted, \"started\\n\");", "category": "hot-path" }, { "path": "db-owner-worker.ts", - "line": 155, + "line": 160, "api": "existsSync", "source": "while (!existsSync(startupRelease)) Atomics.wait(signal, 0, 0, 5);", "category": "hot-path" }, { "path": "db-owner-worker.ts", - "line": 161, + "line": 165, "api": "existsSync", "source": "if (sqlitePath !== undefined && existsSync(sqlitePath) && typeof Database.setCustomSQLite === \"function\") {", "category": "hot-path" }, { "path": "db-owner-worker.ts", - "line": 190, + "line": 200, "api": "writeFileSync", "source": "if (commitMarker !== undefined) writeFileSync(commitMarker, \"started\\n\");", "category": "hot-path" }, { "path": "db-owner-worker.ts", - "line": 904, + "line": 994, "api": "withReadDbAsync", "source": "const embedding = await getDbAccessor().withReadDbAsync(", "category": "hot-path", @@ -887,7 +887,7 @@ }, { "path": "db-owner-worker.ts", - "line": 937, + "line": 1027, "api": "withReadDbAsync", "source": "return await getDbAccessor().withReadDbAsync(", "category": "hot-path", @@ -895,21 +895,21 @@ }, { "path": "db-owner-worker.ts", - "line": 947, + "line": 1037, "api": "writeFileSync", "source": "writeFileSync(startedMarker, \"started\\n\");", "category": "hot-path" }, { "path": "db-owner-worker.ts", - "line": 948, + "line": 1038, "api": "existsSync", "source": "while (!existsSync(releaseMarker)) await new Promise((resolve) => setTimeout(resolve, 5));", "category": "hot-path" }, { "path": "db-owner-worker.ts", - "line": 1042, + "line": 1165, "api": "writeFileSync", "source": "if (activeFile) writeFileSync(activeFile, `${Date.now()}\\n`);", "category": "hot-path" @@ -4413,35 +4413,35 @@ }, { "path": "routes/sources-routes.ts", - "line": 824, + "line": 827, "api": "existsSync", "source": "if (!existsSync(path)) return [];", "category": "hot-path" }, { "path": "routes/sources-routes.ts", - "line": 826, + "line": 829, "api": "readFileSync", "source": "const parsed = JSON.parse(readFileSync(path, \"utf8\")) as unknown;", "category": "hot-path" }, { "path": "routes/sources-routes.ts", - "line": 836, + "line": 839, "api": "mkdirSync", "source": "mkdirSync(dirname(path), { recursive: true });", "category": "hot-path" }, { "path": "routes/sources-routes.ts", - "line": 838, + "line": 841, "api": "writeFileSync", "source": "writeFileSync(tmp, `${JSON.stringify(tombstones, null, 2)}\\n`, \"utf8\");", "category": "hot-path" }, { "path": "routes/sources-routes.ts", - "line": 839, + "line": 842, "api": "renameSync", "source": "renameSync(tmp, path);", "category": "hot-path" diff --git a/surfaces/cli/src/cli.ts b/surfaces/cli/src/cli.ts index 9335edcb84..4a9e63fe7d 100644 --- a/surfaces/cli/src/cli.ts +++ b/surfaces/cli/src/cli.ts @@ -1070,7 +1070,7 @@ async function ensureDaemonForSecrets(): Promise { return ensureDaemonRunning(isDaemonRunning); } -const { fetchFromDaemon, fetchDaemonResult, fetchDaemonStream, secretApiCall } = createDaemonClient( +const { fetchFromDaemon, fetchDaemonResult, fetchDaemonStream, fetchDaemonRaw, secretApiCall } = createDaemonClient( DEFAULT_PORT, AGENTS_DIR, ); @@ -1108,6 +1108,8 @@ registerSkillCommands(program, { registerSourcesCommands(program, { agentsDir: AGENTS_DIR, secretApiCall, + fetchDaemonResult, + fetchDaemonRaw, }); registerMcpCommands(program, { diff --git a/surfaces/cli/src/commands/sources.ts b/surfaces/cli/src/commands/sources.ts index 4d2d844952..e6d64a3ce4 100644 --- a/surfaces/cli/src/commands/sources.ts +++ b/surfaces/cli/src/commands/sources.ts @@ -1,3 +1,4 @@ +import { createReadStream } from "node:fs"; import type { SignetSourceEntry } from "@signet/core"; import type { Command } from "commander"; import { @@ -8,13 +9,20 @@ import { addObsidianVaultSource, exportConfiguredSourceSnapshot, importConfiguredSourceSnapshot, + controlTranscriptImport, + importTranscriptFiles, listSources, removeConfiguredSource, } from "../features/sources.js"; -import type { DaemonApiCall } from "../lib/daemon.js"; +import type { DaemonApiCall, DaemonFetchResult, DaemonStreamResult } from "../lib/daemon.js"; export interface RegisterSourcesCommandsDeps extends SourcesDeps { readonly secretApiCall?: DaemonApiCall; + readonly fetchDaemonResult?: ( + path: string, + opts?: RequestInit & { timeout?: number }, + ) => Promise>; + readonly fetchDaemonRaw?: (path: string, opts?: RequestInit & { timeout?: number }) => Promise; } function collect(value: string, previous: string[]): string[] { @@ -24,6 +32,71 @@ function collect(value: string, previous: string[]): string[] { export function registerSourcesCommands(program: Command, deps: RegisterSourcesCommandsDeps): void { const sources = program.command("sources").description("Manage external read-only knowledge sources"); + const importCommand = sources + .command("import ") + .description("Stage Agent transcript JSONL files as a durable import job") + .requiredOption("--kind ", "Import kind (transcripts)") + .requiredOption("--schema ", "Input schema (signet)") + .requiredOption("--agent ", "Target agent id") + .option("--json", "Emit daemon-shaped JSON"); + importCommand.action((files: string[], options: { kind: string; schema: string; agent: string; json?: boolean }) => + importTranscriptFiles(files, options, { + createDaemonImport: deps.fetchDaemonResult + ? async (body) => { + const result = await deps.fetchDaemonResult?.( + `/api/sources/imports?agentId=${encodeURIComponent(options.agent)}`, + { method: "POST", body: JSON.stringify(body), headers: { "Content-Type": "application/json" } }, + ); + return result as DaemonFetchResult; + } + : undefined, + uploadDaemonImportFile: deps.fetchDaemonRaw + ? async (jobId, fileId, fileName) => { + const result = await deps.fetchDaemonRaw?.( + `/api/sources/imports/${encodeURIComponent(jobId)}/files/${encodeURIComponent(fileId)}?agentId=${encodeURIComponent(options.agent)}`, + { + method: "PUT", + headers: { "Content-Type": "application/jsonl", "x-file-name": fileName }, + body: createReadStream(fileName) as unknown as BodyInit, + ...({ duplex: "half" } as unknown as RequestInit), + }, + ); + return result as DaemonStreamResult; + } + : undefined, + fetchDaemonImport: deps.fetchDaemonResult + ? async (path: string, opts?: RequestInit) => { + const result = await deps.fetchDaemonResult?.( + `${path}${path.includes("?") ? "&" : "?"}agentId=${encodeURIComponent(options.agent)}`, + opts, + ); + return result as DaemonFetchResult; + } + : undefined, + }), + ); + + const imports = sources.command("imports"); + + for (const action of ["list", "status", "pause", "resume", "retry", "cancel"] as const) { + const command = imports + .command(action === "list" ? "list" : `${action} `) + .requiredOption("--agent ", "Target agent id") + .option("--watch", "Poll until terminal (status only)") + .option("--json", "Emit daemon-shaped JSON"); + command.action((jobId: string | undefined, options: { agent: string; watch?: boolean; json?: boolean }) => + controlTranscriptImport(action, jobId, options, { + fetchDaemonImport: deps.fetchDaemonResult + ? async (path: string, opts?: RequestInit) => + deps.fetchDaemonResult?.( + `${path}${path.includes("?") ? "&" : "?"}agentId=${encodeURIComponent(options.agent)}`, + opts, + ) as Promise> + : undefined, + }), + ); + } + sources .command("list") .description("List configured external sources") diff --git a/surfaces/cli/src/features/sources.ts b/surfaces/cli/src/features/sources.ts index 6073f60017..3e0fbf2566 100644 --- a/surfaces/cli/src/features/sources.ts +++ b/surfaces/cli/src/features/sources.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import { readFileSync, writeFileSync } from "node:fs"; import { type AddDiscordSourceInput, @@ -11,7 +12,113 @@ import { } from "@signet/core"; import chalk from "chalk"; -export interface SourcesDeps { +import type { DaemonFetchResult, DaemonStreamResult } from "../lib/daemon.js"; + +export interface SourceImportDeps { + readonly createDaemonImport?: (body: unknown) => Promise>; + readonly uploadDaemonImportFile?: (jobId: string, fileId: string, fileName: string) => Promise; + readonly fetchDaemonImport?: (path: string, opts?: RequestInit) => Promise>; +} + +export interface SourceImportCreateResponse { + readonly id?: string; + readonly jobId?: string; + readonly files?: readonly { readonly id?: string; readonly name?: string }[]; + readonly [key: string]: unknown; +} + +export interface SourceImportOptions { + readonly kind?: string; + readonly schema?: string; + readonly agent?: string; + readonly json?: boolean; +} + +export async function importTranscriptFiles( + files: readonly string[], + options: SourceImportOptions, + deps: SourceImportDeps, +): Promise { + if (options.kind !== "transcripts" || options.schema !== "signet" || !options.agent) { + return failImport("--kind transcripts, --schema signet, and --agent are required"); + } + if (!deps.createDaemonImport || !deps.uploadDaemonImportFile || !deps.fetchDaemonImport) { + return failImport("transcript imports require the Signet daemon API"); + } + const descriptors = files.map((name) => ({ id: randomUUID(), name })); + const created = await deps.createDaemonImport({ schemaId: "signet-export", files: descriptors }); + if (!created.ok) return failImport(created.error ?? "could not create import job"); + const jobId = created.data.jobId ?? created.data.id; + if (!jobId) return failImport("daemon returned no import job id"); + printImportOutput({ ...created.data, id: jobId, jobId }, options.json); + for (const [index, file] of files.entries()) { + const fileId = created.data.files?.[index]?.id ?? descriptors[index]?.id; + if (!fileId) return failImport(`daemon returned no file id for ${file}`); + const uploaded = await deps.uploadDaemonImportFile(jobId, fileId, file); + if (!uploaded.ok) { + console.error( + `Import job ${jobId} remains inspectable; upload failed for ${file}: ${uploaded.error ?? uploaded.reason}`, + ); + process.exitCode = 1; + return; + } + } + const started = await deps.fetchDaemonImport( + `/api/sources/imports/${encodeURIComponent(jobId)}/start`, + ); + if (!started.ok) return failImport(started.error ?? "could not start import job"); + printImportOutput({ ...started.data, id: jobId, jobId }, options.json); +} + +export async function controlTranscriptImport( + action: "list" | "status" | "pause" | "resume" | "retry" | "cancel", + jobId: string | undefined, + options: SourceImportOptions & { watch?: boolean }, + deps: SourceImportDeps, +): Promise { + if (!deps.fetchDaemonImport) return failImport("transcript imports require the Signet daemon API"); + const path = + action === "list" + ? "/api/sources/imports" + : `/api/sources/imports/${encodeURIComponent(jobId ?? "")}${action === "status" ? "" : `/${action}`}`; + if (options.watch && action === "status") { + for (let attempt = 0; attempt < 120; attempt += 1) { + const result = await deps.fetchDaemonImport( + path, + action === "status" ? undefined : { method: "POST" }, + ); + if (!result.ok) return failImport(result.error ?? "status request failed"); + printImportOutput({ ...result.data, id: jobId, jobId }, options.json); + const state = result.data.state; + if (state === "completed" || state === "completed_with_rejections" || state === "cancelled" || state === "failed") + return; + await new Promise((resolve) => setTimeout(resolve, 500)); + } + return failImport("watch stopped after 120 polls"); + } + const result = await deps.fetchDaemonImport( + path, + action === "status" || action === "list" ? undefined : { method: "POST" }, + ); + if (!result.ok) return failImport(result.error ?? "import request failed"); + printImportOutput({ ...result.data, ...(jobId ? { id: jobId, jobId } : {}) }, options.json); +} + +function printImportOutput(value: unknown, json = false): void { + if (json) console.log(JSON.stringify(value)); + else + console.log( + typeof value === "object" && value !== null && "jobId" in value + ? `Import job ${(value as { jobId?: string }).jobId}` + : JSON.stringify(value), + ); +} +function failImport(message: string): void { + console.error(chalk.red(`✗ ${message}`)); + process.exitCode = 1; +} + +export interface SourcesDeps extends SourceImportDeps { readonly agentsDir: string; readonly addDiscordSourceToDaemon?: (input: AddDiscordSourceInput) => Promise; readonly addGitHubSourceToDaemon?: (input: AddGitHubSourceInput) => Promise; diff --git a/surfaces/cli/src/lib/daemon.ts b/surfaces/cli/src/lib/daemon.ts index 4f096910cb..abdb60deed 100644 --- a/surfaces/cli/src/lib/daemon.ts +++ b/surfaces/cli/src/lib/daemon.ts @@ -110,6 +110,8 @@ export function createDaemonClient( opts?: RequestInit & { timeout?: number }, ) => Promise>; readonly fetchDaemonStream: (path: string, opts?: RequestInit & { timeout?: number }) => Promise; + /** Raw request transport for streaming uploads; never serializes the body. */ + readonly fetchDaemonRaw: (path: string, opts?: RequestInit & { timeout?: number }) => Promise; readonly secretApiCall: DaemonApiCall; } { const url = resolveDaemonClientUrl(port, agentsDir); @@ -225,6 +227,7 @@ export function createDaemonClient( fetchFromDaemon, fetchDaemonResult, fetchDaemonStream, + fetchDaemonRaw: fetchDaemonStream, secretApiCall, }; } diff --git a/surfaces/dashboard/src/components/sources/connect-source-dialog.tsx b/surfaces/dashboard/src/components/sources/connect-source-dialog.tsx index c39f13a154..18e051d563 100644 --- a/surfaces/dashboard/src/components/sources/connect-source-dialog.tsx +++ b/surfaces/dashboard/src/components/sources/connect-source-dialog.tsx @@ -6,19 +6,25 @@ import { sourceLogo } from "@/components/icons"; import { type ImportSourcesResponse, api } from "@/lib/api"; import { cn } from "@/lib/utils"; -import { FolderOpen, Loader2, RotateCcw, Upload, X } from "lucide-react"; +import { FolderOpen, Loader2, MessageSquare, RotateCcw, Upload, X } from "lucide-react"; import { useEffect, useRef, useState } from "react"; -type SourceKind = "files" | "obsidian" | "github" | "discord"; +type SourceKind = "files" | "transcripts" | "obsidian" | "github" | "discord"; const KINDS: readonly { id: SourceKind; label: string; namePlaceholder: string }[] = [ { id: "files", label: "Files", namePlaceholder: "" }, + { id: "transcripts", label: "Agent transcripts (JSONL)", namePlaceholder: "Imported agent transcripts" }, { id: "obsidian", label: "Obsidian", namePlaceholder: "Research Vault" }, { id: "github", label: "GitHub", namePlaceholder: "Signet GitHub" }, { id: "discord", label: "Discord", namePlaceholder: "Team Discord" }, ]; const FIELD: Record, { label: string; placeholder: string; hint: string }> = { + transcripts: { + label: "Target agent", + placeholder: "Select an agent", + hint: "Embedded agent_id is provenance only; the selected target owns this import.", + }, obsidian: { label: "Vault path", placeholder: "/home/nicholai/Notes/Research", @@ -38,9 +44,10 @@ const FIELD: Record, { label: string; placeholder: function validate(kind: Exclude, target: string, tokenRef: string): string | null { const value = target.trim(); + if (kind === "transcripts") return value ? null : "Target agent is required"; if (kind === "obsidian") { if (!value) return "Vault path is required"; - if (!/^(?:[A-Za-z]:[\\/]|[\/])/.test(value)) return "Vault path must be absolute"; + if (!/^(?:[A-Za-z]:[\\/]|[/])/.test(value)) return "Vault path must be absolute"; return null; } if (kind === "github") { @@ -54,6 +61,7 @@ function validate(kind: Exclude, target: string, tokenRef: function KindIcon({ kind }: { readonly kind: SourceKind }) { if (kind === "files") return ; + if (kind === "transcripts") return ; return sourceLogo(kind, { className: "size-6" }); } @@ -78,6 +86,15 @@ export function ConnectSourceDialog({ const [browsing, setBrowsing] = useState(false); const [error, setError] = useState(null); const [result, setResult] = useState(null); + const [transcriptJobId, setTranscriptJobId] = useState(null); + const [agents, setAgents] = useState([]); + + useEffect(() => { + if (!open) return; + void api.getAgents().then((response) => { + if (response.data) setAgents(response.data.agents.map((agent) => ({ id: agent.id, name: agent.name }))); + }); + }, [open]); useEffect(() => { if (!open) return; @@ -115,6 +132,55 @@ export function ConnectSourceDialog({ }; const importFiles = async (targetFiles: readonly File[], targetPaths: readonly string[] = []) => { + if (kind === "transcripts") { + if (!targetFiles.length) { + setError("Choose one or more JSONL transcript files"); + return; + } + const selectedAgent = target.trim(); + const problem = validate(kind, selectedAgent, ""); + if (problem) { + setError(problem); + return; + } + setBusy(true); + setError(null); + const created = await api.createSourceImport(selectedAgent, targetFiles); + if (!created.data) { + setBusy(false); + setError(created.error ?? "Could not create import job"); + return; + } + const jobId = created.data.jobId ?? created.data.id; + if (!jobId) { + setBusy(false); + setError("Daemon returned no job ID"); + return; + } + setTranscriptJobId(jobId); + for (const [index, file] of targetFiles.entries()) { + const fileId = created.data.files?.[index]?.id; + if (!fileId) { + setBusy(false); + setError(`Job ${jobId}: daemon returned no file id for ${file.name}`); + return; + } + const uploaded = await api.uploadSourceImportFile(selectedAgent, jobId, fileId, file); + if (uploaded.error) { + setBusy(false); + setError(`Job ${jobId}: ${uploaded.error}`); + return; + } + } + const started = await api.controlSourceImport(jobId, "start"); + setBusy(false); + if (!started) { + setError(`Job ${jobId}: could not start import`); + return; + } + onConnected(); + return; + } if (targetFiles.length === 0 && targetPaths.length === 0) return; if (busy) return; setBusy(true); @@ -189,7 +255,7 @@ export function ConnectSourceDialog({ }; const submit = () => { - if (kind === "files") void importFiles(files, desktopPaths); + if (kind === "files" || kind === "transcripts") void importFiles(files, desktopPaths); else void connect(); }; @@ -198,14 +264,11 @@ export function ConnectSourceDialog({ return (
{ if (event.target === event.currentTarget && !busy) onClose(); }} - onKeyDown={(event) => { - if (event.key === "Escape" && !busy) onClose(); - }} >
@@ -235,7 +298,7 @@ export function ConnectSourceDialog({ ))}
- {kind === "files" ? ( + {kind === "files" || kind === "transcripts" ? ( <> diff --git a/surfaces/dashboard/src/lib/api.ts b/surfaces/dashboard/src/lib/api.ts index 5bc62ba1ca..dc422a751f 100644 --- a/surfaces/dashboard/src/lib/api.ts +++ b/surfaces/dashboard/src/lib/api.ts @@ -85,6 +85,23 @@ async function mutateJSON( } } +async function postSourceImport(agentId: string, files: readonly File[]): Promise> { + const descriptors = files.map((file) => ({ id: crypto.randomUUID(), name: file.name })); + try { + const res = await fetch(`${API_BASE}/api/sources/imports?agentId=${encodeURIComponent(agentId)}`, { + method: "POST", + headers: { "Content-Type": "application/json", Accept: "application/json", ...authHeaders() }, + body: JSON.stringify({ schemaId: "signet-export", files: descriptors }), + }); + const body = await res.json().catch(() => null); + return res.ok + ? { data: body as SourceImportJob, error: null } + : { data: null, error: typeof body?.error === "string" ? body.error : `request failed (${res.status})` }; + } catch { + return { data: null, error: "request failed" }; + } +} + // ── Types (derived from real daemon responses) ────────────────────────────── export interface ConfigFile { @@ -423,6 +440,32 @@ export interface SourceIndexJob { readonly error?: string; } +export interface SourceImportJob { + readonly id: string; + readonly jobId?: string; + readonly agent_id?: string; + readonly kind: "import"; + readonly schema_id?: string; + readonly state: string; + readonly total?: number; + readonly imported?: number; + readonly duplicate?: number; + readonly rejected?: number; + readonly pending?: number; + readonly files?: readonly SourceImportFile[]; +} +export interface SourceImportFile { + readonly id: string; + readonly name?: string; + readonly state?: string; + readonly size_bytes?: number; + readonly content_hash?: string; + readonly checkpoint_byte_offset?: number; +} +export interface SourceImportsResponse { + readonly imports: readonly SourceImportJob[]; +} + export interface SourcesResponse { version: number; sources: SignetSource[]; @@ -789,6 +832,47 @@ export const api = { // Sources getSources: () => getJSON("/api/sources"), + getSourceImports: () => getJSONResult("/api/sources/imports"), + getSourceImport: (jobId: string) => + getJSONResult<{ job: SourceImportJob; files: SourceImportFile[] }>( + `/api/sources/imports/${encodeURIComponent(jobId)}`, + ), + createSourceImport: (agentId: string, files: readonly File[]) => postSourceImport(agentId, files), + uploadSourceImportFile: async ( + agentId: string, + jobId: string, + fileId: string, + file: File, + ): Promise> => { + try { + const res = await fetch( + `${API_BASE}/api/sources/imports/${encodeURIComponent(jobId)}/files/${encodeURIComponent(fileId)}?agentId=${encodeURIComponent(agentId)}`, + { + method: "PUT", + headers: { "Content-Type": "application/jsonl", "x-file-name": file.name, ...authHeaders() }, + body: file.stream(), + }, + ); + const body = await res.json().catch(() => null); + return res.ok + ? { data: body, error: null } + : { data: null, error: body?.error ?? `request failed (${res.status})` }; + } catch { + return { data: null, error: "request failed" }; + } + }, + controlSourceImport: (jobId: string, action: "start" | "pause" | "resume" | "retry" | "cancel") => + postJSON<{ jobId: string; control: string; changed: boolean }>( + `/api/sources/imports/${encodeURIComponent(jobId)}/${action}`, + ), + getSourceImportRejections: (jobId: string) => + getJSON<{ jobId: string; rejections: readonly unknown[] }>( + `/api/sources/imports/${encodeURIComponent(jobId)}/rejections`, + ), + getSourceImportReconciliation: (jobId: string) => + getJSON<{ jobId: string; reconciliation: readonly unknown[] }>( + `/api/sources/imports/${encodeURIComponent(jobId)}/reconciliation`, + ), importSources: async ( files: readonly File[], duplicateMode: "skip" | "replace" | "reimport" = "skip", diff --git a/surfaces/dashboard/src/views/sources.tsx b/surfaces/dashboard/src/views/sources.tsx index a13f3304b5..56c1479c6e 100644 --- a/surfaces/dashboard/src/views/sources.tsx +++ b/surfaces/dashboard/src/views/sources.tsx @@ -1,11 +1,23 @@ import { sourceLogo } from "@/components/icons"; import { ConnectSourceDialog } from "@/components/sources/connect-source-dialog"; import { Surface } from "@/components/ui/surface"; -import { type SignetSource, type SourceHealth, type SourceIndexJob, api } from "@/lib/api"; +import { type SignetSource, type SourceHealth, type SourceImportJob, type SourceIndexJob, api } from "@/lib/api"; import { useAsync } from "@/lib/use-async"; import { cn } from "@/lib/utils"; import { useView } from "@/lib/view-context"; -import { Check, Copy, Download, Folder, GitBranch, Globe, Plus, RotateCw, Trash2, X } from "lucide-react"; +import { + Check, + Copy, + Download, + Folder, + GitBranch, + Globe, + MessageSquare, + Plus, + RotateCw, + Trash2, + X, +} from "lucide-react"; import { useEffect, useRef, useState } from "react"; const HEALTH_STYLES: Record = { @@ -26,6 +38,12 @@ function RootIcon({ kind }: { kind: string }) { export function SourcesView() { const { data, refresh } = useAsync(() => api.getSources(), { intervalMs: 30000 }); const sources = data?.sources; + const { data: importsData, refresh: refreshImports } = useAsync(() => api.getSourceImports(), { intervalMs: 5000 }); + const imports = importsData?.data?.imports ?? []; + const refreshAll = () => { + refresh(); + refreshImports(); + }; const [connectOpen, setConnectOpen] = useState(false); const { connectSourceRequested, clearConnectSource } = useView(); @@ -78,6 +96,9 @@ export function SourcesView() { Connect a source Obsidian · GitHub · Discord + {imports.map((job) => ( + + ))} {importedSources.length > 0 && } {connectedSources.map((source) => ( @@ -88,6 +109,91 @@ export function SourcesView() { ); } +function TranscriptImportCard({ job, onMutate }: { job: SourceImportJob; onMutate: () => void }) { + const { data: detail } = useAsync(() => api.getSourceImport(job.id), { intervalMs: 3000 }); + const current = detail?.data?.job ?? job; + const files = detail?.data?.files ?? job.files ?? []; + const terminal = ["completed", "completed_with_rejections", "cancelled", "failed"].includes(current.state); + const control = async (action: "pause" | "resume" | "retry" | "cancel") => { + await api.controlSourceImport(job.id, action); + onMutate(); + }; + return ( + +
+ +
+
Agent transcripts
+
job {current.id}
+
+ {current.state} +
+
+ + + + +
+ {files.length > 0 && ( +
    + {files.map((file) => ( +
  • + {file.name ?? file.id} + {file.state ?? "pending"} +
    • + ))} +
+ )} + {!terminal && ( +
+ {current.state === "paused" ? ( + void control("resume")}> + + + ) : ( + void control("pause")}> + + + )} + void control("retry")}> + + + void control("cancel")}> + + +
+ )} +
+ + +
+ + ); +} + +async function downloadImportData(jobId: string, kind: "rejections"): Promise { + const data = await api.getSourceImportRejections(jobId); + if (!data) return; + const url = URL.createObjectURL(new Blob([JSON.stringify(data, null, 2)], { type: "application/json" })); + const link = document.createElement("a"); + link.href = url; + link.download = `${jobId}-${kind}.json`; + link.click(); + URL.revokeObjectURL(url); +} + +async function showReconciliation(jobId: string): Promise { + const data = await api.getSourceImportReconciliation(jobId); + if (data) window.alert(JSON.stringify(data.reconciliation)); +} function ImportedDocumentsCard({ documents, onMutate }: { documents: readonly SignetSource[]; onMutate: () => void }) { const totals = documents.reduce( (acc, source) => ({ diff --git a/web/docs/src/content/docs/api/documents-sources.md b/web/docs/src/content/docs/api/documents-sources.md index a8f70a2edf..04db3d84a2 100644 --- a/web/docs/src/content/docs/api/documents-sources.md +++ b/web/docs/src/content/docs/api/documents-sources.md @@ -232,6 +232,100 @@ The default safety bounds are 25 files per request, 25 MiB per file, and 100 MiB per batch. Each file returns an individual result so a mixed batch can partially succeed. +### Durable transcript imports + +Transcript imports are a separate durable API; they do not use the synchronous +`POST /api/sources/import` document importer or `transcript_capture_jobs`. +`POST /api/sources/imports` creates a job, `PUT +/api/sources/imports/:jobId/files/:fileId` streams one JSONL file, and `POST +.../:jobId/start` queues it. `GET .../:jobId`, `.../rejections`, and +`.../reconciliation` report progress and outcomes. `pause`, `resume`, `retry`, +and `cancel` are durable controls. All routes require `modify` permission and +fail closed when `agentId` is not the daemon's resolved target agent. + +### POST /api/sources/imports + +Creates a durable transcript import job. + +### GET /api/sources/imports + +Lists transcript import jobs for the resolved agent. + +### GET /api/sources/imports/:jobId + +Returns one job and its staged files. + +### PUT /api/sources/imports/:jobId/files/:fileId + +Streams one JSONL file into the staged slot. + +### POST /api/sources/imports/:jobId/start + +Queues the staged job for inventory and commit. + +### POST /api/sources/imports/:jobId/pause + +Requests a pause at the next bounded worker checkpoint. + +### POST /api/sources/imports/:jobId/resume + +Resumes a paused job. + +### POST /api/sources/imports/:jobId/retry + +Retries interrupted or retryable records in a job. + +### POST /api/sources/imports/:jobId/cancel + +Cancels a job and terminalizes remaining pending records. + +### GET /api/sources/imports/:jobId/rejections + +Lists rejected records and bounded rejection codes. + +### GET /api/sources/imports/:jobId/reconciliation + +Returns the durable status-count reconciliation. + +The only accepted adapter is `signet-export` version `1` (`source: "signet"`). +Each line is classified exactly once as `imported`, `duplicate`, or `rejected`; +blank lines are counted separately. Validation rejects malformed JSON, unknown +roles, count mismatches, missing/nonempty-invalid messages, invalid timestamps, +and records over 16 MiB or messages over 4 MiB. The hard limits are 25 records +per database batch, 8 MiB canonical batch, 50,000 messages, and one active +job/file. The response counters satisfy `total = imported + duplicate + +rejected + pending` while a job is active and `pending = 0` when terminal. + +Imported transcripts preserve message roles (`user`, `assistant`, `system`, +`tool`, `unknown`), exact content whitespace and newlines, project, and the +historical timestamp. The selected target agent owns the rows; embedded +`agent_id` is provenance only. Source identity and content fingerprints make +same-identity replay a duplicate and same-identity/different-content a +`conversation_identity_conflict` rejection. Staging is an fsynced managed JSONL +file under `imports/transcripts//`; canonical harness files and +`session_transcripts` are written with deterministic IDs. Recovery resumes byte +checkpoints and replays filesystem writes idempotently before finalizing DB +ownership. + +Removing an import Source purges its staged file, canonical lines, +`session_transcripts`, artifacts, indexes, aggregates, and consumption/review +rows. Bounded record fingerprints and audit tombstones remain. Derived +knowledge is marked unsupported/stale and reviewed by the normal Dreaming path; +transcript import creates one attention nudge per committed source batch, not +one Dreaming job per conversation. + +**Create response** + +```json +{ "id": "job-uuid", "jobId": "job-uuid", "agentId": "target", "state": "staging" } +``` + +**Reconciliation response** + +```json +{ "jobId": "job-uuid", "reconciliation": [{ "status": "imported", "count": 42 }] } +``` + **Response** ```json diff --git a/web/docs/src/content/docs/architecture/pipeline-storage.md b/web/docs/src/content/docs/architecture/pipeline-storage.md index bc2d774c46..00886c9ac7 100644 --- a/web/docs/src/content/docs/architecture/pipeline-storage.md +++ b/web/docs/src/content/docs/architecture/pipeline-storage.md @@ -130,6 +130,17 @@ The table includes `session_key`, `content`, `harness`, `project`, `agent_id`, `created_at`, `updated_at`, `completed_at`, and `content_hash`. Reads and writes are agent-scoped. +Imported completed transcripts additionally carry `source_id`, +`source_record_id`, and bounded `source_meta_json`. The importer writes a +lossless typed message representation directly; it does not round-trip through +role-prefixed text. Its managed staged JSONL is immutable evidence, while +SQLite stores offsets, hashes, outcomes, and counters for restart. Canonical +filesystem append happens before DB-owner finalization and is idempotent, so a +restart after either inventory interruption or filesystem write recovery cannot +create duplicate session, record, or turn IDs. Source removal purges imported +evidence and consumption/review rows but retains bounded audit tombstones; +derived ontology is marked unsupported/stale for normal Dreaming review. + The memory recall endpoint supports `expand: true`. When requested, session keys associated with recalled results are batch-looked up in `session_transcripts`, and bounded transcript content is joined into the @@ -253,7 +264,7 @@ implementation. Signet uses SQLite in WAL mode. Migrations are numbered sequentially under `platform/core/src/migrations/`, run in order, and recorded in `schema_migrations` with checksum and timing data in -`schema_migrations_audit`. The latest migration is `128-bounded-queue-diagnostics.ts`. +`schema_migrations_audit`. The latest migration is `148-source-import-attempt-provenance.ts`. ### Evidence and semantic state diff --git a/web/docs/src/content/docs/cli/data-portability.md b/web/docs/src/content/docs/cli/data-portability.md index 6bcdc34f12..c1a896f22a 100644 --- a/web/docs/src/content/docs/cli/data-portability.md +++ b/web/docs/src/content/docs/cli/data-portability.md @@ -39,6 +39,33 @@ Transcript export writes JSONL to standard output by default, or to `--output`. Transcripts can contain credentials, personal data, and private project context. Review access controls before using an export for training or analysis. +### Importing exported transcripts + +`signet sources import --kind transcripts --schema signet --agent ` +streams the current `signet export transcripts` JSONL shape into a durable daemon +job. `signet sources imports list|status|pause|resume|retry|cancel` reads or +controls that job. The daemon records per-line hashes, byte checkpoints, +statuses, rejection codes, and reconciliation counters; it does not store raw +transcript bodies in the ledgers. + +The adapter is `signet-export` version `1`. It preserves exact whitespace, +multiline messages, `user`/`assistant`/`system`/`tool`/`unknown` roles, projects, +historical timestamps, and source provenance. `--agent` is the target scope; +an embedded `agent_id` is not an authorization or routing override. Same +identity plus same content is `duplicate`; same identity plus changed content is +`conversation_identity_conflict`. Blank lines are counted separately from +rejected lines. + +Limits are one active job/file, 25 records per DB batch, 8 MiB per canonical +batch, 16 MiB per record, 4 MiB per message, and 50,000 messages. States are +`staging`, `inventorying`, `queued`, `running`, `paused`, `completed`, +`completed_with_rejections`, and `cancelled`. Restart recovers leases and byte +offsets. The terminal reconciliation invariant is `total = imported + duplicate ++ rejected + pending` with `pending = 0`. Exact corpus replay produces +duplicates, not new evidence. Source removal purges staged/canonical/source +rows, while bounded audit fingerprints/tombstones remain; derived knowledge is +handled as unsupported/stale by the normal Dreaming review path. + ## Schema and vector migrations ```bash diff --git a/web/docs/src/content/docs/dashboard.md b/web/docs/src/content/docs/dashboard.md index 3a15c4eaf3..b718df86c1 100644 --- a/web/docs/src/content/docs/dashboard.md +++ b/web/docs/src/content/docs/dashboard.md @@ -54,6 +54,13 @@ The dashboard connect form intentionally exposes only a small set of fields. In For the source lifecycle and import behavior, see [Sources](/sources/). For HTTP request and response shapes, see [Documents and sources API](/api/documents-sources/). +The Sources dashboard distinguishes file-import job progress from Dreaming +attention and consumption. It shows the job id immediately, per-file states, +imported/duplicate/rejected/pending counts, bounded rejection details, and +reconciliation. Pause, resume, retry, and cancel call the daemon controls; the +browser does not maintain a second client-only queue. The target agent is +required, and any embedded transcript agent id is displayed as provenance only. + ## Serving behavior The daemon serves the built dashboard as a generic static SPA. Its dashboard handler passes `/api/*`, `/health`, and `/sse` through to their own handlers; remaining extensionless paths fall back to `index.html`. diff --git a/web/docs/src/content/docs/pipeline.md b/web/docs/src/content/docs/pipeline.md index ed14640227..f7965c18fd 100644 --- a/web/docs/src/content/docs/pipeline.md +++ b/web/docs/src/content/docs/pipeline.md @@ -20,4 +20,27 @@ Inference routing is configured through the canonical router workloads. The `mem - [Workers and maintenance](/pipeline/workers-maintenance/) - [Continuity and lineage](/pipeline/continuity-lineage/) +## Durable transcript import boundary + +Agent transcript imports are evidence ingestion, not a semantic pipeline stage. +The durable worker inventories streamed `signet-export` v1 JSONL by byte offset, +then commits typed completed-transcript DTOs in bounded batches. It preserves +roles, exact whitespace, multiline content, projects, historical timestamps, and +source provenance. Embedded agent ids never override the selected target scope. + +Each nonblank line has one durable outcome: `pending`, `imported`, `duplicate`, +or `rejected`; blank lines are counted separately. The worker uses one active +job/file, 25 records per database batch, an 8 MiB canonical batch, 16 MiB record +and 4 MiB message caps, and a 50,000-message cap. Job counters reconcile as +`total = imported + duplicate + rejected + pending`, and terminal jobs have no +pending records. Restart recovery reclaims leases and resumes byte checkpoints; +filesystem-first canonical writes are idempotent. Replaying the same export is +reported as duplicates, not as new evidence. + +Dreaming remains separate: a committed batch emits one attention nudge, then the +existing delivery, consumption, and review path determines pending Dreaming +work. Removing the source purges imported evidence, canonical lines, indexes, +and consumption rows while preserving bounded audit tombstones and routing +derived knowledge through unsupported/stale review. + For supported runtime configuration, use [Inference and routing](/configuration/inference-routing/) and [Pipeline configuration](/configuration/pipeline/). This section intentionally does not duplicate operator configuration. \ No newline at end of file diff --git a/web/docs/src/content/docs/sources.md b/web/docs/src/content/docs/sources.md index e314f12295..0e48f3777b 100644 --- a/web/docs/src/content/docs/sources.md +++ b/web/docs/src/content/docs/sources.md @@ -44,6 +44,44 @@ Choose **Import files**, select one or more files, choose how duplicates should The current bounds are 25 files per batch, 25 MiB per file, and 100 MiB per batch. Results are reported per file, so a malformed or unsupported file does not hide successful imports from the same batch. Choose **Skip duplicate** to keep the existing import, **Replace and re-index** to replace it and queue indexing again, or **Import as a new source** to retain a second source for the same content. +### Agent transcript imports + +Transcript JSONL imports use the durable Sources job, not the synchronous document +importer. The supported input is the Signet export schema (`signet-export`, +version `1`): one object per line with `source`, `id`, `harness`, `agent_id`, +`session_key`, `project`, `timestamp`, exact `message_count`, and typed +`messages`. Roles are `user`, `assistant`, `system`, `tool`, and `unknown`. +Whitespace, multiline content, roles, projects, timestamps, and provenance are +retained exactly. The chosen `--agent` owns the import; an embedded `agent_id` +does not change scope. + +```bash +signet sources import ./claude.jsonl ./codex.jsonl --kind transcripts --schema signet --agent my-agent --json +signet sources imports status --agent my-agent --watch --json +signet sources imports pause --agent my-agent +signet sources imports resume --agent my-agent +signet sources imports retry --agent my-agent +signet sources imports cancel --agent my-agent +``` + +The CLI creates the job before streaming each file and starts it after upload. +The daemon stages an immutable fsynced JSONL file under the workspace and keeps +only offsets, hashes, counters, outcomes, and bounded errors in SQLite. Limits +are one active job/file, 25 records per DB batch, 8 MiB per canonical batch, +16 MiB per record, 4 MiB per message, and 50,000 messages. Every nonblank line +is `pending`, then exactly one of `imported`, `duplicate`, or `rejected`. +Blank lines are counted separately. Same identity and content replays as a +duplicate; same identity with changed content is rejected as a conflict. + +Jobs move through `staging -> inventorying -> queued -> running`, may be +`paused`, and finish as `completed`, `completed_with_rejections`, or +`cancelled`. A restart recovers leases and byte checkpoints. Filesystem-first +canonical writes are verified and replayed without duplicate session, record, or +turn IDs. Reconciliation exposes the equation `total = imported + duplicate + +rejected + pending`; terminal jobs have zero pending. Import completion adds one +Dreaming attention nudge per committed source batch. Dreaming consumption is +separate and uses its normal delivery/review path. + In a desktop-local session, **Choose from desktop** can return local paths to a loopback daemon. Remote clients must upload file bytes; a remote daemon never treats a path string as permission to read the client’s filesystem. > TODO: add dark-mode screenshots for the Connect a source and Import files dialogs after capture against a controlled, non-private daemon fixture. This worktree has no such fixture, so no mock or private-path imagery is published here.