diff --git a/.gitignore b/.gitignore
index c57fefe6..ad50ed10 100644
--- a/.gitignore
+++ b/.gitignore
@@ -52,3 +52,20 @@ temp/
authentik-credentials.txt
*.credentials
*.secret
+*.key
+*.pem
+*.p12
+*.pfx
+*.sqlite
+*.db
+*.db-*
+*.session
+
+# Runtime-private Hermes/Kurultai/Brain artifacts
+.hermes/
+brain-index/
+brain-index-private/
+sessions/
+kanban.db
+config/runtime-config/live/
+config/runtime-config/private/
diff --git a/config/launchd/ai.kurultai.brain-public-gateway.plist b/config/launchd/ai.kurultai.brain-public-gateway.plist
new file mode 100644
index 00000000..4cca5034
--- /dev/null
+++ b/config/launchd/ai.kurultai.brain-public-gateway.plist
@@ -0,0 +1,49 @@
+
+
+
+
+ Label
+ ai.kurultai.brain-public-gateway
+ ProgramArguments
+
+ /Users/kublai/.brain-migration-venv/bin/python
+ -m
+ kublai.brain_service
+ --wiki-root
+ /Users/kublai/brain
+ --telemetry-db
+ /Users/kublai/.kublai/telemetry.db
+ --index-db
+ /Users/kublai/.brain-index/brain.db
+ serve-gateway
+ --host
+ 127.0.0.1
+ --port
+ 8765
+ --secret-file
+ /Users/kublai/.kublai/brain-public-gateway.hmac
+
+ WorkingDirectory
+ /Users/kublai/kurultai/kublai-repo
+ EnvironmentVariables
+
+ HOME
+ /Users/kublai
+ PYTHONPATH
+ /Users/kublai/kurultai/kublai-repo
+ PATH
+ /opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
+
+ RunAtLoad
+
+ KeepAlive
+
+ ThrottleInterval
+ 10
+ StandardOutPath
+ /Users/kublai/.openclaw/logs/brain-public-gateway.log
+ StandardErrorPath
+ /Users/kublai/.openclaw/logs/brain-public-gateway.err.log
+
+
diff --git a/config/runtime-config/README.md b/config/runtime-config/README.md
new file mode 100644
index 00000000..486870fd
--- /dev/null
+++ b/config/runtime-config/README.md
@@ -0,0 +1,29 @@
+# Runtime configuration home
+
+This folder is the repository home for sanitized Hermes, Kurultai, and Brain runtime configuration.
+
+It contains templates, rebuild manifests, and schema contracts, not live private runtime files.
+
+Live sources:
+
+- Hermes: `~/.hermes/config.yaml`, profile configs, cron jobs, sessions, Kanban DB.
+- Kurultai: native Hermes profiles, native Kanban, receipts, recovery workflows.
+- Brain: `/Users/kublai/brain`, QMD indexes, public gateway contract.
+
+Committed files:
+
+- `hermes.template.yaml`: sanitized Hermes runtime contract.
+- `profiles.yaml`: Kurultai profile roster and model/provider map.
+- `kurultai.yaml`: coordination contract.
+- `brain.yaml`: Brain root, index, and gateway contract.
+- `cron.manifest.json`: sanitized cron jobs.
+- `skills.manifest.json`: skill inventory without skill bodies.
+- `kanban.schema.json`: native Kanban schema without task data.
+- `brain.manifest.json`: Brain directory inventory without note contents.
+
+Refresh manifests from a live host with:
+
+- `python3 scripts/export_runtime_config_manifest.py`
+- `python3 scripts/export_rebuild_manifests.py`
+
+See `docs/operations/kurultai-rebuild-runbook.md` for the full rebuild procedure.
diff --git a/config/runtime-config/brain.manifest.json b/config/runtime-config/brain.manifest.json
new file mode 100644
index 00000000..cbd6895a
--- /dev/null
+++ b/config/runtime-config/brain.manifest.json
@@ -0,0 +1,135 @@
+{
+ "note": "Directory/file-count inventory only. Brain content and private indexes are not copied into this repo.",
+ "schema": "kurultai.brain-manifest.v1",
+ "top_level_directories": [
+ {
+ "markdown_files": 14,
+ "name": "_archive"
+ },
+ {
+ "markdown_files": 86,
+ "name": "analyses"
+ },
+ {
+ "markdown_files": 1,
+ "name": "apps"
+ },
+ {
+ "markdown_files": 0,
+ "name": "briefs"
+ },
+ {
+ "markdown_files": 2,
+ "name": "captures"
+ },
+ {
+ "markdown_files": 20,
+ "name": "concepts"
+ },
+ {
+ "markdown_files": 0,
+ "name": "config"
+ },
+ {
+ "markdown_files": 56,
+ "name": "content"
+ },
+ {
+ "markdown_files": 0,
+ "name": "dashboards"
+ },
+ {
+ "markdown_files": 44,
+ "name": "docs"
+ },
+ {
+ "markdown_files": 7,
+ "name": "entities"
+ },
+ {
+ "markdown_files": 1,
+ "name": "folds"
+ },
+ {
+ "markdown_files": 2,
+ "name": "generated"
+ },
+ {
+ "markdown_files": 7,
+ "name": "graphify-out"
+ },
+ {
+ "markdown_files": 38,
+ "name": "hard-private"
+ },
+ {
+ "markdown_files": 2,
+ "name": "inbox"
+ },
+ {
+ "markdown_files": 18,
+ "name": "infrastructure"
+ },
+ {
+ "markdown_files": 0,
+ "name": "knowledge"
+ },
+ {
+ "markdown_files": 1,
+ "name": "kublai"
+ },
+ {
+ "markdown_files": 0,
+ "name": "methodology"
+ },
+ {
+ "markdown_files": 0,
+ "name": "migrations"
+ },
+ {
+ "markdown_files": 1732,
+ "name": "operations"
+ },
+ {
+ "markdown_files": 45,
+ "name": "projects"
+ },
+ {
+ "markdown_files": 1,
+ "name": "proposals"
+ },
+ {
+ "markdown_files": 8,
+ "name": "published"
+ },
+ {
+ "markdown_files": 1,
+ "name": "queue"
+ },
+ {
+ "markdown_files": 32,
+ "name": "raw"
+ },
+ {
+ "markdown_files": 3,
+ "name": "receipts"
+ },
+ {
+ "markdown_files": 0,
+ "name": "research"
+ },
+ {
+ "markdown_files": 4,
+ "name": "runbooks"
+ },
+ {
+ "markdown_files": 1,
+ "name": "scripts"
+ },
+ {
+ "markdown_files": 2,
+ "name": "tests"
+ }
+ ],
+ "wiki_root": "/Users/kublai/brain"
+}
diff --git a/config/runtime-config/brain.yaml b/config/runtime-config/brain.yaml
new file mode 100644
index 00000000..44d9d46b
--- /dev/null
+++ b/config/runtime-config/brain.yaml
@@ -0,0 +1,25 @@
+brain:
+ wiki_root: "/Users/kublai/brain"
+ canonical_store: brain_service_sqlite
+ retired_graph_stack:
+ - neo4j
+ - cypher
+ - bolt
+ qmd:
+ update: "qmd update -c brain"
+ embed: "qmd embed -c brain"
+ public_index: "~/.brain-index/brain.db"
+ private_index: "~/.kublai/brain-index-private/brain.db"
+ gateway:
+ name: ai.kurultai.brain-public-gateway
+ bind: "127.0.0.1:8765"
+ exposure: public_only
+ operating_surfaces:
+ queue: "/Users/kublai/brain/queue"
+ generated: "/Users/kublai/brain/generated"
+ receipts: "/Users/kublai/brain/receipts"
+ plans: "/Users/kublai/brain/docs/plans"
+ session_learning:
+ raw_history: "~/.hermes/sessions"
+ recall: session_search
+ ingest_job: brain-ingest-hourly
diff --git a/config/runtime-config/cron.manifest.json b/config/runtime-config/cron.manifest.json
new file mode 100644
index 00000000..ec94f212
--- /dev/null
+++ b/config/runtime-config/cron.manifest.json
@@ -0,0 +1,520 @@
+{
+ "jobs": [
+ {
+ "deliver": "telegram:[REDACTED]",
+ "enabled": true,
+ "enabled_toolsets": null,
+ "job_id": "a3284f141980",
+ "name": "kurultai-weekly-improvement-scan",
+ "repeat": {
+ "completed": 2,
+ "times": null
+ },
+ "schedule": {
+ "display": "0 9 * * 1",
+ "expr": "0 9 * * 1",
+ "kind": "cron"
+ },
+ "script": null,
+ "skills": [
+ "kurultai-operations"
+ ],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "telegram:[REDACTED]",
+ "enabled": true,
+ "enabled_toolsets": null,
+ "job_id": "5bb06f714f5a",
+ "name": "brain-wiki-weekly-lint",
+ "repeat": {
+ "completed": 3,
+ "times": null
+ },
+ "schedule": {
+ "display": "0 18 * * 0",
+ "expr": "0 18 * * 0",
+ "kind": "cron"
+ },
+ "script": null,
+ "skills": [
+ "brain-wiki-operations"
+ ],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "telegram:[REDACTED]",
+ "enabled": true,
+ "enabled_toolsets": null,
+ "job_id": "7b72703dc472",
+ "name": "hermes-quota-daily-check",
+ "repeat": {
+ "completed": 18,
+ "times": null
+ },
+ "schedule": {
+ "display": "0 8 * * *",
+ "expr": "0 8 * * *",
+ "kind": "cron"
+ },
+ "script": null,
+ "skills": [],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "telegram:[REDACTED]",
+ "enabled": true,
+ "enabled_toolsets": null,
+ "job_id": "14ab808fd7a4",
+ "name": "brain-sync-drain",
+ "repeat": {
+ "completed": 73,
+ "times": null
+ },
+ "schedule": {
+ "display": "0 */6 * * *",
+ "expr": "0 */6 * * *",
+ "kind": "cron"
+ },
+ "script": null,
+ "skills": [],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "telegram:[REDACTED]",
+ "enabled": true,
+ "enabled_toolsets": null,
+ "job_id": "c01c8cd1e0bc",
+ "name": "kurultai-daily-queue-action",
+ "repeat": {
+ "completed": 27,
+ "times": null
+ },
+ "schedule": {
+ "display": "0 9 * * *",
+ "expr": "0 9 * * *",
+ "kind": "cron"
+ },
+ "script": null,
+ "skills": [
+ "kurultai-operations"
+ ],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "local",
+ "enabled": true,
+ "enabled_toolsets": [
+ "file",
+ "terminal",
+ "skills",
+ "session_search"
+ ],
+ "job_id": "ef6c8c68ad7c",
+ "name": "brain-ingest-hourly",
+ "repeat": {
+ "completed": 359,
+ "times": null
+ },
+ "schedule": {
+ "display": "every 60m",
+ "kind": "interval",
+ "minutes": 60
+ },
+ "script": null,
+ "skills": [
+ "brain-wiki-operations",
+ "kurultai-retro-learn"
+ ],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "local",
+ "enabled": true,
+ "enabled_toolsets": null,
+ "job_id": "574619205225",
+ "name": "kurultai-terminal-task-notifier",
+ "repeat": {
+ "completed": 7853,
+ "times": null
+ },
+ "schedule": {
+ "display": "every 2m",
+ "kind": "interval",
+ "minutes": 2
+ },
+ "script": "kurultai_terminal_task_notifier.py",
+ "skills": [],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "local",
+ "enabled": true,
+ "enabled_toolsets": null,
+ "job_id": "2607a2789990",
+ "name": "brain-graphify-weekly-refresh",
+ "repeat": {
+ "completed": 1,
+ "times": null
+ },
+ "schedule": {
+ "display": "30 8 * * 1",
+ "expr": "30 8 * * 1",
+ "kind": "cron"
+ },
+ "script": null,
+ "skills": [
+ "brain-wiki-operations"
+ ],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "local",
+ "enabled": true,
+ "enabled_toolsets": null,
+ "job_id": "kurultai-signal-intake-daily",
+ "name": "kurultai-signal-intake-every-30m",
+ "repeat": null,
+ "schedule": {
+ "display": "every 30m",
+ "kind": "interval",
+ "minutes": 30
+ },
+ "script": "kurultai_signal_intake.py",
+ "skills": [],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "local",
+ "enabled": true,
+ "enabled_toolsets": null,
+ "job_id": "kurultai-signal-feedback-apply",
+ "name": "kurultai-signal-feedback-apply",
+ "repeat": null,
+ "schedule": {
+ "display": "0 20 * * *",
+ "expr": "0 20 * * *",
+ "kind": "cron"
+ },
+ "script": "kurultai_signal_feedback.py",
+ "skills": [],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "local",
+ "enabled": true,
+ "enabled_toolsets": null,
+ "job_id": "kurultai-signal-weekly-metrics",
+ "name": "kurultai-signal-weekly-metrics",
+ "repeat": null,
+ "schedule": {
+ "display": "0 9 * * 1",
+ "expr": "0 9 * * 1",
+ "kind": "cron"
+ },
+ "script": "kurultai_signal_metrics.py",
+ "skills": [],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "origin",
+ "enabled": true,
+ "enabled_toolsets": [
+ "terminal",
+ "file",
+ "skills"
+ ],
+ "job_id": "c0dcb8edbbb7",
+ "name": "brain-optimizer-weekly",
+ "repeat": {
+ "completed": 1,
+ "times": 999999
+ },
+ "schedule": {
+ "display": "0 9 * * 0",
+ "expr": "0 9 * * 0",
+ "kind": "cron"
+ },
+ "script": null,
+ "skills": [
+ "brain-wiki-operations"
+ ],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "origin",
+ "enabled": true,
+ "enabled_toolsets": [
+ "terminal",
+ "file"
+ ],
+ "job_id": "b90907649d1e",
+ "name": "kurultai-research-hourly-summary",
+ "repeat": {
+ "completed": 174,
+ "times": 999999
+ },
+ "schedule": {
+ "display": "every 60m",
+ "kind": "interval",
+ "minutes": 60
+ },
+ "script": "kurultai_research_daily_summary.py",
+ "skills": [],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "origin",
+ "enabled": true,
+ "enabled_toolsets": [
+ "terminal",
+ "file",
+ "skills",
+ "cronjob",
+ "session_search"
+ ],
+ "job_id": "1091f6b0614b",
+ "name": "hermes-soul-operating-contract-monthly-audit",
+ "repeat": {
+ "completed": 1,
+ "times": 999999
+ },
+ "schedule": {
+ "display": "0 10 1 * *",
+ "expr": "0 10 1 * *",
+ "kind": "cron"
+ },
+ "script": null,
+ "skills": [
+ "kurultai-operations"
+ ],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "origin",
+ "enabled": true,
+ "enabled_toolsets": [
+ "file",
+ "terminal",
+ "skills",
+ "session_search"
+ ],
+ "job_id": "e6d26c53014e",
+ "name": "brain-content-daily-brief",
+ "repeat": {
+ "completed": 5,
+ "times": null
+ },
+ "schedule": {
+ "display": "0 6 * * *",
+ "expr": "0 6 * * *",
+ "kind": "cron"
+ },
+ "script": null,
+ "skills": [
+ "brain-wiki-operations",
+ "content-os-workflows"
+ ],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "origin",
+ "enabled": true,
+ "enabled_toolsets": [
+ "file",
+ "terminal",
+ "web",
+ "skills"
+ ],
+ "job_id": "8ffe58fe3e16",
+ "name": "brain-vault-weekly-synthesis",
+ "repeat": {
+ "completed": 1,
+ "times": null
+ },
+ "schedule": {
+ "display": "0 9 * * 0",
+ "expr": "0 9 * * 0",
+ "kind": "cron"
+ },
+ "script": null,
+ "skills": [
+ "brain-wiki-operations"
+ ],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "local",
+ "enabled": true,
+ "enabled_toolsets": [
+ "terminal"
+ ],
+ "job_id": "4500a64f8da1",
+ "name": "polymarket-weather-paper-scan",
+ "repeat": {
+ "completed": 128,
+ "times": null
+ },
+ "schedule": {
+ "display": "every 15m",
+ "kind": "interval",
+ "minutes": 15
+ },
+ "script": null,
+ "skills": [],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "origin",
+ "enabled": true,
+ "enabled_toolsets": [
+ "terminal"
+ ],
+ "job_id": "3a329e448164",
+ "name": "polymarket-weather-edge-verdict",
+ "repeat": {
+ "completed": 0,
+ "times": 1
+ },
+ "schedule": {
+ "display": "once at 2026-05-22 09:00",
+ "kind": "once",
+ "run_at": "2026-05-22T09:00:00-04:00"
+ },
+ "script": null,
+ "skills": [],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "local",
+ "enabled": true,
+ "enabled_toolsets": [
+ "terminal"
+ ],
+ "job_id": "cd970d242675",
+ "name": "polymarket-weather-labeler-and-render",
+ "repeat": {
+ "completed": 7,
+ "times": null
+ },
+ "schedule": {
+ "display": "every 360m",
+ "kind": "interval",
+ "minutes": 360
+ },
+ "script": null,
+ "skills": [],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "local",
+ "enabled": true,
+ "enabled_toolsets": [
+ "file",
+ "terminal"
+ ],
+ "job_id": "414cdf604452",
+ "name": "subc-room-walk-v0",
+ "repeat": {
+ "completed": 8,
+ "times": null
+ },
+ "schedule": {
+ "display": "0 */6 * * *",
+ "expr": "0 */6 * * *",
+ "kind": "cron"
+ },
+ "script": null,
+ "skills": [],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "origin",
+ "enabled": true,
+ "enabled_toolsets": [
+ "file",
+ "terminal"
+ ],
+ "job_id": "1fc07abc10a5",
+ "name": "subc-digest-v0",
+ "repeat": {
+ "completed": 7,
+ "times": null
+ },
+ "schedule": {
+ "display": "0 6,12,18 * * *",
+ "expr": "0 6,12,18 * * *",
+ "kind": "cron"
+ },
+ "script": null,
+ "skills": [],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "local",
+ "enabled": true,
+ "enabled_toolsets": [
+ "terminal"
+ ],
+ "job_id": "5f62f75e60e6",
+ "name": "kaidu-prompt-lab-review",
+ "repeat": {
+ "completed": 2,
+ "times": null
+ },
+ "schedule": {
+ "display": "0 8 * * *",
+ "expr": "0 8 * * *",
+ "kind": "cron"
+ },
+ "script": null,
+ "skills": [],
+ "state": "scheduled",
+ "workdir": null
+ },
+ {
+ "deliver": "local",
+ "enabled": true,
+ "enabled_toolsets": [
+ "terminal"
+ ],
+ "job_id": "2234e4abdcf5",
+ "name": "fantasy-sumo-live-ingestion",
+ "repeat": {
+ "completed": 18,
+ "times": null
+ },
+ "schedule": {
+ "display": "every 30m",
+ "kind": "interval",
+ "minutes": 30
+ },
+ "script": null,
+ "skills": [],
+ "state": "scheduled",
+ "workdir": "/Users/kublai/fantasy-sumo-league-site"
+ }
+ ],
+ "schema": "kurultai.cron-manifest.v1"
+}
diff --git a/config/runtime-config/hermes.template.yaml b/config/runtime-config/hermes.template.yaml
new file mode 100644
index 00000000..1b2f3f28
--- /dev/null
+++ b/config/runtime-config/hermes.template.yaml
@@ -0,0 +1,35 @@
+runtime:
+ identity: kublai
+ default_provider: openai-codex
+ default_model: gpt-5.5
+
+security:
+ parse_screening_enabled: true
+ protected_file_flow: hermes-self-mod
+ private_value_placeholder: "[REDACTED]"
+
+memory:
+ enabled: true
+ user_profile_enabled: true
+ session_history:
+ source_of_record: "~/.hermes/sessions"
+ retrieval: session_search
+ auto_prune: false
+
+coordination:
+ substrate: native-hermes
+ kanban_db: "~/.hermes/kanban.db"
+ profiles:
+ - kublai
+ - temujin
+ - mongke
+ - jochi
+ - ogedei
+ - chagatai
+ - tolui
+
+brain:
+ wiki_root: "/Users/kublai/brain"
+ public_index: "~/.brain-index/brain.db"
+ private_index: "~/.kublai/brain-index-private/brain.db"
+ public_gateway: "127.0.0.1:8765"
diff --git a/config/runtime-config/kanban.schema.json b/config/runtime-config/kanban.schema.json
new file mode 100644
index 00000000..32c0a3e7
--- /dev/null
+++ b/config/runtime-config/kanban.schema.json
@@ -0,0 +1,484 @@
+{
+ "note": "No task bodies, comments, chat IDs, or run outputs are exported.",
+ "schema": "kurultai.kanban-schema.v1",
+ "source": "~/.hermes/kanban.db sqlite schema only",
+ "tables": [
+ {
+ "columns": [
+ {
+ "default": null,
+ "name": "task_id",
+ "not_null": true,
+ "primary_key": true,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "platform",
+ "not_null": true,
+ "primary_key": true,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "chat_id",
+ "not_null": true,
+ "primary_key": true,
+ "type": "TEXT"
+ },
+ {
+ "default": "''",
+ "name": "thread_id",
+ "not_null": true,
+ "primary_key": true,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "user_id",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "created_at",
+ "not_null": true,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": "0",
+ "name": "last_event_id",
+ "not_null": true,
+ "primary_key": false,
+ "type": "INTEGER"
+ }
+ ],
+ "name": "kanban_notify_subs",
+ "type": "table"
+ },
+ {
+ "columns": [
+ {
+ "default": null,
+ "name": "id",
+ "not_null": false,
+ "primary_key": true,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "task_id",
+ "not_null": true,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "author",
+ "not_null": true,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "body",
+ "not_null": true,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "created_at",
+ "not_null": true,
+ "primary_key": false,
+ "type": "INTEGER"
+ }
+ ],
+ "name": "task_comments",
+ "type": "table"
+ },
+ {
+ "columns": [
+ {
+ "default": null,
+ "name": "id",
+ "not_null": false,
+ "primary_key": true,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "task_id",
+ "not_null": true,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "run_id",
+ "not_null": false,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "kind",
+ "not_null": true,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "payload",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "created_at",
+ "not_null": true,
+ "primary_key": false,
+ "type": "INTEGER"
+ }
+ ],
+ "name": "task_events",
+ "type": "table"
+ },
+ {
+ "columns": [
+ {
+ "default": null,
+ "name": "parent_id",
+ "not_null": true,
+ "primary_key": true,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "child_id",
+ "not_null": true,
+ "primary_key": true,
+ "type": "TEXT"
+ }
+ ],
+ "name": "task_links",
+ "type": "table"
+ },
+ {
+ "columns": [
+ {
+ "default": null,
+ "name": "id",
+ "not_null": false,
+ "primary_key": true,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "task_id",
+ "not_null": true,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "profile",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "step_key",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "status",
+ "not_null": true,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "claim_lock",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "claim_expires",
+ "not_null": false,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "worker_pid",
+ "not_null": false,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "max_runtime_seconds",
+ "not_null": false,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "last_heartbeat_at",
+ "not_null": false,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "started_at",
+ "not_null": true,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "ended_at",
+ "not_null": false,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "outcome",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "summary",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "metadata",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "error",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "session_id",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ }
+ ],
+ "name": "task_runs",
+ "type": "table"
+ },
+ {
+ "columns": [
+ {
+ "default": null,
+ "name": "id",
+ "not_null": false,
+ "primary_key": true,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "title",
+ "not_null": true,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "body",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "assignee",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "status",
+ "not_null": true,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": "0",
+ "name": "priority",
+ "not_null": false,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "created_by",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "created_at",
+ "not_null": true,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "started_at",
+ "not_null": false,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "completed_at",
+ "not_null": false,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": "'scratch'",
+ "name": "workspace_kind",
+ "not_null": true,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "workspace_path",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "claim_lock",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "claim_expires",
+ "not_null": false,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "tenant",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "result",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "idempotency_key",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": "0",
+ "name": "spawn_failures",
+ "not_null": true,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "worker_pid",
+ "not_null": false,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "last_spawn_error",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "max_runtime_seconds",
+ "not_null": false,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "last_heartbeat_at",
+ "not_null": false,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "current_run_id",
+ "not_null": false,
+ "primary_key": false,
+ "type": "INTEGER"
+ },
+ {
+ "default": null,
+ "name": "workflow_template_id",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "current_step_key",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ },
+ {
+ "default": null,
+ "name": "skills",
+ "not_null": false,
+ "primary_key": false,
+ "type": "TEXT"
+ }
+ ],
+ "name": "tasks",
+ "type": "table"
+ }
+ ]
+}
diff --git a/config/runtime-config/kurultai.yaml b/config/runtime-config/kurultai.yaml
new file mode 100644
index 00000000..b42603c5
--- /dev/null
+++ b/config/runtime-config/kurultai.yaml
@@ -0,0 +1,18 @@
+kurultai:
+ chair: kublai
+ coordination_substrate: native-hermes
+ task_router: native-kanban
+ kanban_db: "~/.hermes/kanban.db"
+ worker_profiles:
+ - temujin
+ - mongke
+ - jochi
+ - ogedei
+ - chagatai
+ - tolui
+ receipts:
+ required_for_meaningful_runtime_changes: true
+ brain_receipts_root: "/Users/kublai/brain/receipts"
+ retired_surfaces:
+ openclaw_internal_telegram_protocol: retired
+ neo4j_brain_memory: retired
diff --git a/config/runtime-config/profiles.yaml b/config/runtime-config/profiles.yaml
new file mode 100644
index 00000000..a994bed8
--- /dev/null
+++ b/config/runtime-config/profiles.yaml
@@ -0,0 +1,30 @@
+profiles:
+ kublai:
+ role: caretaker_orchestrator
+ provider: openai-codex
+ model: gpt-5.5
+ temujin:
+ role: developer
+ provider: openai-codex
+ model: gpt-5.5
+ mongke:
+ role: research
+ provider: openai-codex
+ model: gpt-5.5
+ jochi:
+ role: analyst
+ provider: openai-codex
+ model: gpt-5.5
+ ogedei:
+ role: operations
+ provider: openai-codex
+ model: gpt-5.5
+ chagatai:
+ role: writer
+ provider: openai-codex
+ model: gpt-5.5
+ tolui:
+ role: local_lightweight_worker
+ provider: ollama-local
+ model: gemma3:4b
+ notes: "Local model; tool-call capability must be verified before assignment."
diff --git a/config/runtime-config/skills.manifest.json b/config/runtime-config/skills.manifest.json
new file mode 100644
index 00000000..145abc95
--- /dev/null
+++ b/config/runtime-config/skills.manifest.json
@@ -0,0 +1,567 @@
+{
+ "note": "Names, relative paths, and descriptions only. Skill bodies are intentionally not exported by this manifest.",
+ "schema": "kurultai.skills-manifest.v1",
+ "skills": [
+ {
+ "description": "Audit, optimize, prune, and safely restructure Daniel's `/Users/kurultai/brain/` Obsidian wiki. Use for scheduled Hermes/Codex/Claude Code brain maintenance, retrieval-quality audits, index reconciliation, stale-page pruning, duplicate detection, canonical-page proposals, and approval-gated restructures.",
+ "name": "brain-optimizer",
+ "path": ".archive/brain-optimizer"
+ },
+ {
+ "description": "Primary always-on objective. Maintain the Kurultai OpenClaw implementation. Replaces the custom Hermes custodian.",
+ "name": "kurultai-caretaker",
+ "path": ".archive/kurultai-caretaker"
+ },
+ {
+ "description": "Manage Apple Notes via memo CLI: create, search, edit.",
+ "name": "apple-notes",
+ "path": "apple/apple-notes"
+ },
+ {
+ "description": "Apple Reminders via remindctl: add, list, complete.",
+ "name": "apple-reminders",
+ "path": "apple/apple-reminders"
+ },
+ {
+ "description": "Track Apple devices/AirTags via FindMy.app on macOS.",
+ "name": "findmy",
+ "path": "apple/findmy"
+ },
+ {
+ "description": "Send and receive iMessages/SMS via the imsg CLI on macOS.",
+ "name": "imessage",
+ "path": "apple/imessage"
+ },
+ {
+ "description": "Delegate coding to Claude Code CLI (features, PRs).",
+ "name": "claude-code",
+ "path": "autonomous-ai-agents/claude-code"
+ },
+ {
+ "description": "Delegate coding tasks to OpenAI Codex CLI agent. Use for building features, refactoring, PR reviews, batch issue fixing, and debugging Hermes Kanban codex worker startup/profile/auth crashes. Requires the codex CLI and a git repository.",
+ "name": "codex",
+ "path": "autonomous-ai-agents/codex"
+ },
+ {
+ "description": "Complete guide to using and extending Hermes Agent \u2014 CLI usage, setup, configuration, spawning additional agents, gateway platforms, skills, voice, tools, profiles, and a concise contributor reference. Load this skill when helping users configure Hermes, troubleshoot issues, spawn agent instances, make code contributions, or evaluate bounded multi-agent operating topologies; see references/subconscious-auto-think-auto-build.md for the gkisokay-style topology; see references/human-gated-soul-patch-prompts.md for protected SOUL/config patch handoff prompts, references/human-gated-identity-config-workflow-2026-05-09.md for human-gated identity/config proposal workflows, references/human-gated-identity-config-recovery-2026-05-09.md for SOUL/config/high-blast-radius approval workflows and guardrails, and references/local-ollama-profile-kanban.md for configuring a Hermes profile as a local Ollama Kanban worker.",
+ "name": "hermes-agent",
+ "path": "autonomous-ai-agents/hermes-agent"
+ },
+ {
+ "description": "Delegate coding to OpenCode CLI (features, PR review).",
+ "name": "opencode",
+ "path": "autonomous-ai-agents/opencode"
+ },
+ {
+ "description": "Create executable go-to-market plans with autonomous operating loops, ICP, channels, metrics, and approval gates.",
+ "name": "go-to-market-planning",
+ "path": "business/go-to-market-planning"
+ },
+ {
+ "description": "Dark-themed SVG architecture/cloud/infra diagrams as HTML.",
+ "name": "architecture-diagram",
+ "path": "creative/architecture-diagram"
+ },
+ {
+ "description": "ASCII art: pyfiglet, cowsay, boxes, image-to-ascii.",
+ "name": "ascii-art",
+ "path": "creative/ascii-art"
+ },
+ {
+ "description": "ASCII video: convert video/audio to colored ASCII MP4/GIF.",
+ "name": "ascii-video",
+ "path": "creative/ascii-video"
+ },
+ {
+ "description": "Knowledge comics (\u77e5\u8bc6\u6f2b\u753b): educational, biography, tutorial.",
+ "name": "baoyu-comic",
+ "path": "creative/baoyu-comic"
+ },
+ {
+ "description": "Infographics: 21 layouts x 21 styles (\u4fe1\u606f\u56fe, \u53ef\u89c6\u5316).",
+ "name": "baoyu-infographic",
+ "path": "creative/baoyu-infographic"
+ },
+ {
+ "description": "Design one-off HTML artifacts (landing, deck, prototype).",
+ "name": "claude-design",
+ "path": "creative/claude-design"
+ },
+ {
+ "description": "Generate images, video, and audio with ComfyUI \u2014 install, launch, manage nodes/models, run workflows with parameter injection. Uses the official comfy-cli for lifecycle and direct REST/WebSocket API for execution.",
+ "name": "comfyui",
+ "path": "creative/comfyui"
+ },
+ {
+ "description": "Generate project ideas via creative constraints.",
+ "name": "creative-ideation",
+ "path": "creative/creative-ideation"
+ },
+ {
+ "description": "Author/validate/export Google's DESIGN.md token spec files.",
+ "name": "design-md",
+ "path": "creative/design-md"
+ },
+ {
+ "description": "Hand-drawn Excalidraw JSON diagrams (arch, flow, seq).",
+ "name": "excalidraw",
+ "path": "creative/excalidraw"
+ },
+ {
+ "description": "Humanize text: strip AI-isms and add real voice.",
+ "name": "humanizer",
+ "path": "creative/humanizer"
+ },
+ {
+ "description": "Manim CE animations: 3Blue1Brown math/algo videos.",
+ "name": "manim-video",
+ "path": "creative/manim-video"
+ },
+ {
+ "description": "p5.js sketches: gen art, shaders, interactive, 3D.",
+ "name": "p5js",
+ "path": "creative/p5js"
+ },
+ {
+ "description": "Pixel art w/ era palettes (NES, Game Boy, PICO-8).",
+ "name": "pixel-art",
+ "path": "creative/pixel-art"
+ },
+ {
+ "description": "54 real design systems (Stripe, Linear, Vercel) as HTML/CSS, plus curated references for broadcast/editorial style cues.",
+ "name": "popular-web-designs",
+ "path": "creative/popular-web-designs"
+ },
+ {
+ "description": "Use when building creative browser demos with @chenglou/pretext \u2014 DOM-free text layout for ASCII art, typographic flow around obstacles, text-as-geometry games, kinetic typography, and text-powered generative art. Produces single-file HTML demos by default.",
+ "name": "pretext",
+ "path": "creative/pretext"
+ },
+ {
+ "description": "Throwaway HTML mockups: 2-3 design variants to compare.",
+ "name": "sketch",
+ "path": "creative/sketch"
+ },
+ {
+ "description": "Songwriting craft and Suno AI music prompts.",
+ "name": "songwriting-and-ai-music",
+ "path": "creative/songwriting-and-ai-music"
+ },
+ {
+ "description": "Control a running TouchDesigner instance via twozero MCP \u2014 create operators, set parameters, wire connections, execute Python, build real-time visuals. 36 native tools.",
+ "name": "touchdesigner-mcp",
+ "path": "creative/touchdesigner-mcp"
+ },
+ {
+ "description": "Iterative Python via live Jupyter kernel (hamelnb).",
+ "name": "jupyter-live-kernel",
+ "path": "data-science/jupyter-live-kernel"
+ },
+ {
+ "description": "Preserve interrupted, preempted, blocked, or concurrent work by routing it into Hermes Kanban tasks with context, receipts, exact completion reporting, and active blocker resolution.",
+ "name": "kanban-interruption-handoff",
+ "path": "devops/kanban-interruption-handoff"
+ },
+ {
+ "description": "Decomposition playbook + specialist-roster conventions + anti-temptation rules for an orchestrator profile routing work through Kanban. The \"don't do the work yourself\" rule and the basic lifecycle are auto-injected into every kanban worker's system prompt; this skill is the deeper playbook when you're specifically playing the orchestrator role. See references/kanban-delegation-pitfalls.md for workspace flag, dependency-link, dispatch-verification, and Parse-screening pitfalls. See references/codex-orchestration-routing.md for the convention that Codex-backed implementation work should be assigned to a real Kurultai profile (for example Chagatai) using `codex-implementation-orchestration`, not directly to a pseudo-profile named `codex`. See references/telegram-notification-subscriptions.md for the notification pitfall where direct CLI/DB task creation completes normally but does not notify Telegram unless a `kanban_notify_subs` row or manual completion synthesis is created.",
+ "name": "kanban-orchestrator",
+ "path": "devops/kanban-orchestrator"
+ },
+ {
+ "description": "Use when deciding whether work should become a Hermes Kanban task, choosing the right Kurultai assignee, dispatching it, monitoring it to completion, and actively managing blockers or follow-up work.",
+ "name": "kanban-task-lifecycle",
+ "path": "devops/kanban-task-lifecycle"
+ },
+ {
+ "description": "Pitfalls, examples, and edge cases for Hermes Kanban workers. The lifecycle itself is auto-injected into every worker's system prompt as KANBAN_GUIDANCE (from agent/prompt_builder.py); this skill is what you load when you want deeper detail on specific scenarios.",
+ "name": "kanban-worker",
+ "path": "devops/kanban-worker"
+ },
+ {
+ "description": "Webhook subscriptions: event-driven agent runs.",
+ "name": "webhook-subscriptions",
+ "path": "devops/webhook-subscriptions"
+ },
+ {
+ "description": "Find and verify an x402 payment deposit/payTo address safely, without exposing private keys or secrets.",
+ "name": "x402-payment-address-lookup",
+ "path": "devops/x402-payment-address-lookup"
+ },
+ {
+ "description": "Exploratory QA of web apps: find bugs, evidence, reports.",
+ "name": "dogfood",
+ "path": "dogfood"
+ },
+ {
+ "description": "Himalaya CLI: IMAP/SMTP email from terminal.",
+ "name": "himalaya",
+ "path": "email/himalaya"
+ },
+ {
+ "description": "Host modded Minecraft servers (CurseForge, Modrinth).",
+ "name": "minecraft-modpack-server",
+ "path": "gaming/minecraft-modpack-server"
+ },
+ {
+ "description": "Play Pokemon via headless emulator + RAM reads.",
+ "name": "pokemon-player",
+ "path": "gaming/pokemon-player"
+ },
+ {
+ "description": "Inspect codebases w/ pygount: LOC, languages, ratios.",
+ "name": "codebase-inspection",
+ "path": "github/codebase-inspection"
+ },
+ {
+ "description": "GitHub auth setup: HTTPS tokens, SSH keys, gh CLI login.",
+ "name": "github-auth",
+ "path": "github/github-auth"
+ },
+ {
+ "description": "Review PRs: diffs, inline comments via gh or REST.",
+ "name": "github-code-review",
+ "path": "github/github-code-review"
+ },
+ {
+ "description": "Create, triage, label, assign GitHub issues via gh or REST.",
+ "name": "github-issues",
+ "path": "github/github-issues"
+ },
+ {
+ "description": "GitHub PR lifecycle: branch, commit, open, CI, merge.",
+ "name": "github-pr-workflow",
+ "path": "github/github-pr-workflow"
+ },
+ {
+ "description": "Clone/create/fork repos; manage remotes, releases. For requests to house live runtime configuration in a repo, use the sanitized-template/manifest pattern in `references/sanitized-runtime-config-housing.md`: never copy private live config or secrets directly into Git.",
+ "name": "github-repo-management",
+ "path": "github/github-repo-management"
+ },
+ {
+ "description": "Report/link intake for brain synthesis and proposal packets.",
+ "name": "brain-report-ingest-propose",
+ "path": "kurultai/brain-report-ingest-propose"
+ },
+ {
+ "description": "Operate and maintain Danny's brain wiki: context, search, save, ingest, sync, research, fold/archive, graph/API maintenance, and skill parity.",
+ "name": "brain-wiki-operations",
+ "path": "kurultai/brain-wiki-operations"
+ },
+ {
+ "description": "Operate Danny's Content OS end-to-end: inbox processing, connection mining, brief generation, drafting, return path, and performance review.",
+ "name": "content-os-workflows",
+ "path": "kurultai/content-os-workflows"
+ },
+ {
+ "description": "Operate, diagnose, and improve Kurultai/OpenClaw: queues, runtime config housing (see `references/runtime-config-repo-housing.md`), routing, proposals, caretaker/improver loops, auth/rate limits, dashboards, and model policy.",
+ "name": "kurultai-operations",
+ "path": "kurultai/kurultai-operations"
+ },
+ {
+ "description": "Run a post-task retro/learn pass that compiles meaningful work into memory, skills, Brain doctrine, Kanban follow-up, receipts, regressions, canaries, or explicit no-op rationale.",
+ "name": "kurultai-retro-learn",
+ "path": "kurultai/kurultai-retro-learn"
+ },
+ {
+ "description": "Configure, upgrade, and debug OpenClaw/Hermes integration surfaces including Telegram channels, Parse screening cleanup, CLI/gateway upgrades, and imported helper workflows.",
+ "name": "openclaw-operations",
+ "path": "kurultai/openclaw-operations"
+ },
+ {
+ "description": "Author, install, consolidate, and propagate agent skills across Hermes, Codex, Claude, OpenClaw, and the brain wiki skill-parity layer.",
+ "name": "skill-library-operations",
+ "path": "kurultai/skill-library-operations"
+ },
+ {
+ "description": "MCP client: connect servers, register tools (stdio/HTTP).",
+ "name": "native-mcp",
+ "path": "mcp/native-mcp"
+ },
+ {
+ "description": "Search/download GIFs from Tenor via curl + jq.",
+ "name": "gif-search",
+ "path": "media/gif-search"
+ },
+ {
+ "description": "HeartMuLa: Suno-like song generation from lyrics + tags.",
+ "name": "heartmula",
+ "path": "media/heartmula"
+ },
+ {
+ "description": "Audio spectrograms/features (mel, chroma, MFCC) via CLI.",
+ "name": "songsee",
+ "path": "media/songsee"
+ },
+ {
+ "description": "Spotify: play, search, queue, manage playlists and devices.",
+ "name": "spotify",
+ "path": "media/spotify"
+ },
+ {
+ "description": "YouTube transcripts to summaries, threads, blogs.",
+ "name": "youtube-content",
+ "path": "media/youtube-content"
+ },
+ {
+ "description": "lm-eval-harness: benchmark LLMs (MMLU, GSM8K, etc.).",
+ "name": "lm-evaluation-harness",
+ "path": "mlops/evaluation/lm-evaluation-harness"
+ },
+ {
+ "description": "W&B: log ML experiments, sweeps, model registry, dashboards.",
+ "name": "weights-and-biases",
+ "path": "mlops/evaluation/weights-and-biases"
+ },
+ {
+ "description": "HuggingFace hf CLI: search/download/upload models, datasets.",
+ "name": "huggingface-hub",
+ "path": "mlops/huggingface-hub"
+ },
+ {
+ "description": "llama.cpp local GGUF inference + HF Hub model discovery.",
+ "name": "llama-cpp",
+ "path": "mlops/inference/llama-cpp"
+ },
+ {
+ "description": "OBLITERATUS: abliterate LLM refusals (diff-in-means).",
+ "name": "obliteratus",
+ "path": "mlops/inference/obliteratus"
+ },
+ {
+ "description": "Outlines: structured JSON/regex/Pydantic LLM generation.",
+ "name": "outlines",
+ "path": "mlops/inference/outlines"
+ },
+ {
+ "description": "vLLM: high-throughput LLM serving, OpenAI API, quantization.",
+ "name": "vllm",
+ "path": "mlops/inference/vllm"
+ },
+ {
+ "description": "AudioCraft: MusicGen text-to-music, AudioGen text-to-sound.",
+ "name": "audiocraft",
+ "path": "mlops/models/audiocraft"
+ },
+ {
+ "description": "SAM: zero-shot image segmentation via points, boxes, masks.",
+ "name": "segment-anything",
+ "path": "mlops/models/segment-anything"
+ },
+ {
+ "description": "DSPy: declarative LM programs, auto-optimize prompts, RAG.",
+ "name": "dspy",
+ "path": "mlops/research/dspy"
+ },
+ {
+ "description": "Axolotl: YAML LLM fine-tuning (LoRA, DPO, GRPO).",
+ "name": "axolotl",
+ "path": "mlops/training/axolotl"
+ },
+ {
+ "description": "TRL: SFT, DPO, PPO, GRPO, reward modeling for LLM RLHF.",
+ "name": "trl-fine-tuning",
+ "path": "mlops/training/trl-fine-tuning"
+ },
+ {
+ "description": "Unsloth: 2-5x faster LoRA/QLoRA fine-tuning, less VRAM.",
+ "name": "unsloth",
+ "path": "mlops/training/unsloth"
+ },
+ {
+ "description": "Read, search, and create notes in the Obsidian vault.",
+ "name": "obsidian",
+ "path": "note-taking/obsidian"
+ },
+ {
+ "description": "Create and maintain Obsidian vault artifacts: CLI operations, Markdown conventions, Bases views, JSON Canvas maps, and clean imported documents.",
+ "name": "obsidian-workflows",
+ "path": "note-taking/obsidian-workflows"
+ },
+ {
+ "description": "Build, route, and report Kublai/Kurultai task packets optimized for agent-aware execution and completion reporting.",
+ "name": "kublai-task-workflows",
+ "path": "openclaw-imports/kublai-task-workflows"
+ },
+ {
+ "description": "Airtable REST API via curl. Records CRUD, filters, upserts.",
+ "name": "airtable",
+ "path": "productivity/airtable"
+ },
+ {
+ "description": "Gmail, Calendar, Drive, Docs, Sheets via gws CLI or Python.",
+ "name": "google-workspace",
+ "path": "productivity/google-workspace"
+ },
+ {
+ "description": "Linear: manage issues, projects, teams via GraphQL + curl.",
+ "name": "linear",
+ "path": "productivity/linear"
+ },
+ {
+ "description": "Geocode, POIs, routes, timezones via OpenStreetMap/OSRM.",
+ "name": "maps",
+ "path": "productivity/maps"
+ },
+ {
+ "description": "Edit PDF text/typos/titles via nano-pdf CLI (NL prompts).",
+ "name": "nano-pdf",
+ "path": "productivity/nano-pdf"
+ },
+ {
+ "description": "Notion API via curl: pages, databases, blocks, search.",
+ "name": "notion",
+ "path": "productivity/notion"
+ },
+ {
+ "description": "Extract text from PDFs/scans (pymupdf, marker-pdf).",
+ "name": "ocr-and-documents",
+ "path": "productivity/ocr-and-documents"
+ },
+ {
+ "description": "Create, read, edit .pptx decks, slides, notes, templates.",
+ "name": "powerpoint",
+ "path": "productivity/powerpoint"
+ },
+ {
+ "description": "Jailbreak LLMs: Parseltongue, GODMODE, ULTRAPLINIAN.",
+ "name": "godmode",
+ "path": "red-teaming/godmode"
+ },
+ {
+ "description": "Search arXiv papers by keyword, author, category, or ID.",
+ "name": "arxiv",
+ "path": "research/arxiv"
+ },
+ {
+ "description": "Monitor blogs and RSS/Atom feeds via blogwatcher-cli tool.",
+ "name": "blogwatcher",
+ "path": "research/blogwatcher"
+ },
+ {
+ "description": "Karpathy's LLM Wiki: build/query interlinked markdown KB.",
+ "name": "llm-wiki",
+ "path": "research/llm-wiki"
+ },
+ {
+ "description": "Query Polymarket: markets, prices, orderbooks, history.",
+ "name": "polymarket",
+ "path": "research/polymarket"
+ },
+ {
+ "description": "Operate and evaluate Danny's paper-only Polymarket daily temperature research engine, including simulated portfolio, training export, and goal tuning.",
+ "name": "polymarket-weather-paper-engine",
+ "path": "research/polymarket-weather-paper-engine"
+ },
+ {
+ "description": "End-to-end pipeline for writing ML/AI research papers \u2014 experiment design through submission. Covers NeurIPS, ICML, ICLR, ACL, AAAI, COLM.",
+ "name": "research-paper-writing",
+ "path": "research/research-paper-writing"
+ },
+ {
+ "description": "Control Philips Hue lights, scenes, rooms via OpenHue CLI.",
+ "name": "openhue",
+ "path": "smart-home/openhue"
+ },
+ {
+ "description": "Interact with X/Twitter via xurl, the official X API CLI. Use for posting, replying, quoting, searching, timelines, mentions, likes, reposts, bookmarks, follows, DMs, media upload, raw v2 endpoint access, and Parse-safe research of individual X posts/articles. For Parse blocks, see references/parse-screening-safe-x-research.md; for browser-rendered X article/tweet fallback research when xurl/syndication/oEmbed/search fail, see references/x-article-research-fallbacks.md. For X Article research and agent-first content/doctrine analysis, see references/x-article-agent-first-research.md. For May 2026 single-post research patterns using sanitized metadata, reversed-token diagnostics, and browser-rendered image fallback, see references/parse-safe-individual-x-post-research-2026-05.md.",
+ "name": "xurl",
+ "path": "social-media/xurl"
+ },
+ {
+ "description": "Direct browser control via CDP connected to the user's real Chrome. Use when the user wants to automate, scrape, test, or interact with web pages. Installed at ~/Developer/browser-harness.",
+ "name": "browser-harness",
+ "path": "software-development/browser-harness"
+ },
+ {
+ "description": "Use Codex as a bulk coding worker while Hermes acts as reviewer, verifier, and safety gate for multi-phase implementation work. In Kanban contexts, assign the task to a real accountable Kurultai profile (for example Chagatai) and have that profile use this skill; do not assign the task directly to a pseudo-profile named `codex`. See references/kanban-ownership.md.",
+ "name": "codex-implementation-orchestration",
+ "path": "software-development/codex-implementation-orchestration"
+ },
+ {
+ "description": "Debug Hermes slash commands across Python registry, gateway/Telegram, TUI, and Ink UI.",
+ "name": "debugging-hermes-tui-commands",
+ "path": "software-development/debugging-hermes-tui-commands"
+ },
+ {
+ "description": "Author in-repo SKILL.md: frontmatter, validator, structure.",
+ "name": "hermes-agent-skill-authoring",
+ "path": "software-development/hermes-agent-skill-authoring"
+ },
+ {
+ "description": "Debug Node.js via --inspect + Chrome DevTools Protocol CLI.",
+ "name": "node-inspect-debugger",
+ "path": "software-development/node-inspect-debugger"
+ },
+ {
+ "description": "Metadata receipt pattern for Hermes screened tool events.",
+ "name": "parse-screening-recovery",
+ "path": "software-development/parse-screening-recovery"
+ },
+ {
+ "description": "Plan mode: write markdown plan to .hermes/plans/, no exec.",
+ "name": "plan",
+ "path": "software-development/plan"
+ },
+ {
+ "description": "Debug Python: pdb REPL + debugpy remote (DAP).",
+ "name": "python-debugpy",
+ "path": "software-development/python-debugpy"
+ },
+ {
+ "description": "'Repository workflow orchestration skill for staged implementation, locked artifacts, late-phase receipts, and durable memory maintenance. Use when executing recursive-mode runs, resuming a run, locking a phase, or verifying locks.'",
+ "name": "recursive-mode",
+ "path": "software-development/recursive-mode"
+ },
+ {
+ "description": "Pre-commit review: security scan, quality gates, auto-fix.",
+ "name": "requesting-code-review",
+ "path": "software-development/requesting-code-review"
+ },
+ {
+ "description": "Write shell commands that don't trip Hermes's own security scanner. Avoid pipe-to-interpreter patterns and inline `-c` script execution when a direct-file read works.",
+ "name": "shell-command-hygiene",
+ "path": "software-development/shell-command-hygiene"
+ },
+ {
+ "description": "Throwaway experiments to validate an idea before build.",
+ "name": "spike",
+ "path": "software-development/spike"
+ },
+ {
+ "description": "Wire, deploy, and verify live-data ingestion for static/SPAs that consume JSON snapshots or SSE-style live endpoints.",
+ "name": "static-site-live-data-deployment",
+ "path": "software-development/static-site-live-data-deployment"
+ },
+ {
+ "description": "Execute plans via delegate_task subagents (2-stage review).",
+ "name": "subagent-driven-development",
+ "path": "software-development/subagent-driven-development"
+ },
+ {
+ "description": "4-phase root cause debugging: understand bugs before fixing.",
+ "name": "systematic-debugging",
+ "path": "software-development/systematic-debugging"
+ },
+ {
+ "description": "TDD: enforce RED-GREEN-REFACTOR, tests before code.",
+ "name": "test-driven-development",
+ "path": "software-development/test-driven-development"
+ },
+ {
+ "description": "Use when you have a spec or requirements for a multi-step task. Creates comprehensive implementation plans with bite-sized tasks, exact file paths, and complete code examples.",
+ "name": "writing-plans",
+ "path": "software-development/writing-plans"
+ },
+ {
+ "description": "Operate the meta-harness control plane: curator gate/co-judge behavior, harness controls, read-only evaluation mode, and operator safety boundaries.",
+ "name": "meta-harness-operations",
+ "path": "system/meta-harness-operations"
+ },
+ {
+ "description": "Yuanbao (\u5143\u5b9d) groups: @mention users, query info/members.",
+ "name": "yuanbao",
+ "path": "yuanbao"
+ }
+ ],
+ "source": "~/.hermes/skills/**/SKILL.md"
+}
diff --git a/demo_agent_direct.py b/demo_agent_direct.py
index 9fc55615..efd8a308 100644
--- a/demo_agent_direct.py
+++ b/demo_agent_direct.py
@@ -4,6 +4,8 @@
Shows how agents can now read/write/execute anywhere on the system.
"""
+from __future__ import annotations
+
import sys
sys.path.insert(0, 'tools/kurultai')
from agent_gemini_direct import temujin_gemini_direct
diff --git a/docs/llm-survivor/engine/phase_a_challenge.py b/docs/llm-survivor/engine/phase_a_challenge.py
index b6e93298..442d2e40 100644
--- a/docs/llm-survivor/engine/phase_a_challenge.py
+++ b/docs/llm-survivor/engine/phase_a_challenge.py
@@ -3,6 +3,7 @@
Agents collaborate/compete to solve ARC-style grid transformation.
Implements DeepThink's Master Prompt Architecture.
"""
+from __future__ import annotations
import sqlite3
import json
diff --git a/docs/llm-survivor/engine/phase_b_scramble.py b/docs/llm-survivor/engine/phase_b_scramble.py
index 50c690cd..0cd61d59 100644
--- a/docs/llm-survivor/engine/phase_b_scramble.py
+++ b/docs/llm-survivor/engine/phase_b_scramble.py
@@ -3,6 +3,7 @@
Agents use Action Points to whisper, form alliances, and plot.
Implements DeepThink's Master Prompt Architecture.
"""
+from __future__ import annotations
import sqlite3
import json
diff --git a/docs/operations/kurultai-rebuild-runbook.md b/docs/operations/kurultai-rebuild-runbook.md
new file mode 100644
index 00000000..930dd465
--- /dev/null
+++ b/docs/operations/kurultai-rebuild-runbook.md
@@ -0,0 +1,55 @@
+# Kurultai rebuild runbook
+
+Purpose: make this repository the non-secret rebuild contract for the current Kublai/Kurultai operating system.
+
+This is not a secret backup. Live tokens, private chat targets, session logs, private indexes, and credentials stay outside Git.
+
+## Rebuild target
+
+A rebuilt host should provide these capabilities:
+
+- Hermes runtime for Kublai.
+- Kurultai Hermes profile roster.
+- Native Hermes Kanban with the expected schema.
+- Brain wiki at the configured path, with QMD indexing.
+- Cron jobs recreated from sanitized manifests after review.
+- Skill inventory restored from source or backups.
+- Receipts and recovery directories available.
+- Retired OpenClaw internal Telegram protocol and Neo4j/Bolt/Cypher stack remain retired.
+
+## Repository surfaces
+
+- `config/runtime-config/hermes.template.yaml`: sanitized Hermes runtime contract.
+- `config/runtime-config/profiles.yaml`: Kurultai profiles and roles.
+- `config/runtime-config/kurultai.yaml`: native coordination contract.
+- `config/runtime-config/brain.yaml`: Brain wiki and index contract.
+- `config/runtime-config/cron.manifest.json`: sanitized cron manifest.
+- `config/runtime-config/skills.manifest.json`: skill names, relative paths, descriptions.
+- `config/runtime-config/kanban.schema.json`: Kanban schema only.
+- `config/runtime-config/brain.manifest.json`: Brain directory inventory only.
+- `scripts/export_runtime_config_manifest.py`: refreshes cron manifest.
+- `scripts/export_rebuild_manifests.py`: refreshes skills, Kanban schema, Brain inventory.
+- `scripts/bootstrap_kurultai_runtime.py`: creates a review staging area for a rebuilt host.
+
+## Secret boundary
+
+Do not commit live Hermes config, Kanban databases, session JSON, private Brain indexes, telemetry DBs, API keys, OAuth data, cookies, private keys, account identifiers, raw transcript dumps, or private delivery targets.
+
+Use placeholders and environment-variable names in repo files.
+
+## Rebuild sequence
+
+1. Install host prerequisites: Python, Git, Node/npm where needed, Hermes Agent CLI, QMD, and provider/tool CLIs.
+2. Clone this repository.
+3. Run the bootstrap script in dry-run mode, then normal mode, to create `~/.kurultai-rebuild-staging/`.
+4. Restore Hermes configuration from the sanitized template and local private values.
+5. Recreate Hermes profiles from `profiles.yaml`.
+6. Initialize native Hermes Kanban and compare its schema to `kanban.schema.json`.
+7. Mount or clone the Brain wiki at the configured path and refresh QMD indexes.
+8. Recreate cron jobs from `cron.manifest.json`; manually restore redacted delivery targets.
+9. Restore skills from source repositories or private skill backups using `skills.manifest.json` as the checklist.
+10. Verify canaries: Hermes config check, profile startup, Kanban create/complete, Brain search/indexing, one safe cron run, receipt write/index, and repo secret scan.
+
+## Drift rule
+
+When the live system gains a new non-secret architecture surface, update these manifests and this runbook. The live system may contain private state; the repo should contain the rebuildable contract.
diff --git a/kublai/__init__.py b/kublai/__init__.py
new file mode 100644
index 00000000..5875207d
--- /dev/null
+++ b/kublai/__init__.py
@@ -0,0 +1,15 @@
+"""Local Kublai memory modules for the Neo4j-to-wiki migration."""
+
+from .telemetry import (
+ DEFAULT_LEASE_TTL_MS,
+ StaleClaimError,
+ TelemetryStore,
+)
+from .knowledge import KnowledgeStore
+
+__all__ = [
+ "DEFAULT_LEASE_TTL_MS",
+ "KnowledgeStore",
+ "StaleClaimError",
+ "TelemetryStore",
+]
diff --git a/kublai/audit.py b/kublai/audit.py
new file mode 100644
index 00000000..ad01040d
--- /dev/null
+++ b/kublai/audit.py
@@ -0,0 +1,65 @@
+"""Audit event helpers with conservative secret redaction."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from .tracing import new_trace_id
+
+SENSITIVE_KEYS = {
+ "api_key",
+ "apikey",
+ "authorization",
+ "auth_token",
+ "password",
+ "private_key",
+ "secret",
+ "token",
+}
+REDACTION = "[REDACTED]"
+
+
+def redact_secrets(value: Any) -> Any:
+ """Recursively redact common secret-bearing fields from JSON-like data."""
+ if isinstance(value, dict):
+ redacted: dict[str, Any] = {}
+ for key, item in value.items():
+ if _is_sensitive_key(str(key)):
+ redacted[key] = REDACTION
+ else:
+ redacted[key] = redact_secrets(item)
+ return redacted
+ if isinstance(value, list):
+ return [redact_secrets(item) for item in value]
+ if isinstance(value, tuple):
+ return tuple(redact_secrets(item) for item in value)
+ return value
+
+
+@dataclass(frozen=True)
+class AuditEvent:
+ actor: str
+ action: str
+ decision: str
+ details: dict[str, Any] = field(default_factory=dict)
+ trace_id: str | None = None
+ resource: str | None = None
+
+ def to_record(self) -> dict[str, Any]:
+ return {
+ "actor": self.actor,
+ "action": self.action,
+ "decision": self.decision,
+ "resource": self.resource,
+ "trace_id": self.trace_id,
+ "details": redact_secrets(self.details),
+ }
+
+ def write(self, store: Any) -> str:
+ return store.record_audit_event(**self.to_record())
+
+
+def _is_sensitive_key(key: str) -> bool:
+ key_l = key.lower().replace("-", "_")
+ return key_l in SENSITIVE_KEYS or key_l.endswith("_token") or key_l.endswith("_secret")
diff --git a/kublai/brain_service.py b/kublai/brain_service.py
new file mode 100644
index 00000000..ac70c59e
--- /dev/null
+++ b/kublai/brain_service.py
@@ -0,0 +1,913 @@
+"""Localhost-only brain service for Phase 1 fixtures.
+
+The daemon owns wiki indexing, materialization sweeps, SQLite backups, health,
+and a small Unix-socket JSON RPC surface for same-host JS clients.
+"""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import json
+import os
+import socketserver
+import sqlite3
+import sys
+import threading
+import time
+from pathlib import Path
+from typing import Any
+
+from .knowledge import KnowledgeStore
+from .monitoring import health_snapshot
+from .telemetry import DEFAULT_LEASE_TTL_MS, NoPendingTaskError, StaleClaimError, TelemetryStore
+from .calendar import CalendarError, CalendarService, CalendarStore, GcalcliClient
+from .humans import HumansStore
+from .messages import MessagesStore
+from .sanitizer import HARD_PRIVATE_FOLDERS, HARD_PRIVATE_TAGS, HARD_PRIVATE_TYPES, DEFAULT_SANITIZER
+from .v4 import V4WorkflowService
+
+
+class CapabilityError(RuntimeError):
+ """Raised when runtime prerequisites are not present."""
+
+
+def classify_page(rel_path: str, frontmatter: dict[str, Any] | None = None) -> str:
+ """Classify pages for public-index eligibility.
+
+ Public indexing must be explicit. The sanitizer defaults ambiguous pages to
+ "private"; keep that fail-closed contract here so body text is not read into
+ the public index unless a page opts in via publish/public_stub metadata.
+ """
+ return DEFAULT_SANITIZER.classify(rel_path, frontmatter)
+
+
+class BrainIndex:
+ """Rebuildable SQLite index over markdown pages."""
+
+ def __init__(self, db_path: str | Path):
+ self.db_path = Path(db_path).expanduser()
+ self.db_path.parent.mkdir(parents=True, exist_ok=True)
+ self.init_db()
+
+ def connect(self) -> sqlite3.Connection:
+ conn = sqlite3.connect(self.db_path, timeout=5.0)
+ conn.row_factory = sqlite3.Row
+ conn.execute("PRAGMA busy_timeout = 5000")
+ _load_sqlite_vec(conn)
+ return conn
+
+ def init_db(self) -> None:
+ with self.connect() as conn:
+ vector_ddl = (
+ "CREATE VIRTUAL TABLE IF NOT EXISTS node_vec USING vec0(embedding float[384]);"
+ if _sqlite_vec_loaded(conn)
+ else """
+ CREATE TABLE IF NOT EXISTS node_vec (
+ rowid INTEGER PRIMARY KEY,
+ embedding TEXT NOT NULL CHECK(json_valid(embedding))
+ ) STRICT;
+ """
+ )
+ conn.executescript(
+ f"""
+ PRAGMA journal_mode = WAL;
+ CREATE TABLE IF NOT EXISTS nodes (
+ node_pk INTEGER PRIMARY KEY,
+ id TEXT UNIQUE NOT NULL,
+ type TEXT NOT NULL,
+ rel_path TEXT UNIQUE NOT NULL,
+ title TEXT,
+ status TEXT,
+ agent TEXT,
+ typed_id TEXT,
+ created TEXT NOT NULL,
+ updated TEXT NOT NULL,
+ frontmatter TEXT NOT NULL CHECK(json_valid(frontmatter)),
+ body_hash TEXT NOT NULL,
+ body_text TEXT NOT NULL,
+ mtime_ns INTEGER NOT NULL
+ ) STRICT;
+ CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(
+ title, body_text, content='nodes', content_rowid='node_pk'
+ );
+ {vector_ddl}
+ CREATE TABLE IF NOT EXISTS index_meta (
+ key TEXT PRIMARY KEY,
+ value TEXT
+ ) STRICT;
+ INSERT OR REPLACE INTO index_meta VALUES
+ ('embedding_model', 'all-MiniLM-L6-v2'),
+ ('embedding_dim', '384'),
+ ('embedding_distance', 'cosine'),
+ ('vector_backend', '{'sqlite-vec vec0' if _sqlite_vec_loaded(conn) else 'json-fallback'}'),
+ ('schema_version', '1');
+ """
+ )
+
+ def reindex(self, wiki_root: str | Path, *, privacy_scope: str = "public") -> int:
+ import hashlib
+
+ from .knowledge import KnowledgeStore
+
+ if privacy_scope not in {"public", "hard-private"}:
+ raise ValueError(f"unknown privacy_scope: {privacy_scope}")
+ wiki_root = Path(wiki_root).resolve()
+ count = 0
+ with self.connect() as conn:
+ conn.execute("DELETE FROM node_vec")
+ conn.execute("DELETE FROM nodes_fts")
+ conn.execute("DELETE FROM nodes")
+ for path in sorted(wiki_root.rglob("*.md")):
+ if ".git" in path.parts:
+ continue
+ rel_path = path.relative_to(wiki_root).as_posix()
+ fm = KnowledgeStore.read_frontmatter(path)
+ privacy_class = classify_page(rel_path, fm)
+ if privacy_scope == "public" and privacy_class != "public":
+ continue
+ if privacy_scope == "hard-private" and privacy_class != "hard-private":
+ continue
+ body = _body_without_frontmatter(path.read_text(encoding="utf-8", errors="ignore"))
+ node_id = rel_path.removesuffix(".md")
+ typed_id = _first_present(
+ fm,
+ "task_id",
+ "reflection_id",
+ "decision_id",
+ "capability_id",
+ "rsi_id",
+ "agent_id",
+ )
+ digest = hashlib.sha256(body.encode("utf-8")).hexdigest()
+ row = conn.execute(
+ """
+ INSERT INTO nodes (
+ id, type, rel_path, title, status, agent, typed_id,
+ created, updated, frontmatter, body_hash, body_text, mtime_ns
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ RETURNING node_pk
+ """,
+ (
+ node_id,
+ str(fm.get("type", "unknown")),
+ rel_path,
+ str(fm.get("title", node_id)),
+ str(fm.get("status", "")),
+ str(fm.get("agent", "")),
+ typed_id,
+ str(fm.get("created", "")),
+ str(fm.get("updated", "")),
+ json.dumps(fm, sort_keys=True, default=str),
+ digest,
+ body,
+ path.stat().st_mtime_ns,
+ ),
+ ).fetchone()
+ node_pk = int(row["node_pk"])
+ conn.execute(
+ "INSERT INTO nodes_fts(rowid, title, body_text) VALUES (?, ?, ?)",
+ (node_pk, str(fm.get("title", node_id)), body),
+ )
+ conn.execute(
+ "INSERT INTO node_vec(rowid, embedding) VALUES (?, ?)",
+ (node_pk, json.dumps(_hash_embedding(body))),
+ )
+ count += 1
+ return count
+
+ def vector_orphans(self) -> int:
+ with self.connect() as conn:
+ row = conn.execute(
+ """
+ SELECT count(*) AS n
+ FROM node_vec v LEFT JOIN nodes n ON n.node_pk = v.rowid
+ WHERE n.node_pk IS NULL
+ """
+ ).fetchone()
+ return int(row["n"])
+
+ def stats(self) -> dict[str, Any]:
+ with self.connect() as conn:
+ nodes = conn.execute("SELECT count(*) AS n FROM nodes").fetchone()["n"]
+ fts = conn.execute("SELECT count(*) AS n FROM nodes_fts").fetchone()["n"]
+ vectors = conn.execute("SELECT count(*) AS n FROM node_vec").fetchone()["n"]
+ privacy_rows = conn.execute("SELECT rel_path, frontmatter FROM nodes").fetchall()
+ privacy_counts = {"public": 0, "private": 0, "hard-private": 0}
+ for privacy_row in privacy_rows:
+ try:
+ frontmatter = json.loads(privacy_row["frontmatter"] or "{}")
+ except json.JSONDecodeError:
+ frontmatter = {}
+ privacy_class = classify_page(privacy_row["rel_path"], frontmatter)
+ privacy_counts[privacy_class] = privacy_counts.get(privacy_class, 0) + 1
+ backend = conn.execute("SELECT value FROM index_meta WHERE key = 'vector_backend'").fetchone()
+ return {
+ "nodes": int(nodes),
+ "fts_rows": int(fts),
+ "vector_rows": int(vectors),
+ "vector_orphans": self.vector_orphans(),
+ "public_rows": int(privacy_counts.get("public", 0)),
+ "private_rows": int(privacy_counts.get("private", 0)),
+ "hard_private_rows": int(privacy_counts.get("hard-private", 0)),
+ "vector_backend": backend["value"] if backend else "unknown",
+ }
+
+ def get_by_typed_id(self, node_type: str, typed_id: str) -> dict[str, Any] | None:
+ with self.connect() as conn:
+ row = conn.execute(
+ """
+ SELECT id, type, rel_path, title, status, agent, typed_id,
+ created, updated, frontmatter, body_hash, body_text
+ FROM nodes
+ WHERE type = ? AND typed_id = ?
+ LIMIT 1
+ """,
+ (node_type, typed_id),
+ ).fetchone()
+ return _node_row_to_dict(row) if row else None
+
+ def list_nodes(
+ self,
+ *,
+ node_type: str | None = None,
+ agent: str | None = None,
+ status: str | None = None,
+ limit: int = 100,
+ ) -> list[dict[str, Any]]:
+ clauses: list[str] = []
+ params: list[Any] = []
+ if node_type:
+ clauses.append("type = ?")
+ params.append(node_type)
+ if agent:
+ clauses.append("agent = ?")
+ params.append(agent)
+ if status:
+ clauses.append("status = ?")
+ params.append(status)
+ where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
+ params.append(limit)
+ with self.connect() as conn:
+ rows = conn.execute(
+ f"""
+ SELECT id, type, rel_path, title, status, agent, typed_id,
+ created, updated, frontmatter, body_hash, body_text
+ FROM nodes
+ {where}
+ ORDER BY updated DESC, node_pk DESC
+ LIMIT ?
+ """,
+ params,
+ ).fetchall()
+ return [_node_row_to_dict(row) for row in rows]
+
+ def search(self, query: str, *, node_type: str | None = None, limit: int = 10) -> list[dict[str, Any]]:
+ if not query.strip():
+ return []
+ clauses = ["nodes_fts MATCH ?"]
+ params: list[Any] = [_fts5_phrase(query)]
+ if node_type:
+ clauses.append("n.type = ?")
+ params.append(node_type)
+ params.append(limit)
+ with self.connect() as conn:
+ rows = conn.execute(
+ f"""
+ SELECT n.id, n.type, n.rel_path, n.title, n.status, n.agent, n.typed_id,
+ n.created, n.updated, n.frontmatter, n.body_hash, n.body_text,
+ bm25(nodes_fts) AS score
+ FROM nodes_fts
+ JOIN nodes n ON n.node_pk = nodes_fts.rowid
+ WHERE {' AND '.join(clauses)}
+ ORDER BY score
+ LIMIT ?
+ """,
+ params,
+ ).fetchall()
+ return [_node_row_to_dict(row) | {"score": float(row["score"])} for row in rows]
+
+ def backup_to(self, destination: str | Path) -> Path:
+ destination = Path(destination)
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ with self.connect() as src, sqlite3.connect(destination) as dst:
+ src.backup(dst)
+ return destination
+
+
+ def list_by_tag_in_wiki(self, knowledge_store, *, tag, privacy_scope="public", limit=50):
+ if privacy_scope is None:
+ privacy_scope = "public"
+ return knowledge_store.list_by_tag(tag=tag, privacy_scope=privacy_scope, limit=limit)
+
+class BrainService:
+ def __init__(
+ self,
+ wiki_root: str | Path,
+ telemetry_db: str | Path,
+ index_db: str | Path,
+ *,
+ gcalcli_path: str | Path | None = None,
+ default_calendar: str | None = None,
+ ):
+ self.wiki_root = Path(wiki_root).expanduser().resolve()
+ self.telemetry = TelemetryStore(telemetry_db)
+ self.knowledge = KnowledgeStore(self.wiki_root)
+ self.index = BrainIndex(index_db)
+ private_index_db = os.environ.get(
+ "BRAIN_PRIVATE_INDEX_DB",
+ str(Path.home() / ".kublai/brain-index-private/brain.db"),
+ )
+ self.private_index = BrainIndex(private_index_db)
+ self.private_access_log = Path(
+ os.environ.get("KUBLAI_PRIVATE_ACCESS_LOG", str(Path.home() / ".kublai/private-access-log.ndjson"))
+ ).expanduser()
+ # Phase 2.5 Step 6: gcalcli-backed CalendarService.
+ cal_store = CalendarStore(telemetry_db)
+ gcalcli_bin = (
+ str(gcalcli_path)
+ if gcalcli_path
+ else os.environ.get("GCALCLI_PATH", "/Users/kublai/.brain-migration-venv/bin/gcalcli")
+ )
+ cal_default = default_calendar or os.environ.get("GCALCLI_DEFAULT_CALENDAR")
+ cal_client = GcalcliClient(gcalcli_path=gcalcli_bin, default_calendar=cal_default)
+ self.calendar = CalendarService(
+ cal_store, cal_client,
+ telemetry=self.telemetry,
+ default_calendar=cal_default,
+ )
+ # Phase 2.5 Step 8: humans + messages helpers
+ self.humans = HumansStore(self.wiki_root)
+ msg_path = os.environ.get("KUBLAI_MESSAGES_JSONL", str(Path.home() / ".kublai/messages.jsonl"))
+ self.messages = MessagesStore(msg_path)
+ self.v4 = V4WorkflowService(self)
+
+ def capability_check(self) -> dict[str, Any]:
+ result: dict[str, Any] = {
+ "sqlite_version": sqlite3.sqlite_version,
+ "fts5": False,
+ "json": False,
+ "sqlite_vec": "not_checked",
+ }
+ with sqlite3.connect(":memory:") as conn:
+ options = [row[0] for row in conn.execute("PRAGMA compile_options")]
+ result["fts5"] = any("FTS5" in option for option in options)
+ result["json"] = conn.execute("SELECT json_valid(?)", ('{"ok": true}',)).fetchone()[0] == 1
+ _load_sqlite_vec(conn)
+ result["sqlite_vec"] = _sqlite_vec_version(conn) or "unavailable"
+ if not result["fts5"] or not result["json"]:
+ raise CapabilityError(f"missing SQLite capabilities: {result}")
+ return result
+
+ def reindex(self) -> int:
+ return self.index.reindex(self.wiki_root)
+
+ def reindex_private(self) -> int:
+ return self.private_index.reindex(self.wiki_root, privacy_scope="hard-private")
+
+ def get_node(self, node_type: str, typed_id: str) -> dict[str, Any] | None:
+ return self.index.get_by_typed_id(node_type, typed_id)
+
+ def list_nodes(self, **params: Any) -> list[dict[str, Any]]:
+ return self.index.list_nodes(**params)
+
+ def search(self, **params: Any) -> list[dict[str, Any]]:
+ return self.index.search(**params)
+
+ def search_private(self, *, query: str, requester: str = "daniel-local-kublai", limit: int = 10) -> list[dict[str, Any]]:
+ if requester != "daniel-local-kublai":
+ raise ValueError("private search requires daniel-local-kublai requester")
+ rows = self.private_index.search(query=query, limit=limit)
+ self._audit_private_access("knowledge.search_private", requester, [row["rel_path"] for row in rows])
+ return rows
+
+ def _audit_private_access(self, action: str, requester: str, rel_paths: list[str]) -> None:
+ self.private_access_log.parent.mkdir(parents=True, exist_ok=True)
+ record = {
+ "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+ "action": action,
+ "requester": requester,
+ "rel_paths": rel_paths,
+ }
+ with self.private_access_log.open("a", encoding="utf-8") as handle:
+ handle.write(json.dumps(record, sort_keys=True) + "\n")
+
+ def sweep(self) -> int:
+ count = 0
+ for row in self.telemetry.pending_materializations():
+ try:
+ path = self.knowledge.record_completed_task(
+ task_id=row["id"],
+ agent=row.get("claimed_by") or row.get("assigned_to") or "unknown",
+ delegated_by=row["delegated_by"],
+ completed_at_ms=row["completed_at"],
+ deliverable=row["completion_summary"] or row["description"],
+ results=json.loads(row["results_json"] or "{}"),
+ )
+ self.telemetry.mark_materialized(row["id"], str(path))
+ count += 1
+ except Exception as exc:
+ self.telemetry.mark_materialization_error(row["id"], str(exc))
+ return count
+
+ def health(self) -> dict[str, Any]:
+ health = health_snapshot(self.telemetry, self.wiki_root)
+ health["vector_orphans"] = self.index.vector_orphans()
+ return health
+
+ def backup(self, directory: str | Path) -> dict[str, str]:
+ directory = Path(directory)
+ directory.mkdir(parents=True, exist_ok=True)
+ return {
+ "telemetry": str(self.telemetry.backup_to(directory / "telemetry.db")),
+ "index": str(self.index.backup_to(directory / "brain.db")),
+ }
+
+ def checkpoint(self) -> dict[str, Any]:
+ telemetry_result: tuple[Any, ...]
+ index_result: tuple[Any, ...]
+ with self.telemetry.connect() as conn:
+ telemetry_result = tuple(conn.execute("PRAGMA wal_checkpoint(PASSIVE)").fetchone())
+ with self.index.connect() as conn:
+ index_result = tuple(conn.execute("PRAGMA wal_checkpoint(PASSIVE)").fetchone())
+ return {"telemetry": telemetry_result, "index": index_result}
+
+ def verify_index(self) -> dict[str, Any]:
+ stats = self.index.stats()
+ stats["ok"] = (
+ stats["nodes"] == stats["fts_rows"]
+ and stats["nodes"] == stats["vector_rows"]
+ and stats["vector_orphans"] == 0
+ and stats["private_rows"] == 0
+ and stats["hard_private_rows"] == 0
+ )
+ return stats
+
+ def verify_private_index(self) -> dict[str, Any]:
+ stats = self.private_index.stats()
+ stats["ok"] = (
+ stats["nodes"] == stats["fts_rows"]
+ and stats["nodes"] == stats["vector_rows"]
+ and stats["vector_orphans"] == 0
+ and stats["nodes"] == stats["hard_private_rows"]
+ )
+ return stats
+
+ def replay_dual_write(self, log_path: str | Path | None = None) -> dict[str, Any]:
+ """Verify dual-write log records against materialized wiki files.
+
+ Phase 1 only needs the verifier surface. Phase 2 will populate the JSONL
+ log during soak; each line may include wiki_path and body_hash fields.
+ """
+ if log_path is None:
+ return {"ok": True, "checked": 0, "mismatches": 0, "missing_log": True}
+ path = Path(log_path).expanduser()
+ if not path.exists():
+ return {"ok": True, "checked": 0, "mismatches": 0, "missing_log": True}
+ mismatches: list[dict[str, str]] = []
+ checked = 0
+ for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
+ if not line.strip():
+ continue
+ record = json.loads(line)
+ rel_path = record.get("wiki_path")
+ expected_hash = record.get("body_hash")
+ if not rel_path or not expected_hash:
+ continue
+ checked += 1
+ target = (self.wiki_root / rel_path).resolve()
+ if self.wiki_root not in target.parents or not target.exists():
+ mismatches.append({"line": str(lineno), "wiki_path": str(rel_path), "error": "missing"})
+ continue
+ actual = _sha256(_body_without_frontmatter(target.read_text(encoding="utf-8", errors="ignore")))
+ if actual != expected_hash:
+ mismatches.append({"line": str(lineno), "wiki_path": str(rel_path), "error": "body_hash"})
+ return {"ok": not mismatches, "checked": checked, "mismatches": len(mismatches), "details": mismatches}
+
+ def start_background_tasks(
+ self,
+ *,
+ sweep_interval_s: float = 30.0,
+ checkpoint_interval_s: float = 300.0,
+ ) -> list[threading.Thread]:
+ stop = threading.Event()
+ self._background_stop = stop
+ threads = [
+ threading.Thread(
+ target=self._loop_until_stopped,
+ args=(stop, sweep_interval_s, self.sweep),
+ daemon=True,
+ name="brain-service-sweep",
+ ),
+ threading.Thread(
+ target=self._loop_until_stopped,
+ args=(stop, checkpoint_interval_s, self.checkpoint),
+ daemon=True,
+ name="brain-service-checkpoint",
+ ),
+ ]
+ for thread in threads:
+ thread.start()
+ return threads
+
+ def start_file_watcher(self) -> Any | None:
+ try:
+ from watchdog.events import FileSystemEventHandler
+ from watchdog.observers import Observer
+ except Exception:
+ return None
+
+ service = self
+
+ class Handler(FileSystemEventHandler):
+ def on_any_event(self, event: Any) -> None:
+ if event.is_directory or not str(event.src_path).endswith(".md"):
+ return
+ with contextlib.suppress(Exception):
+ service.reindex()
+
+ observer = Observer()
+ observer.schedule(Handler(), str(self.wiki_root), recursive=True)
+ observer.start()
+ self._watchdog_observer = observer
+ return observer
+
+ @staticmethod
+ def _loop_until_stopped(stop: threading.Event, interval_s: float, callback: Any) -> None:
+ while not stop.wait(interval_s):
+ with contextlib.suppress(Exception):
+ callback()
+
+ def handle_rpc(self, request: dict[str, Any]) -> dict[str, Any]:
+ method = request.get("method")
+ params = request.get("params") or {}
+ try:
+ if method == "telemetry.create_task":
+ return {"ok": True, "result": self.telemetry.create_task(**params)}
+ if method == "telemetry.claim_task":
+ claimed = self.telemetry.claim_task(**params)
+ return {"ok": True, "result": claimed.__dict__}
+ if method == "telemetry.renew_claim":
+ renewed = self.telemetry.renew_claim(**params)
+ return {"ok": True, "result": renewed.__dict__}
+ if method == "telemetry.complete_task":
+ self.telemetry.complete_task(**params)
+ return {"ok": True, "result": None}
+ if method == "telemetry.heartbeat":
+ self.telemetry.heartbeat(**params)
+ return {"ok": True, "result": None}
+ if method == "knowledge.record_reflection":
+ path = self.knowledge.record_reflection(**params)
+ return {"ok": True, "result": str(path)}
+ if method == "knowledge.get":
+ return {"ok": True, "result": self.get_node(**params)}
+ if method == "knowledge.list":
+ return {"ok": True, "result": self.list_nodes(**params)}
+ if method == "knowledge.search":
+ return {"ok": True, "result": self.search(**params)}
+ if method == "knowledge.search_private":
+ return {"ok": True, "result": self.search_private(**params)}
+ if method == "knowledge.list_by_tag":
+ return {"ok": True, "result": self.index.list_by_tag_in_wiki(self.knowledge, **params)}
+ if method == "knowledge.public_search":
+ return {"ok": True, "result": self.v4.public_search(**params)}
+ if method == "knowledge.public_get":
+ return {"ok": True, "result": self.v4.public_get(**params)}
+ if method == "knowledge.public_stub_rebuild":
+ return {"ok": True, "result": self.v4.public_stub_rebuild(**params)}
+ if method == "capture.dry_run":
+ return {"ok": True, "result": self.v4.capture_dry_run(**params)}
+ if method == "capture.apply":
+ return {"ok": True, "result": self.v4.capture_apply(**params)}
+ if method == "ingest.dry_run":
+ return {"ok": True, "result": self.v4.ingest_dry_run(**params)}
+ if method == "ingest.apply":
+ return {"ok": True, "result": self.v4.ingest_apply(**params)}
+ if method == "publish.dry_run":
+ return {"ok": True, "result": self.v4.publish_dry_run(**params)}
+ if method == "publish.apply":
+ return {"ok": True, "result": self.v4.publish_apply(**params)}
+ if method == "research.public_dossier":
+ return {"ok": True, "result": self.v4.research_public_dossier(**params)}
+ if method == "audit.private_summary":
+ return {"ok": True, "result": self.v4.audit_private_summary(**params)}
+ if method == "doctor.full":
+ return {"ok": True, "result": self.v4.doctor_full(**params)}
+ if method == "telemetry.list_reminders":
+ return {"ok": True, "result": self.telemetry.list_reminders(**params)}
+ if method == "telemetry.list_due_reminders":
+ return {"ok": True, "result": self.telemetry.list_due_reminders(**params)}
+ if method == "telemetry.create_reminder":
+ return {"ok": True, "result": self.telemetry.create_reminder(**params)}
+ if method == "telemetry.cancel_reminder":
+ return {"ok": True, "result": self.telemetry.cancel_reminder(**params)}
+ if method == "telemetry.mark_reminder_sent":
+ return {"ok": True, "result": self.telemetry.mark_reminder_sent(**params)}
+ if method == "telemetry.record_reminder_error":
+ return {"ok": True, "result": self.telemetry.record_reminder_error(**params)}
+ if method == "telemetry.replace_event_reminders":
+ return {"ok": True, "result": self.telemetry.replace_event_reminders(**params)}
+ if method == "telemetry.record_operator_action":
+ return {"ok": True, "result": self.telemetry.record_operator_action(**params)}
+ if method == "telemetry.list_operator_actions":
+ return {"ok": True, "result": self.telemetry.list_operator_actions(**params)}
+ if method == "telemetry.list_tasks":
+ return {"ok": True, "result": self.telemetry.list_tasks(**params)}
+ if method == "telemetry.get_task":
+ return {"ok": True, "result": self.telemetry.get_task(**params)}
+ if method == "telemetry.list_task_events":
+ return {"ok": True, "result": self.telemetry.list_task_events(**params)}
+ if method == "telemetry.append_task_event":
+ return {"ok": True, "result": self.telemetry.append_task_event(**params)}
+ if method == "telemetry.list_task_outputs":
+ return {"ok": True, "result": self.telemetry.list_task_outputs(**params)}
+ if method == "telemetry.append_task_output":
+ return {"ok": True, "result": self.telemetry.append_task_output(**params)}
+ if method == "telemetry.list_task_outcomes":
+ return {"ok": True, "result": self.telemetry.list_task_outcomes(**params)}
+ if method == "telemetry.append_task_outcome":
+ return {"ok": True, "result": self.telemetry.append_task_outcome(**params)}
+ if method == "telemetry.list_failure_reports":
+ return {"ok": True, "result": self.telemetry.list_failure_reports(**params)}
+ if method == "telemetry.append_failure_report":
+ return {"ok": True, "result": self.telemetry.append_failure_report(**params)}
+ if method == "telemetry.task_analytics":
+ return {"ok": True, "result": self.telemetry.task_analytics(**params)}
+ if method == "telemetry.cap_retry_count":
+ return {"ok": True, "result": self.telemetry.cap_retry_count(**params)}
+ if method == "telemetry.cancel_tasks":
+ return {"ok": True, "result": self.telemetry.cancel_tasks(**params)}
+ if method == "telemetry.reset_to_pending":
+ return {"ok": True, "result": self.telemetry.reset_to_pending(**params)}
+ if method == "telemetry.recover_expired_claims":
+ return {"ok": True, "result": self.telemetry.recover_expired_claims(**params)}
+ if method == "telemetry.promote_orphaned_tasks":
+ return {"ok": True, "result": self.telemetry.promote_orphaned_tasks(**params)}
+ if method == "telemetry.promote_ready_pipeline_tasks":
+ return {"ok": True, "result": self.telemetry.promote_ready_pipeline_tasks(**params)}
+ if method == "telemetry.list_pipeline_status":
+ return {"ok": True, "result": self.telemetry.list_pipeline_status(**params)}
+ if method == "telemetry.cleanup_pipeline":
+ return {"ok": True, "result": self.telemetry.cleanup_pipeline(**params)}
+ if method == "telemetry.list_terminal_tasks":
+ return {"ok": True, "result": self.telemetry.list_terminal_tasks(**params)}
+ # ---- Phase 3 step 11: task CRUD RPCs (the-kurultai dashboard) ----
+ if method == "telemetry.create_task_full":
+ return {"ok": True, "result": self.telemetry.create_task_full(**params)}
+ if method == "telemetry.set_task_status":
+ return {"ok": True, "result": self.telemetry.set_task_status(**params)}
+ if method == "telemetry.retry_task":
+ return {"ok": True, "result": self.telemetry.retry_task(**params)}
+ if method == "telemetry.redo_task":
+ return {"ok": True, "result": self.telemetry.redo_task(**params)}
+ if method == "telemetry.retry_all_tasks":
+ return {"ok": True, "result": self.telemetry.retry_all_tasks(**params)}
+ if method == "telemetry.set_task_obsolete":
+ return {"ok": True, "result": self.telemetry.set_task_obsolete(**params)}
+ if method == "telemetry.revert_task_status":
+ return {"ok": True, "result": self.telemetry.revert_task_status(**params)}
+ if method == "telemetry.revert_task_prompt":
+ return {"ok": True, "result": self.telemetry.revert_task_prompt(**params)}
+ if method == "telemetry.reassign_task":
+ return {"ok": True, "result": self.telemetry.reassign_task(**params)}
+ if method == "telemetry.update_task_prompt":
+ return {"ok": True, "result": self.telemetry.update_task_prompt(**params)}
+ if method == "telemetry.delete_task":
+ return {"ok": True, "result": self.telemetry.delete_task(**params)}
+ if method == "telemetry.pause_task":
+ return {"ok": True, "result": self.telemetry.pause_task(**params)}
+ if method == "telemetry.unpause_task":
+ return {"ok": True, "result": self.telemetry.unpause_task(**params)}
+ if method == "telemetry.reorder_tasks":
+ return {"ok": True, "result": self.telemetry.reorder_tasks(**params)}
+ if method == "telemetry.move_task_to_top":
+ return {"ok": True, "result": self.telemetry.move_task_to_top(**params)}
+ if method == "telemetry.bulk_reassign_tasks":
+ return {"ok": True, "result": self.telemetry.bulk_reassign_tasks(**params)}
+ if method == "telemetry.get_task_with_output":
+ return {"ok": True, "result": self.telemetry.get_task_with_output(**params)}
+ if method == "knowledge.list_proposals":
+ return {"ok": True, "result": self.knowledge.list_proposals(**params)}
+ if method == "humans.list":
+ return {"ok": True, "result": self.humans.list(**params)}
+ if method == "humans.get":
+ return {"ok": True, "result": self.humans.get(**params)}
+ if method == "humans.update_consent":
+ return {"ok": True, "result": self.humans.update_consent(**params)}
+ if method == "messages.list_recent":
+ return {"ok": True, "result": self.messages.list_recent(**params)}
+ if method == "messages.append":
+ return {"ok": True, "result": self.messages.append(**params)}
+ if method == "messages.search":
+ return {"ok": True, "result": self.messages.search(**params)}
+ if method == "calendar.list_events":
+ return {"ok": True, "result": self.calendar.list_events(**params)}
+ if method == "calendar.create_event":
+ return {"ok": True, "result": self.calendar.create_event(**params)}
+ if method == "calendar.update_event":
+ return {"ok": True, "result": self.calendar.update_event(**params)}
+ if method == "calendar.cancel_event":
+ return {"ok": True, "result": self.calendar.cancel_event(**params)}
+ if method == "calendar.list_due_reminders":
+ return {"ok": True, "result": self.calendar.list_due_reminders(**params)}
+ if method == "calendar.health":
+ return {"ok": True, "result": self.calendar.health()}
+ if method == "health":
+ return {"ok": True, "result": self.health()}
+ raise ValueError(f"unknown method: {method}")
+ except (NoPendingTaskError, StaleClaimError, ValueError, TypeError, CalendarError) as exc:
+ return {"ok": False, "error": type(exc).__name__, "message": str(exc)}
+
+ def serve_socket(self, socket_path: str | Path) -> socketserver.UnixStreamServer:
+ socket_path = Path(socket_path)
+ if socket_path.exists():
+ socket_path.unlink()
+ service = self
+
+ class Handler(socketserver.StreamRequestHandler):
+ def handle(self) -> None:
+ for line in self.rfile:
+ request = json.loads(line.decode("utf-8"))
+ response = service.handle_rpc(request)
+ try:
+ self.wfile.write((json.dumps(response, sort_keys=True) + "\n").encode("utf-8"))
+ except BrokenPipeError:
+ return
+
+ server = socketserver.ThreadingUnixStreamServer(str(socket_path), Handler)
+ os.chmod(socket_path, 0o600)
+ return server
+
+
+def _body_without_frontmatter(text: str) -> str:
+ if not text.startswith("---\n"):
+ return text
+ end = text.find("\n---\n", 4)
+ return text[end + 5 :] if end >= 0 else text
+
+
+def _sha256(text: str) -> str:
+ import hashlib
+
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def _fts5_phrase(query: str) -> str:
+ escaped = query.strip().replace('"', '""')
+ return f'"{escaped}"'
+
+
+def _node_row_to_dict(row: sqlite3.Row) -> dict[str, Any]:
+ value = dict(row)
+ value["frontmatter"] = json.loads(value["frontmatter"])
+ value["body_text"] = value.pop("body_text")
+ return value
+
+
+def _first_present(data: dict[str, Any], *keys: str) -> str | None:
+ for key in keys:
+ value = data.get(key)
+ if value:
+ return str(value)
+ return None
+
+
+def _load_sqlite_vec(conn: sqlite3.Connection) -> bool:
+ if not hasattr(conn, "load_extension"):
+ return False
+ try:
+ if hasattr(conn, "enable_load_extension"):
+ conn.enable_load_extension(True)
+ import sqlite_vec
+
+ sqlite_vec.load(conn)
+ return True
+ except Exception:
+ return False
+
+
+def _sqlite_vec_loaded(conn: sqlite3.Connection) -> bool:
+ return _sqlite_vec_version(conn) is not None
+
+
+def _sqlite_vec_version(conn: sqlite3.Connection) -> str | None:
+ try:
+ return str(conn.execute("SELECT vec_version()").fetchone()[0])
+ except Exception:
+ return None
+
+
+def _hash_embedding(text: str, dim: int = 384) -> list[float]:
+ import hashlib
+
+ digest = hashlib.sha256(text.encode("utf-8")).digest()
+ values = []
+ while len(values) < dim:
+ for byte in digest:
+ values.append((byte / 127.5) - 1.0)
+ if len(values) == dim:
+ break
+ digest = hashlib.sha256(digest).digest()
+ return values
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(prog="brain-service")
+ parser.add_argument("--wiki-root", default=os.getenv("BRAIN_WIKI_ROOT", str(Path.home() / "brain")))
+ parser.add_argument("--telemetry-db", default=os.getenv("KUBLAI_TELEMETRY_DB", str(Path.home() / ".kublai/telemetry.db")))
+ parser.add_argument("--index-db", default=os.getenv("BRAIN_INDEX_DB", str(Path.home() / ".brain-index/brain.db")))
+ sub = parser.add_subparsers(dest="command", required=True)
+ sub.add_parser("healthcheck")
+ sub.add_parser("reindex")
+ sub.add_parser("reindex-private")
+ sub.add_parser("verify-index")
+ sub.add_parser("verify-private-index")
+ replay = sub.add_parser("replay-dual-write")
+ replay.add_argument("--log", default=os.getenv("BRAIN_DUAL_WRITE_LOG"))
+ sub.add_parser("sweep")
+ backup = sub.add_parser("backup")
+ backup.add_argument("directory")
+ serve = sub.add_parser("serve")
+ serve.add_argument("--socket", default=os.getenv("BRAIN_SERVICE_SOCKET", "/tmp/brain-service.sock"))
+ gateway = sub.add_parser("serve-gateway")
+ gateway.add_argument("--host", default=os.getenv("KUBLAI_GATEWAY_HOST", "127.0.0.1"))
+ gateway.add_argument("--port", type=int, default=int(os.getenv("KUBLAI_GATEWAY_PORT", "8765")))
+ gateway.add_argument("--secret", default=os.getenv("KUBLAI_GATEWAY_HMAC_SECRET"))
+ gateway.add_argument("--secret-file", default=os.getenv("KUBLAI_GATEWAY_HMAC_SECRET_FILE"))
+ gateway.add_argument("--certfile", default=os.getenv("KUBLAI_GATEWAY_TLS_CERT"))
+ gateway.add_argument("--keyfile", default=os.getenv("KUBLAI_GATEWAY_TLS_KEY"))
+ return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+ args = build_parser().parse_args(argv)
+ service = BrainService(args.wiki_root, args.telemetry_db, args.index_db)
+ if args.command == "healthcheck":
+ service.capability_check()
+ print(json.dumps(service.health(), sort_keys=True))
+ return 0
+ if args.command == "reindex":
+ print(json.dumps({"indexed": service.reindex()}))
+ return 0
+ if args.command == "reindex-private":
+ print(json.dumps({"indexed": service.reindex_private()}))
+ return 0
+ if args.command == "verify-index":
+ result = service.verify_index()
+ print(json.dumps(result, sort_keys=True))
+ return 0 if result["ok"] else 1
+ if args.command == "verify-private-index":
+ result = service.verify_private_index()
+ print(json.dumps(result, sort_keys=True))
+ return 0 if result["ok"] else 1
+ if args.command == "replay-dual-write":
+ result = service.replay_dual_write(args.log)
+ print(json.dumps(result, sort_keys=True))
+ return 0 if result["ok"] else 1
+ if args.command == "sweep":
+ print(json.dumps({"materialized": service.sweep()}))
+ return 0
+ if args.command == "backup":
+ print(json.dumps(service.backup(args.directory), sort_keys=True))
+ return 0
+ if args.command == "serve":
+ service.start_background_tasks()
+ observer = service.start_file_watcher()
+ server = service.serve_socket(args.socket)
+ thread = threading.Thread(target=server.serve_forever, daemon=False)
+ thread.start()
+ try:
+ while thread.is_alive():
+ time.sleep(1)
+ except KeyboardInterrupt:
+ server.shutdown()
+ if observer is not None:
+ observer.stop()
+ observer.join(timeout=5)
+ return 0
+ if args.command == "serve-gateway":
+ from .v4_gateway import serve_gateway
+
+ secret = args.secret
+ if args.secret_file:
+ secret = Path(args.secret_file).expanduser().read_text(encoding="utf-8").strip()
+ if not secret:
+ raise ValueError("gateway secret file is empty")
+ server = serve_gateway(
+ service,
+ host=args.host,
+ port=args.port,
+ secret=secret,
+ certfile=args.certfile,
+ keyfile=args.keyfile,
+ )
+ try:
+ server.serve_forever()
+ except KeyboardInterrupt:
+ server.shutdown()
+ return 0
+ return 2
+
+
+if __name__ == "__main__":
+ raise SystemExit(main(sys.argv[1:]))
diff --git a/kublai/brain_service_client.py b/kublai/brain_service_client.py
new file mode 100644
index 00000000..d66b050a
--- /dev/null
+++ b/kublai/brain_service_client.py
@@ -0,0 +1,21 @@
+"""Python helper client for the local brain-service Unix socket."""
+
+from __future__ import annotations
+
+import json
+import socket
+from pathlib import Path
+from typing import Any
+
+
+def call(socket_path: str | Path, method: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
+ with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
+ sock.connect(str(socket_path))
+ sock.sendall((json.dumps({"method": method, "params": params or {}}) + "\n").encode("utf-8"))
+ data = b""
+ while not data.endswith(b"\n"):
+ chunk = sock.recv(65536)
+ if not chunk:
+ break
+ data += chunk
+ return json.loads(data.decode("utf-8"))
diff --git a/kublai/cache.py b/kublai/cache.py
new file mode 100644
index 00000000..6059db4c
--- /dev/null
+++ b/kublai/cache.py
@@ -0,0 +1,16 @@
+"""Cache-key helpers for v4 prompt/model work."""
+
+from __future__ import annotations
+
+import hashlib
+
+
+def cache_key(body: str | bytes, model_family: str, prompt_template_hash: str) -> str:
+ body_bytes = body if isinstance(body, bytes) else body.encode("utf-8")
+ digest = hashlib.sha256()
+ digest.update(hashlib.sha256(body_bytes).hexdigest().encode("ascii"))
+ digest.update(b"\0")
+ digest.update(model_family.encode("utf-8"))
+ digest.update(b"\0")
+ digest.update(prompt_template_hash.encode("utf-8"))
+ return digest.hexdigest()
diff --git a/kublai/calendar.py b/kublai/calendar.py
new file mode 100644
index 00000000..a0335072
--- /dev/null
+++ b/kublai/calendar.py
@@ -0,0 +1,473 @@
+"""Phase 2.5 Step 6 — calendar helper for brain-service.
+
+Wraps ``gcalcli`` subprocess as the source of truth for CalendarEvent reads
+and writes, with a short-TTL cache in ``calendar_event_cache``. Replaces the
+Neo4j ``CalendarEvent`` access pattern that used to live in ``server.js``.
+
+Public surface used by ``brain_service.handle_rpc``:
+
+* ``CalendarService.list_events(time_min, time_max, person=None, calendar=None)``
+* ``CalendarService.create_event(...)``
+* ``CalendarService.update_event(...)``
+* ``CalendarService.cancel_event(...)``
+* ``CalendarService.list_due_reminders(now_ms=None, limit=50)``
+* ``CalendarService.health()``
+
+The ``server.js`` calendar route migration in Step 8/9 calls these via the
+brain-service Unix socket.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sqlite3
+import subprocess
+import time
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Iterator
+
+
+DEFAULT_GCALCLI = "/Users/kublai/.brain-migration-venv/bin/gcalcli"
+DEFAULT_CACHE_TTL_SECONDS = 300
+
+
+class CalendarError(Exception):
+ """Generic calendar failure."""
+
+
+class CalendarUnavailableError(CalendarError):
+ """Raised when gcalcli is missing or auth has expired."""
+
+
+class CalendarValidationError(CalendarError):
+ """Raised on invalid caller input."""
+
+
+@dataclass(frozen=True)
+class CalendarEvent:
+ id: str
+ external_id: str | None
+ provider: str
+ calendar_id: str | None
+ title: str
+ start_at: int
+ end_at: int | None
+ status: str
+ payload: dict[str, Any]
+ fetched_at: int
+ expires_at: int
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "id": self.id,
+ "external_id": self.external_id,
+ "provider": self.provider,
+ "calendar_id": self.calendar_id,
+ "title": self.title,
+ "start_at": self.start_at,
+ "end_at": self.end_at,
+ "status": self.status,
+ "payload": self.payload,
+ "fetched_at": self.fetched_at,
+ "expires_at": self.expires_at,
+ }
+
+
+def _parse_iso_or_ms(value: str | int | float) -> int:
+ if isinstance(value, (int, float)):
+ return int(value)
+ s = str(value).strip()
+ if s.lstrip("-").isdigit():
+ return int(s)
+ if s.endswith("Z"):
+ s = s[:-1] + "+00:00"
+ try:
+ dt = datetime.fromisoformat(s)
+ except ValueError as exc:
+ raise CalendarValidationError(f"invalid datetime: {value!r}") from exc
+ if dt.tzinfo is None:
+ dt = dt.replace(tzinfo=timezone.utc)
+ return int(dt.timestamp() * 1000)
+
+
+def _ms_to_iso(ms: int | None) -> str | None:
+ if ms is None:
+ return None
+ return datetime.fromtimestamp(ms / 1000, tz=timezone.utc).isoformat()
+
+
+def _ms_to_gcalcli_date(ms: int) -> str:
+ return datetime.fromtimestamp(ms / 1000, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M")
+
+
+class CalendarStore:
+ """SQLite-backed cache for CalendarEvent reads."""
+
+ def __init__(self, db_path: str | Path):
+ self.db_path = Path(db_path).expanduser()
+
+ @contextmanager
+ def connect(self) -> Iterator[sqlite3.Connection]:
+ conn = sqlite3.connect(self.db_path, timeout=5.0, isolation_level=None)
+ conn.row_factory = sqlite3.Row
+ conn.execute("PRAGMA busy_timeout = 5000")
+ conn.execute("PRAGMA foreign_keys = ON")
+ try:
+ yield conn
+ finally:
+ conn.close()
+
+ def upsert(self, event: CalendarEvent) -> None:
+ with self.connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO calendar_event_cache
+ (id, external_id, provider, calendar_id, title,
+ start_at, end_at, status, payload_json, fetched_at, expires_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(id) DO UPDATE SET
+ external_id = excluded.external_id,
+ provider = excluded.provider,
+ calendar_id = excluded.calendar_id,
+ title = excluded.title,
+ start_at = excluded.start_at,
+ end_at = excluded.end_at,
+ status = excluded.status,
+ payload_json = excluded.payload_json,
+ fetched_at = excluded.fetched_at,
+ expires_at = excluded.expires_at
+ """,
+ (
+ event.id, event.external_id, event.provider, event.calendar_id,
+ event.title, event.start_at, event.end_at, event.status,
+ json.dumps(event.payload, ensure_ascii=False, sort_keys=True),
+ event.fetched_at, event.expires_at,
+ ),
+ )
+
+ def query_window(self, *, time_min_ms: int, time_max_ms: int,
+ now_ms: int | None = None) -> list[CalendarEvent]:
+ now_ms = now_ms if now_ms is not None else int(time.time() * 1000)
+ with self.connect() as conn:
+ rows = conn.execute(
+ """
+ SELECT * FROM calendar_event_cache
+ WHERE start_at >= ? AND start_at < ? AND expires_at >= ?
+ ORDER BY start_at
+ """,
+ (time_min_ms, time_max_ms, now_ms),
+ ).fetchall()
+ return [self._row_to_event(r) for r in rows]
+
+ def is_window_fresh(self, *, time_min_ms: int, time_max_ms: int,
+ now_ms: int | None = None) -> bool:
+ now_ms = now_ms if now_ms is not None else int(time.time() * 1000)
+ with self.connect() as conn:
+ row = conn.execute(
+ "SELECT min(expires_at) AS exp, count(*) AS n FROM calendar_event_cache "
+ "WHERE start_at >= ? AND start_at < ?",
+ (time_min_ms, time_max_ms),
+ ).fetchone()
+ if not row or row["n"] == 0:
+ return False
+ return row["exp"] is not None and row["exp"] >= now_ms
+
+ def purge_expired(self, *, now_ms: int | None = None) -> int:
+ now_ms = now_ms if now_ms is not None else int(time.time() * 1000)
+ with self.connect() as conn:
+ cur = conn.execute("DELETE FROM calendar_event_cache WHERE expires_at < ?", (now_ms,))
+ return cur.rowcount
+
+ def delete(self, event_id: str) -> None:
+ with self.connect() as conn:
+ conn.execute("DELETE FROM calendar_event_cache WHERE id = ?", (event_id,))
+
+ @staticmethod
+ def _row_to_event(row: sqlite3.Row) -> CalendarEvent:
+ payload = json.loads(row["payload_json"]) if row["payload_json"] else {}
+ return CalendarEvent(
+ id=row["id"], external_id=row["external_id"],
+ provider=row["provider"], calendar_id=row["calendar_id"],
+ title=row["title"], start_at=row["start_at"], end_at=row["end_at"],
+ status=row["status"], payload=payload,
+ fetched_at=row["fetched_at"], expires_at=row["expires_at"],
+ )
+
+
+class GcalcliClient:
+ """Subprocess wrapper for gcalcli on the Mac mini."""
+
+ def __init__(self, *, gcalcli_path: str | Path = DEFAULT_GCALCLI,
+ default_calendar: str | None = None,
+ timeout_seconds: float = 30.0):
+ self.gcalcli_path = str(gcalcli_path)
+ self.default_calendar = default_calendar
+ self.timeout_seconds = timeout_seconds
+
+ def _run(self, args: list[str], *, stdin: str | None = None) -> subprocess.CompletedProcess:
+ cmd = [self.gcalcli_path, *args]
+ try:
+ return subprocess.run(
+ cmd, input=stdin, capture_output=True, text=True,
+ timeout=self.timeout_seconds, check=False,
+ )
+ except FileNotFoundError as exc:
+ raise CalendarUnavailableError(f"gcalcli not found at {self.gcalcli_path}") from exc
+
+ def list_calendars(self) -> list[dict[str, str]]:
+ result = self._run(["--nocolor", "list"])
+ if result.returncode != 0:
+ raise CalendarUnavailableError(f"gcalcli list failed: {result.stderr.strip()}")
+ out: list[dict[str, str]] = []
+ for line in result.stdout.splitlines():
+ stripped = line.strip()
+ if not stripped or stripped.startswith("Access") or stripped.startswith("------"):
+ continue
+ access, _, title = stripped.partition(" ")
+ if title:
+ out.append({"access": access, "title": title.strip()})
+ return out
+
+ def agenda_tsv(self, *, time_min: str, time_max: str,
+ calendar: str | None = None) -> list[dict[str, str]]:
+ args = ["--nocolor"]
+ cal = calendar or self.default_calendar
+ if cal:
+ args.extend(["--calendar", cal])
+ args.extend(["agenda", "--tsv", time_min, time_max])
+ result = self._run(args)
+ if result.returncode != 0:
+ stderr = result.stderr.strip()
+ if "credentials" in stderr.lower() or "auth" in stderr.lower():
+ raise CalendarUnavailableError(stderr)
+ raise CalendarError(f"gcalcli agenda failed: {stderr}")
+ events: list[dict[str, str]] = []
+ for line in result.stdout.splitlines():
+ if not line.strip():
+ continue
+ cols = line.split("\t")
+ # gcalcli >= 4.x emits a header row plus 5-column rows:
+ # start_date, start_time, end_date, end_time, title
+ # Older versions emit a 6-column form with a leading link column;
+ # detect by header-row "start_date" first cell, then route columns.
+ if cols[0] == "start_date":
+ continue
+ if len(cols) >= 6 and cols[4].startswith("http"):
+ events.append({
+ "start_date": cols[0], "start_time": cols[1],
+ "end_date": cols[2], "end_time": cols[3],
+ "link": cols[4], "title": cols[5],
+ })
+ elif len(cols) >= 5:
+ events.append({
+ "start_date": cols[0], "start_time": cols[1],
+ "end_date": cols[2], "end_time": cols[3],
+ "link": "", "title": cols[4],
+ })
+ return events
+
+ def add_event(self, *, title: str, when: str, length_minutes: int | None = None,
+ where: str | None = None, description: str | None = None,
+ reminders: list[int] | None = None,
+ calendar: str | None = None) -> dict[str, Any]:
+ args = ["--nocolor"]
+ cal = calendar or self.default_calendar
+ if cal:
+ args.extend(["--calendar", cal])
+ args.extend(["add", "--title", title, "--when", when])
+ if length_minutes is not None:
+ args.extend(["--duration", str(length_minutes)])
+ if where:
+ args.extend(["--where", where])
+ if description:
+ args.extend(["--description", description])
+ for offset in reminders or []:
+ args.extend(["--reminder", str(int(offset))])
+ args.append("--default-reminders")
+ result = self._run(args)
+ if result.returncode != 0:
+ raise CalendarError(
+ f"gcalcli add failed: {result.stderr.strip() or result.stdout.strip()}"
+ )
+ return {"stdout": result.stdout, "stderr": result.stderr}
+
+ def edit_events(self, *, search_text: str, action: str = "title",
+ value: str | None = None,
+ calendar: str | None = None) -> dict[str, Any]:
+ args = ["--nocolor"]
+ cal = calendar or self.default_calendar
+ if cal:
+ args.extend(["--calendar", cal])
+ args.extend(["edit", search_text])
+ result = self._run(args, stdin=f"{action}\n{value or ''}\nq\n")
+ if result.returncode != 0:
+ raise CalendarError(
+ f"gcalcli edit failed: {result.stderr.strip() or result.stdout.strip()}"
+ )
+ return {"stdout": result.stdout, "stderr": result.stderr}
+
+ def delete_events(self, *, search_text: str,
+ calendar: str | None = None) -> dict[str, Any]:
+ args = ["--nocolor"]
+ cal = calendar or self.default_calendar
+ if cal:
+ args.extend(["--calendar", cal])
+ args.extend(["delete", search_text])
+ result = self._run(args, stdin="y\n" * 50)
+ if result.returncode != 0:
+ raise CalendarError(
+ f"gcalcli delete failed: {result.stderr.strip() or result.stdout.strip()}"
+ )
+ return {"stdout": result.stdout, "stderr": result.stderr}
+
+
+_EID_RE = re.compile(r"[?&]eid=([^&\s]+)")
+
+
+def _row_to_event(row: dict[str, str], *, fetched_at: int, ttl_seconds: int,
+ default_calendar: str | None) -> CalendarEvent:
+ is_all_day = row["start_time"].strip().lower() in ("", "all-day")
+ start_date = row["start_date"]
+ end_date = row["end_date"] or start_date
+ if is_all_day:
+ start_at = _parse_iso_or_ms(start_date + "T00:00:00+00:00")
+ end_at = _parse_iso_or_ms(end_date + "T00:00:00+00:00")
+ else:
+ start_at = _parse_iso_or_ms(f"{start_date}T{row['start_time']}")
+ if row["end_time"] and row["end_time"].lower() not in ("", "all-day"):
+ end_at = _parse_iso_or_ms(f"{end_date}T{row['end_time']}")
+ else:
+ end_at = None
+ link = row.get("link") or ""
+ eid_match = _EID_RE.search(link)
+ external_id = eid_match.group(1) if eid_match else None
+ cache_id = external_id or f"{start_date}|{row['start_time']}|{row['title']}"
+ payload = {
+ "link": link,
+ "is_all_day": is_all_day,
+ "raw_row": row,
+ "calendar_hint": default_calendar,
+ }
+ return CalendarEvent(
+ id=cache_id,
+ external_id=external_id,
+ provider="google-calendar",
+ calendar_id=default_calendar,
+ title=row["title"],
+ start_at=start_at,
+ end_at=end_at,
+ status="active",
+ payload=payload,
+ fetched_at=fetched_at,
+ expires_at=fetched_at + ttl_seconds * 1000,
+ )
+
+
+class CalendarService:
+ """High-level calendar API for brain-service."""
+
+ def __init__(self, store: CalendarStore, client: GcalcliClient,
+ telemetry: Any | None = None, *,
+ default_ttl_seconds: int = DEFAULT_CACHE_TTL_SECONDS,
+ default_calendar: str | None = None):
+ self.store = store
+ self.client = client
+ self.telemetry = telemetry
+ self.default_ttl_seconds = default_ttl_seconds
+ self.default_calendar = default_calendar
+
+ def health(self) -> dict[str, Any]:
+ try:
+ calendars = self.client.list_calendars()
+ return {
+ "ok": True,
+ "calendars": calendars,
+ "gcalcli": self.client.gcalcli_path,
+ "default_calendar": self.default_calendar,
+ }
+ except CalendarUnavailableError as exc:
+ return {"ok": False, "error": str(exc), "gcalcli": self.client.gcalcli_path}
+
+ def list_events(self, *, time_min: str | int, time_max: str | int,
+ person: str | None = None, calendar: str | None = None,
+ use_cache: bool = True) -> list[dict[str, Any]]:
+ time_min_ms = _parse_iso_or_ms(time_min)
+ time_max_ms = _parse_iso_or_ms(time_max)
+ if time_max_ms < time_min_ms:
+ raise CalendarValidationError("time_max must be >= time_min")
+
+ now_ms = int(time.time() * 1000)
+ cal = calendar or self.default_calendar
+
+ if use_cache and self.store.is_window_fresh(
+ time_min_ms=time_min_ms, time_max_ms=time_max_ms, now_ms=now_ms,
+ ):
+ cached = self.store.query_window(
+ time_min_ms=time_min_ms, time_max_ms=time_max_ms, now_ms=now_ms,
+ )
+ return [
+ ev.to_dict() for ev in cached if self._matches_person(ev, person)
+ ]
+
+ rows = self.client.agenda_tsv(
+ time_min=_ms_to_gcalcli_date(time_min_ms),
+ time_max=_ms_to_gcalcli_date(time_max_ms),
+ calendar=cal,
+ )
+ events = [
+ _row_to_event(
+ r, fetched_at=now_ms,
+ ttl_seconds=self.default_ttl_seconds,
+ default_calendar=cal,
+ )
+ for r in rows
+ ]
+ for ev in events:
+ self.store.upsert(ev)
+ return [ev.to_dict() for ev in events if self._matches_person(ev, person)]
+
+ @staticmethod
+ def _matches_person(event: CalendarEvent, person: str | None) -> bool:
+ if not person:
+ return True
+ haystack = (event.title or "").lower()
+ return person.lower() in haystack
+
+ def create_event(self, *, title: str, when: str, length_minutes: int | None = None,
+ where: str | None = None, description: str | None = None,
+ reminders: list[int] | None = None,
+ calendar: str | None = None) -> dict[str, Any]:
+ cal = calendar or self.default_calendar
+ result = self.client.add_event(
+ title=title, when=when, length_minutes=length_minutes,
+ where=where, description=description, reminders=reminders, calendar=cal,
+ )
+ self.store.purge_expired()
+ return result
+
+ def update_event(self, *, search_text: str, action: str = "title",
+ value: str | None = None,
+ calendar: str | None = None) -> dict[str, Any]:
+ cal = calendar or self.default_calendar
+ result = self.client.edit_events(
+ search_text=search_text, action=action, value=value, calendar=cal,
+ )
+ self.store.purge_expired()
+ return result
+
+ def cancel_event(self, *, search_text: str,
+ calendar: str | None = None) -> dict[str, Any]:
+ cal = calendar or self.default_calendar
+ result = self.client.delete_events(search_text=search_text, calendar=cal)
+ self.store.purge_expired()
+ return result
+
+ def list_due_reminders(self, *, now_ms: int | None = None,
+ limit: int = 50) -> list[dict[str, Any]]:
+ if self.telemetry is None:
+ raise CalendarError("telemetry not wired on calendar service")
+ return self.telemetry.list_due_reminders(now_ms=now_ms, limit=limit)
diff --git a/kublai/context_bundle.py b/kublai/context_bundle.py
new file mode 100644
index 00000000..617b8072
--- /dev/null
+++ b/kublai/context_bundle.py
@@ -0,0 +1,83 @@
+"""Privacy-tagged LLM context bundles.
+
+All LLM-facing calls should carry provenance instead of raw, untyped strings.
+The sanitizer and LLM boundary can then reject private material before any
+provider request is built.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+PrivacyClass = Literal["public", "private", "hard-private"]
+
+PRIVACY_ORDER: dict[str, int] = {
+ "public": 0,
+ "private": 1,
+ "hard-private": 2,
+}
+
+
+class ContextBundleError(ValueError):
+ """Raised when context provenance is missing or inconsistent."""
+
+
+@dataclass(frozen=True)
+class Source:
+ rel_path: str
+ privacy_class: PrivacyClass
+ title: str | None = None
+ frontmatter: dict[str, Any] = field(default_factory=dict)
+ content: str = ""
+
+ def __post_init__(self) -> None:
+ rel_path = self.rel_path.strip()
+ if not rel_path or rel_path.startswith("/") or ".." in rel_path.split("/"):
+ raise ContextBundleError(f"unresolvable rel_path: {self.rel_path!r}")
+ if self.privacy_class not in PRIVACY_ORDER:
+ raise ContextBundleError(f"invalid privacy_class: {self.privacy_class!r}")
+
+
+@dataclass(frozen=True)
+class ContextBundle:
+ sources: tuple[Source, ...]
+ purpose: str = ""
+ instructions: str = ""
+
+ def __init__(
+ self,
+ sources: list[Source] | tuple[Source, ...],
+ *,
+ purpose: str = "",
+ instructions: str = "",
+ ) -> None:
+ if not sources:
+ raise ContextBundleError("ContextBundle requires at least one Source")
+ object.__setattr__(self, "sources", tuple(sources))
+ object.__setattr__(self, "purpose", purpose)
+ object.__setattr__(self, "instructions", instructions)
+
+ @property
+ def privacy_class(self) -> PrivacyClass:
+ return max(self.sources, key=lambda source: PRIVACY_ORDER[source.privacy_class]).privacy_class
+
+ def render(self) -> str:
+ parts: list[str] = []
+ if self.purpose:
+ parts.append(f"Purpose: {self.purpose}")
+ if self.instructions:
+ parts.append(f"Instructions: {self.instructions}")
+ for source in self.sources:
+ title = source.title or source.rel_path
+ parts.append(
+ "\n".join(
+ [
+ f"Source: {source.rel_path}",
+ f"Privacy: {source.privacy_class}",
+ f"Title: {title}",
+ source.content,
+ ]
+ ).strip()
+ )
+ return "\n\n---\n\n".join(parts)
diff --git a/kublai/doctor.py b/kublai/doctor.py
new file mode 100644
index 00000000..29c2b6e4
--- /dev/null
+++ b/kublai/doctor.py
@@ -0,0 +1,158 @@
+"""Kublai operational doctor checks."""
+
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+import stat
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+
+EXPECTED_LABELS = (
+ "ai.kurultai.brain-service",
+ "ai.kurultai.brain-public-gateway",
+ "ai.kurultai.brain-reconciliation",
+ "ai.hermes.gateway",
+ "com.kurultai.daily-backup",
+)
+RETIRED_GRAPH_LABEL = "homebrew.mxcl.neo4j"
+RETIRED_GRAPH_PORTS = ("7474", "7687")
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+
+def _mode(path: Path) -> int | None:
+ try:
+ return stat.S_IMODE(path.stat().st_mode)
+ except FileNotFoundError:
+ return None
+
+
+def _run(args: list[str]) -> subprocess.CompletedProcess[str]:
+ return subprocess.run(args, text=True, capture_output=True, check=False)
+
+
+def _port_dark(port: str) -> bool:
+ result = _run(["lsof", "-nP", f"-iTCP:{port}", "-sTCP:LISTEN"])
+ return result.returncode != 0 or not result.stdout.strip()
+
+
+def _brain_service_ok(brain_root: Path) -> bool:
+ result = _run([
+ sys.executable,
+ "-m",
+ "kublai.brain_service",
+ "--wiki-root",
+ str(brain_root),
+ "--telemetry-db",
+ "/Users/kublai/.kublai/telemetry.db",
+ "--index-db",
+ "/Users/kublai/.brain-index/brain.db",
+ "healthcheck",
+ ])
+ if result.returncode != 0:
+ return False
+ try:
+ return bool(json.loads(result.stdout).get("ok"))
+ except Exception:
+ return False
+
+
+def _doctor_full_ok(brain_root: Path) -> bool:
+ try:
+ from .brain_service import BrainService
+
+ service = BrainService(
+ brain_root,
+ "/Users/kublai/.kublai/telemetry.db",
+ "/Users/kublai/.brain-index/brain.db",
+ )
+ return bool(service.v4.doctor_full().get("ok"))
+ except Exception:
+ return False
+
+
+def _gateway_ok() -> bool:
+ result = _run(["curl", "-fsS", "http://127.0.0.1:8765/health"])
+ if result.returncode != 0:
+ return False
+ try:
+ return bool(json.loads(result.stdout).get("ok"))
+ except Exception:
+ return False
+
+
+def _index_hard_private_rows(db_path: Path) -> int | None:
+ if not db_path.exists():
+ return None
+ try:
+ with sqlite3.connect(db_path) as conn:
+ row = conn.execute(
+ "SELECT count(*) FROM nodes WHERE rel_path LIKE 'hard-private/%'"
+ ).fetchone()
+ return int(row[0])
+ except Exception:
+ return None
+
+
+def doctor(
+ *,
+ brain_root: str | Path = "/Users/kublai/brain",
+ private_index_dir: str | Path = "/Users/kublai/.kublai/brain-index-private",
+ public_index_db: str | Path = "/Users/kublai/.brain-index/brain.db",
+ private_index_db: str | Path = "/Users/kublai/.kublai/brain-index-private/brain.db",
+) -> dict[str, Any]:
+ brain_root = Path(brain_root)
+ hard_private = brain_root / "hard-private"
+ private_index_dir = Path(private_index_dir)
+ public_index_db = Path(public_index_db)
+ private_index_db = Path(private_index_db)
+ launchctl = _run(["launchctl", "list"])
+ disabled = _run(["launchctl", "print-disabled", f"gui/{os.getuid()}"])
+ labels_present = {
+ label: label in launchctl.stdout
+ for label in EXPECTED_LABELS
+ }
+ filevault = _run(["fdesetup", "status"])
+ tm_hard = _run(["tmutil", "isexcluded", str(hard_private)])
+ tm_private_index = _run(["tmutil", "isexcluded", str(private_index_dir)])
+ tailscale = _run(["tailscale", "serve", "status"]) if Path("/usr/local/bin/tailscale").exists() or Path("/opt/homebrew/bin/tailscale").exists() else None
+ active_path_lint = _run([sys.executable, str(REPO_ROOT / "lints" / "no_active_graph_store.py")])
+ public_hard_rows = _index_hard_private_rows(public_index_db)
+ private_hard_rows = _index_hard_private_rows(private_index_db)
+
+ checks: dict[str, bool] = {
+ "brain_service_health": _brain_service_ok(brain_root),
+ "v4_gateway_health": _gateway_ok(),
+ "v4_full": _doctor_full_ok(brain_root),
+ "brain_mode_700": _mode(brain_root) == 0o700,
+ "hard_private_mode_700": _mode(hard_private) == 0o700,
+ "metadata_never_index": (hard_private / ".metadata_never_index").exists(),
+ "private_index_dir_exists": private_index_dir.exists(),
+ "public_index_no_hard_private": public_hard_rows == 0,
+ "private_index_has_hard_private": private_hard_rows is not None and private_hard_rows > 0,
+ "active_path_graph_lint": active_path_lint.returncode == 0,
+ "filevault_on": "FileVault is On" in filevault.stdout,
+ "tm_hard_private_excluded": "[Excluded]" in tm_hard.stdout or tm_hard.returncode == 0 and "excluded" in tm_hard.stdout.lower(),
+ "tm_private_index_excluded": "[Excluded]" in tm_private_index.stdout or tm_private_index.returncode == 0 and "excluded" in tm_private_index.stdout.lower(),
+ "not_icloud_path": "Library/Mobile Documents" not in str(brain_root.resolve()),
+ "funnel_not_serving_brain": not (tailscale and "brain-service" in tailscale.stdout.lower()),
+ "retired_graph_launchd_unloaded": RETIRED_GRAPH_LABEL not in launchctl.stdout,
+ "retired_graph_launchd_not_enabled": f'"{RETIRED_GRAPH_LABEL}" => enabled' not in disabled.stdout,
+ **{f"retired_graph_port_{port}_dark": _port_dark(port) for port in RETIRED_GRAPH_PORTS},
+ **{f"launchd_{label}": present for label, present in labels_present.items()},
+ }
+ return {"ok": all(checks.values()), "checks": checks}
+
+
+def main() -> int:
+ result = doctor()
+ print(json.dumps(result, sort_keys=True))
+ return 0 if result["ok"] else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/kublai/graph_adapter.py b/kublai/graph_adapter.py
new file mode 100644
index 00000000..83827922
--- /dev/null
+++ b/kublai/graph_adapter.py
@@ -0,0 +1,66 @@
+"""Narrow adapter for graphify JSON artifacts."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+class GraphAdapterError(ValueError):
+ """Raised when graphify output cannot be normalized."""
+
+
+@dataclass(frozen=True)
+class GraphNode:
+ id: str
+ label: str
+ community: int | None = None
+ data: dict[str, Any] | None = None
+
+
+@dataclass(frozen=True)
+class GraphEdge:
+ source: str
+ target: str
+ relation: str
+ weight: float = 1.0
+ data: dict[str, Any] | None = None
+
+
+@dataclass(frozen=True)
+class NormalizedGraph:
+ nodes: list[GraphNode]
+ edges: list[GraphEdge]
+
+
+class GraphifyAdapter:
+ GRAPHIFY_VERSION_RANGE = ("0.5.0", "0.7.0")
+ GRAPHIFY_OUT_DIR = Path("/Users/kublai/brain/graphify-out")
+
+ def normalize(self, graph_json_path: str | Path | None = None) -> NormalizedGraph:
+ path = Path(graph_json_path) if graph_json_path else self.GRAPHIFY_OUT_DIR / "graph.json"
+ data = json.loads(path.read_text(encoding="utf-8"))
+ if not isinstance(data, dict) or "nodes" not in data or "links" not in data:
+ raise GraphAdapterError("expected graphify JSON with nodes and links")
+ nodes = [
+ GraphNode(
+ id=str(node.get("id")),
+ label=str(node.get("label") or node.get("id")),
+ community=node.get("community"),
+ data=dict(node),
+ )
+ for node in data.get("nodes", [])
+ ]
+ edges = [
+ GraphEdge(
+ source=str(edge.get("source") or edge.get("_src")),
+ target=str(edge.get("target") or edge.get("_tgt")),
+ relation=str(edge.get("relation") or "related"),
+ weight=float(edge.get("weight") or edge.get("confidence_score") or 1.0),
+ data=dict(edge),
+ )
+ for edge in data.get("links", [])
+ ]
+ return NormalizedGraph(nodes=nodes, edges=edges)
diff --git a/kublai/humans.py b/kublai/humans.py
new file mode 100644
index 00000000..9f1acb96
--- /dev/null
+++ b/kublai/humans.py
@@ -0,0 +1,169 @@
+"""Phase 2.5 Step 8 — Human read helper backed by hard-private/human-contacts/.
+
+Intentionally read-mostly. The only mutation is consent updates, which
+overwrite the consent block in the human-contact frontmatter.
+"""
+from __future__ import annotations
+
+import os
+import re
+import tempfile
+from pathlib import Path
+from typing import Any
+
+
+_FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*(?:\n|$)", re.DOTALL)
+
+
+def _split(text: str) -> tuple[dict[str, Any], str, str]:
+ m = _FRONTMATTER_RE.match(text)
+ if not m:
+ return {}, "", text
+ raw = m.group(1)
+ body = text[m.end():]
+ fm: dict[str, Any] = {}
+ current_key = None
+ for line in raw.splitlines():
+ if not line.strip():
+ current_key = None
+ continue
+ if line.startswith(" ") and current_key is not None and line.lstrip().startswith("- "):
+ fm.setdefault(current_key, []).append(line.lstrip()[2:].strip())
+ continue
+ if ": " in line:
+ k, _, v = line.partition(": ")
+ k = k.strip()
+ v = v.strip()
+ if v == "":
+ fm[k] = []
+ current_key = k
+ elif v.startswith("[") and v.endswith("]"):
+ inner = v[1:-1].strip()
+ fm[k] = [x.strip() for x in inner.split(",")] if inner else []
+ current_key = None
+ else:
+ fm[k] = v.strip('"').strip("'")
+ current_key = None
+ return fm, raw, body
+
+
+def _atomic_write(path: Path, content: str) -> None:
+ fd, tmp = tempfile.mkstemp(prefix=f".{path.name}.", dir=str(path.parent))
+ try:
+ with os.fdopen(fd, "w", encoding="utf-8") as f:
+ f.write(content)
+ os.replace(tmp, path)
+ except Exception:
+ try:
+ os.unlink(tmp)
+ except FileNotFoundError:
+ pass
+ raise
+
+
+class HumansStore:
+ """Read/limited-write helper for hard-private/human-contacts/*.md."""
+
+ def __init__(self, wiki_root: str | Path):
+ self.wiki_root = Path(wiki_root).expanduser().resolve()
+ self.dir = self.wiki_root / "hard-private" / "human-contacts"
+
+ def _iter_pages(self):
+ if not self.dir.is_dir():
+ return
+ for p in sorted(self.dir.glob("*.md")):
+ yield p
+
+ def list(self, *, limit: int = 100, offset: int = 0,
+ search: str | None = None,
+ include_private: bool = False,
+ include_consents: bool = False,
+ include_identifiers: bool = False) -> list[dict[str, Any]]:
+ out: list[dict[str, Any]] = []
+ skipped = 0
+ for p in self._iter_pages():
+ try:
+ fm, _, _ = _split(p.read_text(encoding="utf-8"))
+ except Exception:
+ continue
+ display = fm.get("display_name") or fm.get("name") or p.stem
+ if search and search.lower() not in str(display).lower():
+ continue
+ if skipped < offset:
+ skipped += 1
+ continue
+ item = {
+ "human_id": fm.get("human_id") or p.stem,
+ "name": display,
+ "status": fm.get("status"),
+ "social_cluster": fm.get("social_cluster"),
+ "rel_path": str(p.relative_to(self.wiki_root)),
+ }
+ if include_private or include_consents:
+ item["consents"] = fm.get("consents") or []
+ if include_private or include_identifiers:
+ for key in (
+ "identifiers",
+ "signal_phone",
+ "phone",
+ "phone_number",
+ "timezone",
+ ):
+ if key in fm:
+ item[key] = fm.get(key)
+ out.append(item)
+ if len(out) >= limit:
+ break
+ return out
+
+ def get(self, *, human_id: str) -> dict[str, Any] | None:
+ for p in self._iter_pages():
+ try:
+ fm, _, _ = _split(p.read_text(encoding="utf-8"))
+ except Exception:
+ continue
+ if (fm.get("human_id") or p.stem) == human_id:
+ return {
+ "human_id": fm.get("human_id") or p.stem,
+ "name": fm.get("display_name") or fm.get("name") or p.stem,
+ "status": fm.get("status"),
+ "social_cluster": fm.get("social_cluster"),
+ "first_known": fm.get("first_known"),
+ "last_contact": fm.get("last_contact"),
+ "topics": fm.get("topics") or [],
+ "consents": fm.get("consents") or [],
+ "rel_path": str(p.relative_to(self.wiki_root)),
+ }
+ return None
+
+ def update_consent(self, *, human_id: str, category: str, granted: bool = True) -> bool:
+ for p in self._iter_pages():
+ try:
+ text = p.read_text(encoding="utf-8")
+ fm, raw, body = _split(text)
+ except Exception:
+ continue
+ if (fm.get("human_id") or p.stem) != human_id:
+ continue
+ consents = fm.get("consents") or []
+ existing = [c for c in consents if not c.startswith(f"{category}:")]
+ existing.append(f"{category}:{'granted' if granted else 'revoked'}")
+ new_lines = []
+ inserted = False
+ in_consents_block = False
+ for line in raw.splitlines():
+ if line.startswith("consents:"):
+ new_lines.append("consents: [" + ", ".join(existing) + "]")
+ inserted = True
+ in_consents_block = True
+ continue
+ if in_consents_block and line.startswith(" - "):
+ continue
+ in_consents_block = False
+ new_lines.append(line)
+ if not inserted:
+ new_lines.append("consents: [" + ", ".join(existing) + "]")
+ new_text = "---\n" + "\n".join(new_lines) + "\n---\n" + body
+ _atomic_write(p, new_text)
+ return True
+ return False
diff --git a/kublai/knowledge-client.js b/kublai/knowledge-client.js
new file mode 100644
index 00000000..2506d171
--- /dev/null
+++ b/kublai/knowledge-client.js
@@ -0,0 +1,11 @@
+'use strict';
+
+const { call } = require('./telemetry-client');
+
+module.exports = {
+ recordReflection: params => call('knowledge.record_reflection', params),
+ get: params => call('knowledge.get', params),
+ list: params => call('knowledge.list', params),
+ search: params => call('knowledge.search', params),
+ health: () => call('health', {}),
+};
diff --git a/kublai/knowledge.py b/kublai/knowledge.py
new file mode 100644
index 00000000..7e378460
--- /dev/null
+++ b/kublai/knowledge.py
@@ -0,0 +1,430 @@
+"""Typed markdown write helpers for the Kublai brain wiki."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+import tempfile
+import uuid
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from .sanitizer import classify_page
+
+ALLOWED_SUBTREES = (
+ "operations/reflections",
+ "operations/decisions",
+ "operations/rsi-cycles",
+ "operations/capabilities",
+ "operations/tasks",
+ "agents",
+)
+
+
+class KnowledgeError(Exception):
+ """Base exception for knowledge write failures."""
+
+
+class PathPolicyError(KnowledgeError):
+ """Raised when a write attempts to escape the allowed wiki subtrees."""
+
+
+def utc_date() -> str:
+ return datetime.now(timezone.utc).date().isoformat()
+
+
+def slugify(value: str) -> str:
+ value = value.lower()
+ value = re.sub(r"[^a-z0-9]+", "-", value).strip("-")
+ return value or "item"
+
+
+def body_hash(body: str) -> str:
+ return hashlib.sha256(body.encode("utf-8")).hexdigest()
+
+
+def normalize_body(body: str) -> str:
+ return "\n".join(line.rstrip() for line in body.rstrip().splitlines())
+
+
+class KnowledgeStore:
+ """Writes operational knowledge pages under a configured wiki root."""
+
+ def __init__(self, wiki_root: str | Path):
+ self.wiki_root = Path(wiki_root).expanduser().resolve()
+ self.wiki_root.mkdir(parents=True, exist_ok=True)
+ self.lock_path = self.wiki_root / ".knowledge-write.lock"
+
+ @contextmanager
+ def write_lock(self):
+ self.lock_path.parent.mkdir(parents=True, exist_ok=True)
+ with self.lock_path.open("a+") as lock_file:
+ try:
+ import fcntl
+
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
+ yield
+ finally:
+ try:
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
+ except Exception:
+ pass
+
+ def record_reflection(
+ self,
+ *,
+ agent: str,
+ body: str,
+ reflection_id: str | None = None,
+ status: str = "active",
+ tags: list[str] | None = None,
+ ) -> Path:
+ reflection_id = reflection_id or str(uuid.uuid4())
+ date = utc_date()
+ return self.write_page(
+ f"operations/reflections/{date}-{slugify(agent)}-{slugify(reflection_id)}.md",
+ {
+ "type": "reflection",
+ "reflection_id": reflection_id,
+ "agent": agent,
+ "status": status,
+ "created": date,
+ "updated": date,
+ "sources": 1,
+ "tags": tags or ["kublai", "reflection"],
+ },
+ body,
+ typed_field="reflection_id",
+ typed_id=reflection_id,
+ scan_existing=False,
+ )
+
+ def record_decision(
+ self,
+ *,
+ agent: str,
+ title: str,
+ body: str,
+ decision_id: str | None = None,
+ status: str = "active",
+ ) -> Path:
+ decision_id = decision_id or str(uuid.uuid4())
+ date = utc_date()
+ return self.write_page(
+ f"operations/decisions/{date}-{slugify(decision_id)}.md",
+ {
+ "type": "decision",
+ "decision_id": decision_id,
+ "agent": agent,
+ "title": title,
+ "status": status,
+ "created": date,
+ "updated": date,
+ "sources": 1,
+ "tags": ["kublai", "decision"],
+ },
+ body,
+ typed_field="decision_id",
+ typed_id=decision_id,
+ scan_existing=False,
+ )
+
+ def record_capability(
+ self,
+ *,
+ capability_id: str,
+ title: str,
+ body: str,
+ status: str = "active",
+ learned_by: str | None = None,
+ ) -> Path:
+ date = utc_date()
+ frontmatter: dict[str, Any] = {
+ "type": "capability",
+ "capability_id": capability_id,
+ "title": title,
+ "status": status,
+ "created": date,
+ "updated": date,
+ "sources": 1,
+ "tags": ["kublai", "capability"],
+ }
+ if learned_by:
+ frontmatter["learned_by"] = learned_by
+ return self.write_page(
+ f"operations/capabilities/{slugify(capability_id)}.md",
+ frontmatter,
+ body,
+ typed_field="capability_id",
+ typed_id=capability_id,
+ scan_existing=False,
+ )
+
+ def record_completed_task(
+ self,
+ *,
+ task_id: str,
+ agent: str,
+ delegated_by: str,
+ completed_at_ms: int,
+ deliverable: str,
+ results: dict[str, Any] | None = None,
+ historical: bool = False,
+ ) -> Path:
+ completed = datetime.fromtimestamp(completed_at_ms / 1000, timezone.utc)
+ date = completed.date().isoformat()
+ rel_path = f"operations/tasks/{completed:%Y/%m}/task-{slugify(task_id)}.md"
+ body = (
+ f"# Task {task_id}\n\n"
+ f"## Deliverable\n\n{deliverable}\n\n"
+ f"## Results\n\n```yaml\n{yaml.safe_dump(results or {}, sort_keys=True)}```\n"
+ )
+ return self.write_page(
+ rel_path,
+ {
+ "type": "task",
+ "task_id": task_id,
+ "status": "completed",
+ "agent": agent,
+ "delegated_by": delegated_by,
+ "created": date,
+ "updated": date,
+ "sources": 1,
+ "historical": historical,
+ "completion_body_hash": body_hash(body),
+ "tags": ["kublai", "task"],
+ },
+ body,
+ typed_field="task_id",
+ typed_id=task_id,
+ scan_existing=False,
+ )
+
+ def update_agent_profile(self, *, agent: str, body: str, status: str = "active") -> Path:
+ date = utc_date()
+ return self.write_page(
+ f"agents/{slugify(agent)}.md",
+ {
+ "type": "agent",
+ "agent_id": slugify(agent),
+ "status": status,
+ "created": date,
+ "updated": date,
+ "sources": 1,
+ "tags": ["kublai", "agent"],
+ },
+ body,
+ typed_field="agent_id",
+ typed_id=slugify(agent),
+ scan_existing=False,
+ )
+
+ def write_page(
+ self,
+ rel_path: str | Path,
+ frontmatter: dict[str, Any],
+ body: str,
+ *,
+ typed_field: str,
+ typed_id: str,
+ scan_existing: bool = True,
+ ) -> Path:
+ target = self.resolve_allowed(rel_path)
+ content = self.render(frontmatter, body)
+ if target.exists():
+ parsed = self.read_frontmatter(target)
+ if parsed.get(typed_field) == typed_id:
+ if target.read_text(encoding="utf-8") != content:
+ self.atomic_write(target, content)
+ return target
+ raise KnowledgeError(f"target path exists for a different {typed_field}: {target}")
+ if scan_existing:
+ existing = self.find_by_frontmatter(typed_field, typed_id)
+ if existing is not None:
+ return existing
+ target.parent.mkdir(parents=True, exist_ok=True)
+ self.atomic_write(target, content)
+ return target
+
+ def atomic_write(self, target: Path, content: str) -> None:
+ with self.write_lock():
+ fd, tmp_name = tempfile.mkstemp(prefix=f".{target.name}.", dir=str(target.parent))
+ try:
+ with os.fdopen(fd, "w", encoding="utf-8") as tmp:
+ tmp.write(content)
+ tmp.flush()
+ os.fsync(tmp.fileno())
+ os.replace(tmp_name, target)
+ finally:
+ if os.path.exists(tmp_name):
+ os.unlink(tmp_name)
+
+ def resolve_allowed(self, rel_path: str | Path) -> Path:
+ rel = Path(rel_path)
+ if rel.is_absolute() or ".." in rel.parts:
+ raise PathPolicyError(f"invalid wiki path: {rel_path}")
+ rel_text = rel.as_posix()
+ if not any(rel_text == prefix or rel_text.startswith(prefix + "/") for prefix in ALLOWED_SUBTREES):
+ raise PathPolicyError(f"path outside allowed operational subtrees: {rel_path}")
+ target = (self.wiki_root / rel).resolve()
+ if self.wiki_root not in target.parents:
+ raise PathPolicyError(f"path escapes wiki root: {rel_path}")
+ return target
+
+ def find_by_frontmatter(self, field: str, value: str) -> Path | None:
+ for path in self.wiki_root.rglob("*.md"):
+ if "_archive" in path.parts:
+ continue
+ parsed = self.read_frontmatter(path)
+ if parsed.get(field) == value:
+ return path
+ return None
+
+ @staticmethod
+ def read_frontmatter(path: Path) -> dict[str, Any]:
+ try:
+ text = path.read_text(encoding="utf-8")
+ except UnicodeDecodeError:
+ return {}
+ if not text.startswith("---\n"):
+ return {}
+ end = text.find("\n---\n", 4)
+ if end < 0:
+ return {}
+ data = yaml.safe_load(text[4:end]) or {}
+ return data if isinstance(data, dict) else {}
+
+ def list_proposals(
+ self,
+ *,
+ status: str | None = None,
+ since_ms: int | None = None,
+ limit: int = 50,
+ ) -> list[dict[str, Any]]:
+ """List wiki pages that represent proposals.
+
+ Proposals live as wiki pages with frontmatter ``type: proposal`` (or
+ with ``proposal`` in the tags list as a legacy fallback). Optional
+ ``status`` filters on frontmatter ``status``; ``since_ms`` filters by
+ page mtime so callers can poll for recent activity. Returns a flat
+ list of ``{rel_path, title, type, status, tags, updated, created,
+ mtime_ms}`` summaries — no body, intentionally light.
+
+ Replaces the LLM-emitted ``MATCH (p:Proposal) ...`` Cypher path used
+ by ``kurultai-proposal-decree-review/SKILL.md``.
+ """
+ results: list[dict[str, Any]] = []
+ for md in self.wiki_root.rglob("*.md"):
+ rel = md.relative_to(self.wiki_root).as_posix()
+ if rel.startswith("hard-private/"):
+ continue
+ try:
+ fm = self.read_frontmatter(md)
+ except Exception:
+ continue
+ tags = fm.get("tags") or []
+ if isinstance(tags, str):
+ tags = [tags]
+ is_proposal = (fm.get("type") == "proposal") or ("proposal" in tags)
+ if not is_proposal:
+ continue
+ if status is not None and fm.get("status") != status:
+ continue
+ try:
+ mtime_ms = int(md.stat().st_mtime * 1000)
+ except OSError:
+ continue
+ if since_ms is not None and mtime_ms < since_ms:
+ continue
+ results.append({
+ "rel_path": rel,
+ "title": fm.get("title") or md.stem,
+ "type": fm.get("type"),
+ "status": fm.get("status"),
+ "tags": tags,
+ "updated": fm.get("updated"),
+ "created": fm.get("created"),
+ "mtime_ms": mtime_ms,
+ })
+ results.sort(key=lambda r: r["mtime_ms"], reverse=True)
+ return results[:limit]
+
+ def list_by_tag(
+ self,
+ *,
+ tag: str,
+ privacy_scope: str | None = None,
+ limit: int = 50,
+ ) -> list[dict[str, Any]]:
+ results: list[dict[str, Any]] = []
+ for md in self.wiki_root.rglob("*.md"):
+ rel = md.relative_to(self.wiki_root).as_posix()
+ try:
+ fm = self.read_frontmatter(md)
+ except Exception:
+ continue
+ tags = fm.get("tags") or []
+ if isinstance(tags, str):
+ tags = [tags]
+ if privacy_scope is None:
+ privacy_scope = "public"
+ privacy_class = classify_page(rel, fm)
+ if privacy_scope == "public" and privacy_class != "public":
+ continue
+ if privacy_scope == "hard-private" and privacy_class != "hard-private":
+ continue
+ if tag in tags:
+ results.append({
+ "rel_path": rel,
+ "title": fm.get("title") or md.stem,
+ "type": fm.get("type"),
+ "status": fm.get("status"),
+ "tags": tags,
+ })
+ if len(results) >= limit:
+ break
+ return results
+
+ @staticmethod
+ def render(frontmatter: dict[str, Any], body: str) -> str:
+ yaml_text = yaml.safe_dump(frontmatter, sort_keys=False, allow_unicode=False).strip()
+ return f"---\n{yaml_text}\n---\n\n{normalize_body(body)}\n"
+
+import re as _re_kt
+
+_KT_FRONTMATTER_RE = _re_kt.compile(r"^---\s*\n(.*?)\n---\s*(?:\n|$)", _re_kt.DOTALL)
+
+
+def _read_frontmatter(text: str) -> dict[str, Any]:
+ m = _KT_FRONTMATTER_RE.match(text)
+ if not m:
+ return {}
+ fm: dict[str, Any] = {}
+ current_key = None
+ for line in m.group(1).splitlines():
+ if not line.strip():
+ current_key = None
+ continue
+ if line.startswith(" ") and current_key is not None and line.lstrip().startswith("- "):
+ fm.setdefault(current_key, []).append(line.lstrip()[2:].strip())
+ continue
+ if ": " in line:
+ k, _, v = line.partition(": ")
+ k = k.strip()
+ v = v.strip()
+ if v == "":
+ fm[k] = []
+ current_key = k
+ elif v.startswith("[") and v.endswith("]"):
+ inner = v[1:-1].strip()
+ fm[k] = [x.strip() for x in inner.split(",")] if inner else []
+ current_key = None
+ else:
+ fm[k] = v.strip("\"\'")
+ current_key = None
+ return fm
diff --git a/kublai/llm.py b/kublai/llm.py
new file mode 100644
index 00000000..11a90192
--- /dev/null
+++ b/kublai/llm.py
@@ -0,0 +1,199 @@
+"""LLM boundary for Kublai v4.
+
+This module centralizes model defaults, retries, structured-output validation,
+and privacy checks. It does not silently fall through to an external provider:
+callers must either configure a provider later or supply a transport in tests.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import inspect
+import json
+import os
+from typing import Any, Awaitable, Callable
+
+from .context_bundle import ContextBundle
+from .sanitizer import PrivacyBoundaryError, Sanitizer, SanitizerContext
+
+
+class LlmError(Exception):
+ """Base LLM exception."""
+
+
+class RateLimitError(LlmError):
+ pass
+
+
+class RefusedError(LlmError):
+ pass
+
+
+class MalformedError(LlmError):
+ pass
+
+
+class LlmTimeoutError(LlmError):
+ pass
+
+
+class ContextOverflowError(LlmError):
+ pass
+
+
+class MissingProviderError(LlmError):
+ pass
+
+
+MODEL_DEFAULTS = {
+ "ask": "claude-opus-4-7",
+ "ingest": "claude-opus-4-7",
+ "research_lens": "claude-opus-4-7",
+ "research_synthesis": "claude-opus-4-7",
+ "process_inbox": "claude-opus-4-7",
+ "connect": "claude-opus-4-7",
+ "brief": "claude-opus-4-7",
+ "write": "claude-opus-4-7",
+}
+
+Transport = Callable[..., str | Awaitable[str]]
+
+
+def model_for(command: str) -> str:
+ return MODEL_DEFAULTS.get(command, MODEL_DEFAULTS["ask"])
+
+
+def ensure_external_safe(bundle: ContextBundle, *, destination: str) -> None:
+ if destination != "external":
+ return
+ if bundle.privacy_class != "public":
+ raise PrivacyBoundaryError(
+ f"refusing external LLM call with {bundle.privacy_class} context"
+ )
+
+
+async def llm_call(
+ *,
+ command: str,
+ system: str = "",
+ user: str | None = None,
+ bundle: ContextBundle | None = None,
+ destination: str = "external",
+ max_tokens: int = 4096,
+ timeout_seconds: int = 120,
+ transport: Transport | None = None,
+) -> str:
+ sanitizer = Sanitizer()
+ if bundle is not None:
+ ensure_external_safe(bundle, destination=destination)
+ user = bundle.render()
+ if user is None:
+ raise MalformedError("llm_call requires user text or ContextBundle")
+ user, findings = sanitizer.scrub(
+ user,
+ target_class="public" if destination == "external" else "private",
+ context=SanitizerContext.LLM_PROMPT,
+ )
+ if findings and destination == "external":
+ raise PrivacyBoundaryError(f"refusing external LLM call after sanitizer findings: {findings}")
+
+ request = {
+ "command": command,
+ "model": model_for(command),
+ "system": system,
+ "user": user,
+ "max_tokens": max_tokens,
+ }
+ _append_request_log(request)
+
+ if transport is None:
+ mock_response = os.getenv("KUBLAI_LLM_MOCK_RESPONSE")
+ if mock_response is not None:
+ return mock_response
+ raise MissingProviderError("no LLM transport configured")
+
+ try:
+ result = transport(**request)
+ if inspect.isawaitable(result):
+ return await asyncio.wait_for(result, timeout=timeout_seconds)
+ return await asyncio.wait_for(asyncio.to_thread(lambda: result), timeout=timeout_seconds)
+ except asyncio.TimeoutError as exc:
+ raise LlmTimeoutError(str(exc)) from exc
+
+
+async def llm_call_structured(
+ *,
+ command: str,
+ system: str = "",
+ user: str | None = None,
+ bundle: ContextBundle | None = None,
+ schema: dict[str, Any],
+ max_retries: int = 2,
+ validator: Callable[[dict[str, Any]], None] | None = None,
+ transport: Transport | None = None,
+) -> dict[str, Any]:
+ prompt = user
+ last_error: Exception | None = None
+ for attempt in range(max_retries + 1):
+ raw = await llm_call(
+ command=command,
+ system=system,
+ user=prompt,
+ bundle=bundle if attempt == 0 else None,
+ transport=transport,
+ )
+ try:
+ parsed = json.loads(raw)
+ _validate_schema_shape(parsed, schema)
+ if validator:
+ validator(parsed)
+ return parsed
+ except Exception as exc:
+ last_error = exc
+ prompt = (
+ f"{user or (bundle.render() if bundle else '')}\n\n"
+ f"Previous response was invalid: {exc}. Return only valid JSON matching the schema."
+ )
+ raise MalformedError(str(last_error))
+
+
+async def llm_call_fresh_context(
+ *,
+ command: str,
+ system: str,
+ user: str,
+ max_tokens: int = 4096,
+ transport: Transport | None = None,
+) -> str:
+ return await llm_call(
+ command=command,
+ system=system,
+ user=user,
+ max_tokens=max_tokens,
+ transport=transport,
+ )
+
+
+def _validate_schema_shape(parsed: dict[str, Any], schema: dict[str, Any]) -> None:
+ required = schema.get("required") or []
+ missing = [key for key in required if key not in parsed]
+ if missing:
+ raise MalformedError(f"missing required keys: {missing}")
+
+
+def _append_request_log(request: dict[str, Any]) -> None:
+ log_path = os.getenv("KUBLAI_LLM_REQUEST_LOG")
+ if not log_path:
+ return
+ safe_request = {
+ "command": request.get("command"),
+ "model": request.get("model"),
+ "max_tokens": request.get("max_tokens"),
+ "system_sha256": hashlib.sha256(str(request.get("system") or "").encode("utf-8")).hexdigest(),
+ "system_chars": len(str(request.get("system") or "")),
+ "user_sha256": hashlib.sha256(str(request.get("user") or "").encode("utf-8")).hexdigest(),
+ "user_chars": len(str(request.get("user") or "")),
+ }
+ with open(log_path, "a", encoding="utf-8") as handle:
+ handle.write(json.dumps(safe_request, sort_keys=True) + "\n")
diff --git a/kublai/messages.py b/kublai/messages.py
new file mode 100644
index 00000000..9ba83839
--- /dev/null
+++ b/kublai/messages.py
@@ -0,0 +1,105 @@
+"""Phase 2.5 Step 8 — Messages helper backed by ~/.kublai/messages.jsonl."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+class MessagesStore:
+ def __init__(self, jsonl_path: str | Path):
+ self.jsonl_path = Path(jsonl_path).expanduser()
+
+ def append(self, record: dict[str, Any]) -> None:
+ self.jsonl_path.parent.mkdir(parents=True, exist_ok=True)
+ with self.jsonl_path.open("a", encoding="utf-8") as f:
+ f.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+ def list_recent(self, *, human_id: str | None = None,
+ limit: int = 100, offset: int = 0,
+ topic: str | None = None) -> list[dict[str, Any]]:
+ if not self.jsonl_path.exists():
+ return []
+ out: list[dict[str, Any]] = []
+ # Read in reverse for newest-first.
+ try:
+ lines = self.jsonl_path.read_text(encoding="utf-8").splitlines()
+ except Exception:
+ return []
+ skipped = 0
+ for line in reversed(lines):
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ rec = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if human_id and rec.get("humanId") != human_id and rec.get("human_id") != human_id:
+ continue
+ if topic and topic not in (rec.get("topics") or []):
+ continue
+ if skipped < offset:
+ skipped += 1
+ continue
+ out.append(rec)
+ if len(out) >= limit:
+ break
+ return out
+
+ def search(self, query: str, *, human_id: str | None = None,
+ limit: int = 50, offset: int = 0,
+ scope: str | None = None) -> list[dict[str, Any]]:
+ """Phase 5 — fulltext-equivalent substring search across messages.jsonl.
+
+ Replaces server.js Neo4j ``message_text_search`` fulltext index. Splits
+ ``query`` into whitespace-separated terms and requires every term to
+ appear (case-insensitive) in any of the searched fields:
+ ``content_scrubbed``, ``contentScrubbed``, ``summary``, ``topics``.
+
+ ``scope`` (optional) filters by an explicit ``scope`` field on records,
+ if present (forward-compat hook; ignored when records lack the field).
+ """
+ if not self.jsonl_path.exists():
+ return []
+ if not query or not query.strip():
+ return []
+ terms = [t.lower() for t in query.split() if t.strip()]
+ if not terms:
+ return []
+ try:
+ lines = self.jsonl_path.read_text(encoding="utf-8").splitlines()
+ except Exception:
+ return []
+ out: list[dict[str, Any]] = []
+ skipped = 0
+ for line in reversed(lines):
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ rec = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if human_id and rec.get("humanId") != human_id and rec.get("human_id") != human_id:
+ continue
+ if scope is not None and rec.get("scope") != scope:
+ continue
+ haystack_parts: list[str] = []
+ for key in ("content_scrubbed", "contentScrubbed", "summary"):
+ v = rec.get(key)
+ if isinstance(v, str):
+ haystack_parts.append(v)
+ topics = rec.get("topics")
+ if isinstance(topics, list):
+ haystack_parts.append(" ".join(str(t) for t in topics))
+ haystack = " ".join(haystack_parts).lower()
+ if not all(term in haystack for term in terms):
+ continue
+ if skipped < offset:
+ skipped += 1
+ continue
+ out.append(rec)
+ if len(out) >= limit:
+ break
+ return out
diff --git a/kublai/monitoring.py b/kublai/monitoring.py
new file mode 100644
index 00000000..683e7912
--- /dev/null
+++ b/kublai/monitoring.py
@@ -0,0 +1,36 @@
+"""Lightweight health helpers for Phase 1 fixtures."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from .telemetry import TelemetryStore
+
+
+def health_snapshot(telemetry: TelemetryStore, wiki_root: str | Path) -> dict:
+ wiki_root = Path(wiki_root)
+ with telemetry.connect() as conn:
+ tasks = conn.execute("SELECT status, count(*) AS n FROM in_flight_tasks GROUP BY status").fetchall()
+ agents = conn.execute("SELECT count(*) AS n FROM agent_state").fetchone()["n"]
+ materialization_errors = conn.execute(
+ """
+ SELECT count(*) AS n FROM in_flight_tasks
+ WHERE materialized_at IS NULL
+ AND completion_attempt_count > 0
+ """
+ ).fetchone()["n"]
+ materialization_alerts = conn.execute(
+ """
+ SELECT count(*) AS n FROM in_flight_tasks
+ WHERE materialized_at IS NULL
+ AND completion_attempt_count >= 3
+ """
+ ).fetchone()["n"]
+ return {
+ "ok": materialization_alerts == 0,
+ "wiki_root_exists": wiki_root.exists(),
+ "agent_count": agents,
+ "tasks": {row["status"]: row["n"] for row in tasks},
+ "materialization_errors": materialization_errors,
+ "materialization_alerts": materialization_alerts,
+ }
diff --git a/kublai/prompt_registry.py b/kublai/prompt_registry.py
new file mode 100644
index 00000000..39398ec0
--- /dev/null
+++ b/kublai/prompt_registry.py
@@ -0,0 +1,37 @@
+"""Prompt template loading and hashing for Kublai LLM calls."""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from pathlib import Path
+
+
+class PromptRegistryError(FileNotFoundError):
+ """Raised when a prompt template cannot be resolved."""
+
+
+@dataclass(frozen=True)
+class PromptTemplate:
+ name: str
+ path: Path
+ text: str
+
+ @property
+ def hash(self) -> str:
+ return hashlib.sha256(self.text.encode("utf-8")).hexdigest()
+
+
+class PromptRegistry:
+ def __init__(self, prompt_root: str | Path):
+ self.prompt_root = Path(prompt_root).expanduser()
+
+ def load(self, name: str) -> PromptTemplate:
+ safe_name = name.strip().replace("..", "")
+ path = self.prompt_root / f"{safe_name}.md"
+ if not path.exists():
+ raise PromptRegistryError(str(path))
+ return PromptTemplate(name=safe_name, path=path, text=path.read_text(encoding="utf-8"))
+
+ def hash(self, name: str) -> str:
+ return self.load(name).hash
diff --git a/kublai/rsi.py b/kublai/rsi.py
new file mode 100644
index 00000000..2350b153
--- /dev/null
+++ b/kublai/rsi.py
@@ -0,0 +1,63 @@
+"""RSI tracking helpers backed by typed wiki pages."""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+
+from .knowledge import KnowledgeStore, slugify, utc_date
+
+
+@dataclass(frozen=True)
+class RsiCycle:
+ rsi_id: str
+ hypothesis: str
+ intervention: str
+ outcome: str
+ path: Path
+
+
+class RsiStore:
+ """Records recursive-improvement cycles as operational wiki pages."""
+
+ def __init__(self, knowledge: KnowledgeStore):
+ self.knowledge = knowledge
+
+ def record_cycle(
+ self,
+ *,
+ hypothesis: str,
+ intervention: str,
+ outcome: str = "pending",
+ agent: str = "kublai",
+ rsi_id: str | None = None,
+ cycle_number: int | None = None,
+ ) -> RsiCycle:
+ rsi_id = rsi_id or str(uuid.uuid4())
+ cycle = cycle_number or 1
+ date = utc_date()
+ body = (
+ f"# RSI Cycle {cycle}\n\n"
+ f"## Hypothesis\n\n{hypothesis}\n\n"
+ f"## Intervention\n\n{intervention}\n\n"
+ f"## Outcome\n\n{outcome}\n"
+ )
+ path = self.knowledge.write_page(
+ f"operations/rsi-cycles/cycle-{cycle:03d}-{slugify(rsi_id)}.md",
+ {
+ "type": "rsi-cycle",
+ "rsi_id": rsi_id,
+ "agent": agent,
+ "hypothesis": hypothesis,
+ "status": "active" if outcome == "pending" else "completed",
+ "created": date,
+ "updated": date,
+ "sources": 1,
+ "tags": ["kublai", "rsi"],
+ },
+ body,
+ typed_field="rsi_id",
+ typed_id=rsi_id,
+ )
+ return RsiCycle(rsi_id, hypothesis, intervention, outcome, path)
diff --git a/kublai/sanitizer.py b/kublai/sanitizer.py
new file mode 100644
index 00000000..9124ae27
--- /dev/null
+++ b/kublai/sanitizer.py
@@ -0,0 +1,106 @@
+"""Privacy classifier and early sanitizer for Kublai brain v4.
+
+This Phase -1 stub is intentionally conservative. Phase 4 expands publishing
+rules, but the public indexer, LLM boundary, and MCP/gateway surfaces can depend
+on this single classifier now.
+"""
+
+from __future__ import annotations
+
+import re
+from enum import Enum
+from typing import Any
+
+from .context_bundle import ContextBundle, PrivacyClass, Source
+
+HARD_PRIVATE_FOLDERS = ("hard-private/",)
+HARD_PRIVATE_TYPES = {"human-contact"}
+HARD_PRIVATE_TAGS = {"pii", "tax", "financial", "legal", "medical"}
+PRIVATE_PATH_RE = re.compile(r"\bhard-private/[^\s)\]}>\"']+")
+CANARY_RE = re.compile(r"\bKUBLAI_HARD_PRIVATE_CANARY_[A-Z0-9_:-]*\b")
+
+
+class SanitizerContext(Enum):
+ PUBLISH = "publish"
+ QUERY_RESPONSE = "query_response"
+ LLM_PROMPT = "llm_prompt"
+ MCP_RESPONSE = "mcp_response"
+ GATEWAY_RESPONSE = "gateway_response"
+
+
+class SanitizerError(ValueError):
+ """Raised when content cannot be safely scrubbed for the requested target."""
+
+
+class PrivacyBoundaryError(SanitizerError):
+ """Raised when private context would cross an external boundary."""
+
+
+class Sanitizer:
+ HARD_PRIVATE_FOLDERS = HARD_PRIVATE_FOLDERS
+ HARD_PRIVATE_TYPES = HARD_PRIVATE_TYPES
+ HARD_PRIVATE_TAGS = HARD_PRIVATE_TAGS
+
+ def classify(self, rel_path: str, frontmatter: dict[str, Any] | None = None) -> PrivacyClass:
+ rel_path = rel_path.strip()
+ if any(rel_path.startswith(prefix) for prefix in self.HARD_PRIVATE_FOLDERS):
+ return "hard-private"
+ if frontmatter:
+ if frontmatter.get("type") in self.HARD_PRIVATE_TYPES:
+ return "hard-private"
+ tags = frontmatter.get("tags") or []
+ if isinstance(tags, str):
+ tags = [tags]
+ if {str(tag) for tag in tags} & self.HARD_PRIVATE_TAGS:
+ return "hard-private"
+ if frontmatter.get("publish") is True or frontmatter.get("public_stub") is True:
+ return "public"
+ return "private"
+
+ def scrub(
+ self,
+ content: str,
+ *,
+ target_class: PrivacyClass,
+ context: SanitizerContext,
+ ) -> tuple[str, list[str]]:
+ findings: list[str] = []
+ scrubbed = content
+ if CANARY_RE.search(scrubbed):
+ findings.append("hard_private_canary")
+ scrubbed = CANARY_RE.sub("[REDACTED:HARD_PRIVATE_CANARY]", scrubbed)
+ if PRIVATE_PATH_RE.search(scrubbed):
+ findings.append("hard_private_path")
+ if target_class == "public" or context in {
+ SanitizerContext.PUBLISH,
+ SanitizerContext.GATEWAY_RESPONSE,
+ SanitizerContext.LLM_PROMPT,
+ }:
+ scrubbed = PRIVATE_PATH_RE.sub("[REDACTED:HARD_PRIVATE_PATH]", scrubbed)
+ return scrubbed, findings
+
+ def build_bundle(self, *sources: Source | dict[str, Any], purpose: str = "", instructions: str = "") -> ContextBundle:
+ normalized: list[Source] = []
+ for source in sources:
+ if isinstance(source, Source):
+ normalized.append(source)
+ continue
+ rel_path = str(source.get("rel_path") or "")
+ frontmatter = dict(source.get("frontmatter") or {})
+ normalized.append(
+ Source(
+ rel_path=rel_path,
+ privacy_class=self.classify(rel_path, frontmatter),
+ title=source.get("title"),
+ frontmatter=frontmatter,
+ content=str(source.get("content") or ""),
+ )
+ )
+ return ContextBundle(normalized, purpose=purpose, instructions=instructions)
+
+
+DEFAULT_SANITIZER = Sanitizer()
+
+
+def classify_page(rel_path: str, frontmatter: dict[str, Any] | None = None) -> PrivacyClass:
+ return DEFAULT_SANITIZER.classify(rel_path, frontmatter)
diff --git a/kublai/schema_registry.py b/kublai/schema_registry.py
new file mode 100644
index 00000000..b1f78eaf
--- /dev/null
+++ b/kublai/schema_registry.py
@@ -0,0 +1,46 @@
+"""Flat frontmatter schema registry for the brain wiki."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+class SchemaError(ValueError):
+ """Raised when frontmatter does not satisfy a registered schema."""
+
+
+@dataclass(frozen=True)
+class SchemaRegistry:
+ root: Path
+
+ def __init__(self, root: str | Path):
+ object.__setattr__(self, "root", Path(root).expanduser())
+
+ def schema_path(self, page_type: str, schema_version: int = 1) -> Path:
+ return self.root / page_type / f"v{schema_version}.schema.json"
+
+ def load(self, page_type: str, schema_version: int = 1) -> dict[str, Any]:
+ path = self.schema_path(page_type, schema_version)
+ if not path.exists():
+ path = self.schema_path("default", 1)
+ return json.loads(path.read_text(encoding="utf-8"))
+
+ def validate_frontmatter(self, page_type: str, frontmatter: dict[str, Any], schema_version: int = 1) -> None:
+ schema = self.load(page_type, schema_version)
+ required = schema.get("required") or []
+ missing = [key for key in required if key not in frontmatter]
+ if missing:
+ raise SchemaError(f"{page_type} missing required frontmatter keys: {missing}")
+ for key in schema.get("flat_only", []):
+ value = frontmatter.get(key)
+ if isinstance(value, dict):
+ raise SchemaError(f"{page_type}.{key} must be flat")
+
+ def publishable_fields(self, page_type: str, schema_version: int = 1) -> list[str]:
+ return list(self.load(page_type, schema_version).get("publishable_fields") or [])
+
+ def edge_sidecar_fields(self, page_type: str, schema_version: int = 1) -> list[str]:
+ return list(self.load(page_type, schema_version).get("edge_sidecar_fields") or [])
diff --git a/kublai/telemetry-client.js b/kublai/telemetry-client.js
new file mode 100644
index 00000000..f44c14b6
--- /dev/null
+++ b/kublai/telemetry-client.js
@@ -0,0 +1,52 @@
+'use strict';
+
+const net = require('net');
+
+function call(method, params = {}, options = {}) {
+ const socketPath = options.socketPath || process.env.BRAIN_SERVICE_SOCKET || '/tmp/brain-service.sock';
+ const timeoutMs = options.timeoutMs || 5000;
+
+ return new Promise((resolve, reject) => {
+ const client = net.createConnection(socketPath);
+ let buffer = '';
+ const timer = setTimeout(() => {
+ client.destroy();
+ reject(new Error(`brain-service RPC timed out after ${timeoutMs}ms`));
+ }, timeoutMs);
+
+ client.on('connect', () => {
+ client.write(`${JSON.stringify({ method, params })}\n`);
+ });
+
+ client.on('data', chunk => {
+ buffer += chunk.toString('utf8');
+ if (buffer.includes('\n')) {
+ clearTimeout(timer);
+ client.end();
+ const line = buffer.split('\n')[0];
+ const response = JSON.parse(line);
+ if (!response.ok) {
+ const error = new Error(response.message || response.error || 'brain-service RPC failed');
+ error.code = response.error;
+ reject(error);
+ return;
+ }
+ resolve(response.result);
+ }
+ });
+
+ client.on('error', error => {
+ clearTimeout(timer);
+ reject(error);
+ });
+ });
+}
+
+module.exports = {
+ call,
+ createTask: params => call('telemetry.create_task', params),
+ claimTask: params => call('telemetry.claim_task', params),
+ renewClaim: params => call('telemetry.renew_claim', params),
+ completeTask: params => call('telemetry.complete_task', params),
+ heartbeat: params => call('telemetry.heartbeat', params),
+};
diff --git a/kublai/telemetry.py b/kublai/telemetry.py
new file mode 100644
index 00000000..4a59fa83
--- /dev/null
+++ b/kublai/telemetry.py
@@ -0,0 +1,2853 @@
+"""SQLite telemetry store for Kublai task lifecycle and hot-path state.
+
+Telemetry is intentionally separate from the markdown wiki. It owns fast,
+atomic state: task claims, leases, heartbeats, rate counters, and notifications.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import time
+import uuid
+from contextlib import contextmanager
+from dataclasses import asdict, dataclass, is_dataclass
+from pathlib import Path
+from typing import Any, Iterator
+
+from .audit import redact_secrets
+from .tracing import new_trace_id
+
+DEFAULT_LEASE_TTL_MS = 1_103_329
+
+
+class TelemetryError(Exception):
+ """Base exception for telemetry failures."""
+
+
+class NoPendingTaskError(TelemetryError):
+ """Raised when no claimable task exists."""
+
+
+class StaleClaimError(TelemetryError):
+ """Raised when a worker tries to act with an expired or superseded claim."""
+
+
+def utc_ms() -> int:
+ return int(time.time() * 1000)
+
+
+def _percentile(values: list[int] | list[float], pct: float) -> float:
+ """Nearest-rank percentile on a presorted ascending list (no numpy dep)."""
+ if not values:
+ return 0.0
+ if pct <= 0:
+ return float(values[0])
+ if pct >= 1:
+ return float(values[-1])
+ rank = int(round(pct * (len(values) - 1)))
+ return float(values[rank])
+
+
+@dataclass(frozen=True)
+class ClaimedTask:
+ id: str
+ claim_token: str
+ lease_version: int
+ expires_at: int
+ payload: dict[str, Any]
+
+
+class TelemetryStore:
+ """Small SQLite store with fenced task claims."""
+
+ def __init__(self, db_path: str | Path):
+ self.db_path = Path(db_path).expanduser()
+ self.db_path.parent.mkdir(parents=True, exist_ok=True)
+ self._init_db()
+
+ @contextmanager
+ def connect(self) -> Iterator[sqlite3.Connection]:
+ conn = sqlite3.connect(self.db_path, timeout=5.0, isolation_level=None)
+ conn.row_factory = sqlite3.Row
+ conn.execute("PRAGMA busy_timeout = 5000")
+ conn.execute("PRAGMA foreign_keys = ON")
+ try:
+ yield conn
+ finally:
+ conn.close()
+
+ def _init_db(self) -> None:
+ with self.connect() as conn:
+ conn.executescript(
+ """
+ PRAGMA journal_mode = WAL;
+ PRAGMA synchronous = NORMAL;
+ PRAGMA user_version = 1;
+
+ CREATE TABLE IF NOT EXISTS schema_migrations (
+ version INTEGER PRIMARY KEY,
+ applied_at INTEGER NOT NULL,
+ description TEXT
+ ) STRICT;
+
+ CREATE TABLE IF NOT EXISTS agent_state (
+ agent TEXT PRIMARY KEY,
+ last_heartbeat INTEGER NOT NULL,
+ status TEXT NOT NULL,
+ meta TEXT CHECK(meta IS NULL OR json_valid(meta))
+ ) STRICT;
+
+ CREATE TABLE IF NOT EXISTS rate_limits (
+ agent TEXT NOT NULL,
+ operation TEXT NOT NULL,
+ window_ms INTEGER NOT NULL,
+ bucket_start_ms INTEGER NOT NULL,
+ count INTEGER NOT NULL DEFAULT 0,
+ last_hit INTEGER NOT NULL,
+ PRIMARY KEY (agent, operation, window_ms, bucket_start_ms)
+ ) STRICT, WITHOUT ROWID;
+
+ CREATE TABLE IF NOT EXISTS in_flight_tasks (
+ id TEXT PRIMARY KEY,
+ type TEXT NOT NULL,
+ description TEXT NOT NULL,
+ delegated_by TEXT NOT NULL,
+ assigned_to TEXT,
+ priority INTEGER NOT NULL,
+ status TEXT NOT NULL CHECK(status IN ('pending','in_progress','completed','failed','cancelled')),
+ claimed_by TEXT,
+ claimed_at INTEGER,
+ active_claim_token TEXT,
+ active_lease_version INTEGER,
+ completed_at INTEGER,
+ failed_at INTEGER,
+ results_json TEXT CHECK(results_json IS NULL OR json_valid(results_json)),
+ completion_summary TEXT,
+ completion_body_hash TEXT,
+ target_wiki_path TEXT,
+ wiki_path TEXT,
+ materialized_at INTEGER,
+ completion_attempt_count INTEGER NOT NULL DEFAULT 0,
+ last_error TEXT,
+ created_at INTEGER NOT NULL,
+ updated_at INTEGER NOT NULL
+ ) STRICT;
+
+ CREATE TABLE IF NOT EXISTS claim_locks (
+ task_id TEXT PRIMARY KEY REFERENCES in_flight_tasks(id) ON DELETE CASCADE,
+ claimed_by TEXT NOT NULL,
+ claim_token TEXT NOT NULL,
+ lease_version INTEGER NOT NULL,
+ claimed_at INTEGER NOT NULL,
+ expires_at INTEGER NOT NULL
+ ) STRICT;
+
+ CREATE TABLE IF NOT EXISTS notifications (
+ id TEXT PRIMARY KEY,
+ agent TEXT NOT NULL,
+ kind TEXT NOT NULL,
+ body TEXT NOT NULL,
+ read_at INTEGER,
+ created_at INTEGER NOT NULL
+ ) STRICT;
+
+ CREATE TABLE IF NOT EXISTS traces (
+ trace_id TEXT PRIMARY KEY,
+ actor TEXT NOT NULL,
+ operation TEXT NOT NULL,
+ resource TEXT,
+ parent_span_id TEXT,
+ metadata_json TEXT CHECK(metadata_json IS NULL OR json_valid(metadata_json)),
+ started_at INTEGER NOT NULL,
+ created_at INTEGER NOT NULL
+ ) STRICT;
+
+ CREATE TABLE IF NOT EXISTS trace_events (
+ id TEXT PRIMARY KEY,
+ trace_id TEXT NOT NULL,
+ event_type TEXT NOT NULL,
+ actor TEXT,
+ span_id TEXT,
+ payload_json TEXT CHECK(payload_json IS NULL OR json_valid(payload_json)),
+ created_at INTEGER NOT NULL,
+ FOREIGN KEY(trace_id) REFERENCES traces(trace_id) ON DELETE CASCADE
+ ) STRICT;
+
+ CREATE TABLE IF NOT EXISTS audit_events (
+ id TEXT PRIMARY KEY,
+ trace_id TEXT,
+ actor TEXT NOT NULL,
+ action TEXT NOT NULL,
+ decision TEXT NOT NULL,
+ resource TEXT,
+ details_json TEXT CHECK(details_json IS NULL OR json_valid(details_json)),
+ created_at INTEGER NOT NULL
+ ) STRICT;
+
+ INSERT OR IGNORE INTO schema_migrations
+ VALUES (1, unixepoch() * 1000, 'initial telemetry schema');
+
+ -- Phase 3 step 1: task lifecycle audit tables (migration v4).
+ -- These tables back the append_task_event / list_task_outputs / failure_reports
+ -- RPCs and the new Phase 3 step 11 task CRUD methods. On fresh installs we
+ -- create them here so the test harness and Mac-mini cold starts see the
+ -- same schema. Idempotent on existing DBs.
+ CREATE TABLE IF NOT EXISTS task_events (
+ id TEXT PRIMARY KEY,
+ task_id TEXT NOT NULL,
+ event_type TEXT NOT NULL,
+ agent TEXT,
+ details_json TEXT NOT NULL CHECK(json_valid(details_json)),
+ occurred_at INTEGER NOT NULL,
+ created_at INTEGER NOT NULL
+ ) STRICT;
+ CREATE INDEX IF NOT EXISTS idx_task_events_task ON task_events(task_id, occurred_at);
+ CREATE INDEX IF NOT EXISTS idx_task_events_type ON task_events(event_type, occurred_at);
+
+ CREATE TABLE IF NOT EXISTS task_outputs (
+ id TEXT PRIMARY KEY,
+ task_id TEXT NOT NULL,
+ kind TEXT NOT NULL,
+ summary TEXT,
+ payload_json TEXT NOT NULL CHECK(json_valid(payload_json)),
+ artifact_paths_json TEXT NOT NULL CHECK(json_valid(artifact_paths_json)),
+ created_at INTEGER NOT NULL
+ ) STRICT;
+ CREATE INDEX IF NOT EXISTS idx_task_outputs_task ON task_outputs(task_id, created_at);
+
+ CREATE TABLE IF NOT EXISTS task_outcomes (
+ id TEXT PRIMARY KEY,
+ task_id TEXT NOT NULL,
+ status TEXT NOT NULL,
+ reason TEXT,
+ payload_json TEXT NOT NULL CHECK(json_valid(payload_json)),
+ occurred_at INTEGER NOT NULL,
+ created_at INTEGER NOT NULL
+ ) STRICT;
+ CREATE INDEX IF NOT EXISTS idx_task_outcomes_task ON task_outcomes(task_id, occurred_at);
+
+ CREATE TABLE IF NOT EXISTS failure_reports (
+ id TEXT PRIMARY KEY,
+ task_id TEXT,
+ agent TEXT,
+ error_class TEXT NOT NULL,
+ message TEXT NOT NULL,
+ stack TEXT,
+ recovery TEXT,
+ payload_json TEXT NOT NULL CHECK(json_valid(payload_json)),
+ occurred_at INTEGER NOT NULL,
+ created_at INTEGER NOT NULL
+ ) STRICT;
+ CREATE INDEX IF NOT EXISTS idx_failure_reports_task ON failure_reports(task_id, occurred_at);
+ CREATE INDEX IF NOT EXISTS idx_failure_reports_agent ON failure_reports(agent, occurred_at);
+
+ CREATE TABLE IF NOT EXISTS task_dependencies (
+ task_id TEXT NOT NULL,
+ depends_on_task_id TEXT NOT NULL,
+ created_at INTEGER NOT NULL,
+ PRIMARY KEY (task_id, depends_on_task_id),
+ FOREIGN KEY(task_id) REFERENCES in_flight_tasks(id) ON DELETE CASCADE,
+ FOREIGN KEY(depends_on_task_id) REFERENCES in_flight_tasks(id) ON DELETE CASCADE
+ ) STRICT;
+ CREATE INDEX IF NOT EXISTS idx_task_dependencies_depends_on
+ ON task_dependencies(depends_on_task_id);
+
+ INSERT OR IGNORE INTO schema_migrations
+ VALUES (4, unixepoch() * 1000, 'phase 3 step 1 task lifecycle audit tables');
+ """
+ )
+ self._ensure_column(conn, "in_flight_tasks", "trace_id", "TEXT")
+ self._ensure_column(conn, "in_flight_tasks", "reliability_state", "TEXT NOT NULL DEFAULT 'pending'")
+ self._ensure_column(conn, "in_flight_tasks", "retry_count", "INTEGER NOT NULL DEFAULT 0")
+ self._apply_migration_5_if_needed(conn)
+ self._repair_claim_locks_fk_if_needed(conn)
+
+ # Phase 3 step 11: extend in_flight_tasks with dashboard columns + widen status CHECK.
+ # SQLite STRICT tables forbid altering CHECK in place, so this runs the table-recreate
+ # procedure on existing v4 databases. Fresh installs hit this same path right after
+ # _ensure_column finishes (it's idempotent because the migration check looks at
+ # schema_migrations).
+ _MIGRATION_5_DESC = "phase 3 step 11 in_flight_tasks extended"
+ _MIGRATION_5_SQL = """
+ DROP INDEX IF EXISTS idx_inflight_tasks_assigned_status;
+ DROP INDEX IF EXISTS idx_inflight_tasks_status_priority;
+ DROP INDEX IF EXISTS idx_inflight_tasks_pipeline;
+ DROP INDEX IF EXISTS idx_inflight_tasks_parent;
+ DROP INDEX IF EXISTS idx_inflight_tasks_paused;
+ DROP INDEX IF EXISTS idx_in_flight_tasks_dashboard;
+
+ CREATE TEMP TABLE IF NOT EXISTS _claim_locks_v4_pre_phase_3 AS
+ SELECT task_id, claimed_by, claim_token, lease_version, claimed_at, expires_at
+ FROM claim_locks;
+ DROP TABLE claim_locks;
+
+ ALTER TABLE in_flight_tasks RENAME TO _in_flight_tasks_v4_pre_phase_3;
+
+ CREATE TABLE in_flight_tasks (
+ id TEXT PRIMARY KEY,
+ type TEXT NOT NULL,
+ description TEXT NOT NULL,
+ delegated_by TEXT NOT NULL,
+ assigned_to TEXT,
+ priority INTEGER NOT NULL,
+ status TEXT NOT NULL CHECK(status IN (
+ 'pending','in_progress','completed','failed','cancelled',
+ 'blocked','awaiting_approval','obsolete',
+ 'PENDING','WORKING','COMPLETED','FAILED','ORPHANED',
+ 'OBSOLETE','DONE','STALE','CANCELLED','IN_PROGRESS'
+ )),
+ claimed_by TEXT,
+ claimed_at INTEGER,
+ active_claim_token TEXT,
+ active_lease_version INTEGER,
+ completed_at INTEGER,
+ failed_at INTEGER,
+ results_json TEXT CHECK(results_json IS NULL OR json_valid(results_json)),
+ completion_summary TEXT,
+ completion_body_hash TEXT,
+ target_wiki_path TEXT,
+ wiki_path TEXT,
+ materialized_at INTEGER,
+ completion_attempt_count INTEGER NOT NULL DEFAULT 0,
+ last_error TEXT,
+ created_at INTEGER NOT NULL,
+ updated_at INTEGER NOT NULL,
+ trace_id TEXT,
+ reliability_state TEXT NOT NULL DEFAULT 'pending',
+ retry_count INTEGER NOT NULL DEFAULT 0,
+ title TEXT,
+ prompt TEXT,
+ domain TEXT,
+ source TEXT,
+ parent_task TEXT,
+ reflection_id TEXT,
+ pipeline_id TEXT,
+ sort_order INTEGER,
+ paused INTEGER NOT NULL DEFAULT 0,
+ paused_at INTEGER,
+ dispatch_phase TEXT,
+ max_retries INTEGER NOT NULL DEFAULT 3,
+ timeout_s INTEGER,
+ depth INTEGER NOT NULL DEFAULT 0,
+ requires_computer_use INTEGER NOT NULL DEFAULT 0,
+ skill_hint TEXT,
+ reassigned_from TEXT,
+ previous_status TEXT,
+ previous_agent TEXT,
+ original_prompt TEXT,
+ previous_prompt TEXT,
+ cancelled_at INTEGER,
+ started_at INTEGER,
+ claim_epoch INTEGER,
+ score REAL,
+ obsolete_reason TEXT,
+ obsolete_by TEXT,
+ obsolete_at INTEGER,
+ rewrite_reason TEXT,
+ rewrite_by TEXT,
+ rewritten_at INTEGER,
+ reassign_reason TEXT,
+ reassigned_by TEXT,
+ reassigned_at INTEGER,
+ optimized_prompt TEXT
+ ) STRICT;
+
+ INSERT INTO in_flight_tasks (
+ id, type, description, delegated_by, assigned_to, priority, status,
+ claimed_by, claimed_at, active_claim_token, active_lease_version,
+ completed_at, failed_at, results_json, completion_summary, completion_body_hash,
+ target_wiki_path, wiki_path, materialized_at, completion_attempt_count, last_error,
+ created_at, updated_at, trace_id, reliability_state, retry_count
+ )
+ SELECT
+ id, type, description, delegated_by, assigned_to, priority, status,
+ claimed_by, claimed_at, active_claim_token, active_lease_version,
+ completed_at, failed_at, results_json, completion_summary, completion_body_hash,
+ target_wiki_path, wiki_path, materialized_at, completion_attempt_count, last_error,
+ created_at, updated_at, trace_id, reliability_state, retry_count
+ FROM _in_flight_tasks_v4_pre_phase_3;
+
+ DROP TABLE _in_flight_tasks_v4_pre_phase_3;
+
+ CREATE INDEX IF NOT EXISTS idx_inflight_tasks_assigned_status
+ ON in_flight_tasks(assigned_to, status);
+ CREATE INDEX IF NOT EXISTS idx_inflight_tasks_status_priority
+ ON in_flight_tasks(status, priority, created_at);
+ CREATE INDEX IF NOT EXISTS idx_inflight_tasks_pipeline
+ ON in_flight_tasks(pipeline_id);
+ CREATE INDEX IF NOT EXISTS idx_inflight_tasks_parent
+ ON in_flight_tasks(parent_task);
+ CREATE INDEX IF NOT EXISTS idx_inflight_tasks_paused
+ ON in_flight_tasks(paused, status);
+ CREATE INDEX IF NOT EXISTS idx_in_flight_tasks_dashboard
+ ON in_flight_tasks(status, sort_order DESC, priority, created_at DESC);
+
+ CREATE TABLE claim_locks (
+ task_id TEXT PRIMARY KEY REFERENCES in_flight_tasks(id) ON DELETE CASCADE,
+ claimed_by TEXT NOT NULL,
+ claim_token TEXT NOT NULL,
+ lease_version INTEGER NOT NULL,
+ claimed_at INTEGER NOT NULL,
+ expires_at INTEGER NOT NULL
+ ) STRICT;
+ INSERT OR IGNORE INTO claim_locks (
+ task_id, claimed_by, claim_token, lease_version, claimed_at, expires_at
+ )
+ SELECT task_id, claimed_by, claim_token, lease_version, claimed_at, expires_at
+ FROM _claim_locks_v4_pre_phase_3
+ WHERE task_id IN (SELECT id FROM in_flight_tasks);
+ DROP TABLE _claim_locks_v4_pre_phase_3;
+ """
+
+ def _claim_locks_fk_target(self, conn: sqlite3.Connection) -> str | None:
+ row = conn.execute("PRAGMA foreign_key_list(claim_locks)").fetchone()
+ if row is None:
+ return None
+ return row["table"] if isinstance(row, sqlite3.Row) else row[2]
+
+ def _repair_claim_locks_fk_if_needed(self, conn: sqlite3.Connection) -> None:
+ if self._claim_locks_fk_target(conn) == "in_flight_tasks":
+ return
+ prev_fk = conn.execute("PRAGMA foreign_keys").fetchone()[0]
+ try:
+ conn.execute("PRAGMA foreign_keys = OFF")
+ conn.executescript(
+ """
+ BEGIN IMMEDIATE;
+ CREATE TEMP TABLE IF NOT EXISTS _claim_locks_fk_repair AS
+ SELECT task_id, claimed_by, claim_token, lease_version, claimed_at, expires_at
+ FROM claim_locks;
+ DROP TABLE claim_locks;
+ CREATE TABLE claim_locks (
+ task_id TEXT PRIMARY KEY REFERENCES in_flight_tasks(id) ON DELETE CASCADE,
+ claimed_by TEXT NOT NULL,
+ claim_token TEXT NOT NULL,
+ lease_version INTEGER NOT NULL,
+ claimed_at INTEGER NOT NULL,
+ expires_at INTEGER NOT NULL
+ ) STRICT;
+ INSERT OR IGNORE INTO claim_locks (
+ task_id, claimed_by, claim_token, lease_version, claimed_at, expires_at
+ )
+ SELECT task_id, claimed_by, claim_token, lease_version, claimed_at, expires_at
+ FROM _claim_locks_fk_repair
+ WHERE task_id IN (SELECT id FROM in_flight_tasks);
+ DROP TABLE _claim_locks_fk_repair;
+ COMMIT;
+ """
+ )
+ finally:
+ if prev_fk:
+ conn.execute("PRAGMA foreign_keys = ON")
+
+ def _apply_migration_5_if_needed(self, conn: sqlite3.Connection) -> None:
+ row = conn.execute(
+ "SELECT 1 FROM schema_migrations WHERE version = 5"
+ ).fetchone()
+ if row is not None:
+ return
+ # Detect whether the live in_flight_tasks already has the new columns. If
+ # someone applied migration-5 SQL directly (dashboard cutover path), we
+ # only need to record the migration row.
+ cols = {r["name"] for r in conn.execute("PRAGMA table_info(in_flight_tasks)").fetchall()}
+ already_extended = "title" in cols and "dispatch_phase" in cols and "paused" in cols
+ if already_extended:
+ conn.execute(
+ "INSERT OR IGNORE INTO schema_migrations(version, applied_at, description) VALUES (5, unixepoch() * 1000, ?)",
+ (self._MIGRATION_5_DESC,),
+ )
+ return
+ # SQLite rewrites FK references during ALTER RENAME; recreating the parent
+ # leaves claim_locks FK'd to the renamed table, which we then drop. The
+ # standard 12-step recreate procedure for FK-targeted tables requires
+ # foreign_keys = OFF for the duration. Restore after.
+ prev_fk = conn.execute("PRAGMA foreign_keys").fetchone()[0]
+ try:
+ conn.execute("PRAGMA foreign_keys = OFF")
+ conn.executescript("BEGIN IMMEDIATE;" + self._MIGRATION_5_SQL + "COMMIT;")
+ # Verify integrity (catches dangling FKs in claim_locks).
+ problems = list(conn.execute("PRAGMA foreign_key_check"))
+ if problems:
+ raise RuntimeError(f"foreign_key_check failed after migration v5: {problems}")
+ finally:
+ if prev_fk:
+ conn.execute("PRAGMA foreign_keys = ON")
+ conn.execute(
+ "INSERT OR IGNORE INTO schema_migrations(version, applied_at, description) VALUES (5, unixepoch() * 1000, ?)",
+ (self._MIGRATION_5_DESC,),
+ )
+
+ def create_task(
+ self,
+ task_id: str | None = None,
+ *,
+ type: str = "task",
+ description: str,
+ delegated_by: str,
+ assigned_to: str | None = None,
+ priority: int = 0,
+ results: dict[str, Any] | None = None,
+ trace_id: str | None = None,
+ ) -> str:
+ now = utc_ms()
+ task_id = task_id or f"task-{uuid.uuid4()}"
+ trace_id = trace_id or new_trace_id()
+ with self.connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO in_flight_tasks (
+ id, type, description, delegated_by, assigned_to, priority,
+ status, results_json, trace_id, reliability_state, retry_count, created_at, updated_at
+ ) VALUES (?, ?, ?, ?, ?, ?, 'pending', ?, ?, 'pending', 0, ?, ?)
+ """,
+ (
+ task_id,
+ type,
+ description,
+ delegated_by,
+ assigned_to,
+ priority,
+ json.dumps(results or {}, sort_keys=True),
+ trace_id,
+ now,
+ now,
+ ),
+ )
+ return task_id
+
+ def create_trace(self, trace: Any | None = None, **kwargs: Any) -> str:
+ data = self._trace_data(trace, kwargs)
+ trace_id = data.get("trace_id") or new_trace_id()
+ now = utc_ms()
+ started_at = int(data.get("started_at") or now)
+ with self.connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO traces(trace_id, actor, operation, resource, parent_span_id, metadata_json, started_at, created_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+ """,
+ (
+ trace_id,
+ data["actor"],
+ data["operation"],
+ data.get("resource"),
+ data.get("parent_span_id"),
+ json.dumps(data.get("metadata") or {}, sort_keys=True),
+ started_at,
+ now,
+ ),
+ )
+ return trace_id
+
+ def append_trace_event(
+ self,
+ trace_id: str,
+ event_type: str,
+ *,
+ actor: str | None = None,
+ span_id: str | None = None,
+ payload: dict[str, Any] | None = None,
+ now_ms: int | None = None,
+ ) -> str:
+ event_id = f"event-{uuid.uuid4().hex}"
+ with self.connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO trace_events(id, trace_id, event_type, actor, span_id, payload_json, created_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?)
+ """,
+ (event_id, trace_id, event_type, actor, span_id, json.dumps(redact_secrets(payload or {}), sort_keys=True), now_ms or utc_ms()),
+ )
+ return event_id
+
+ def get_trace(self, trace_id: str) -> dict[str, Any] | None:
+ with self.connect() as conn:
+ trace = conn.execute("SELECT * FROM traces WHERE trace_id = ?", (trace_id,)).fetchone()
+ if trace is None:
+ return None
+ events = conn.execute(
+ "SELECT * FROM trace_events WHERE trace_id = ? ORDER BY created_at ASC",
+ (trace_id,),
+ ).fetchall()
+ return {"trace": self._decode_json_fields(dict(trace)), "events": [self._decode_json_fields(dict(row)) for row in events]}
+
+ def list_traces(self, *, actor: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
+ clause = "WHERE actor = ?" if actor else ""
+ params: tuple[Any, ...] = (actor, limit) if actor else (limit,)
+ with self.connect() as conn:
+ rows = conn.execute(f"SELECT * FROM traces {clause} ORDER BY started_at DESC LIMIT ?", params).fetchall()
+ return [self._decode_json_fields(dict(row)) for row in rows]
+
+ def record_audit_event(
+ self,
+ *,
+ actor: str,
+ action: str,
+ decision: str,
+ details: dict[str, Any] | None = None,
+ trace_id: str | None = None,
+ resource: str | None = None,
+ now_ms: int | None = None,
+ ) -> str:
+ event_id = f"audit-{uuid.uuid4().hex}"
+ with self.connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO audit_events(id, trace_id, actor, action, decision, resource, details_json, created_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+ """,
+ (event_id, trace_id, actor, action, decision, resource, json.dumps(redact_secrets(details or {}), sort_keys=True), now_ms or utc_ms()),
+ )
+ return event_id
+
+ def list_audit_events(self, *, actor: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
+ clause = "WHERE actor = ?" if actor else ""
+ params: tuple[Any, ...] = (actor, limit) if actor else (limit,)
+ with self.connect() as conn:
+ rows = conn.execute(f"SELECT * FROM audit_events {clause} ORDER BY created_at DESC LIMIT ?", params).fetchall()
+ return [self._decode_json_fields(dict(row)) for row in rows]
+
+ def claim_task(
+ self,
+ agent: str,
+ *,
+ lease_ttl_ms: int = DEFAULT_LEASE_TTL_MS,
+ now_ms: int | None = None,
+ ) -> ClaimedTask:
+ now = now_ms or utc_ms()
+ expires_at = now + lease_ttl_ms
+ claim_token = str(uuid.uuid4())
+ with self.connect() as conn:
+ conn.execute("BEGIN IMMEDIATE")
+ # Real-time callers get a generous grace period before automatic
+ # recovery to avoid reclaiming long-running but still-live workers
+ # under local SQLite/thread scheduling stalls. Tests and recovery
+ # code can pass now_ms or call sweep_expired_claims for exact expiry.
+ sweep_now = now if now_ms is not None else now - (lease_ttl_ms * 10)
+ self._sweep_expired_claims(conn, sweep_now)
+ row = conn.execute(
+ """
+ SELECT id FROM in_flight_tasks
+ WHERE status = 'pending'
+ AND (assigned_to IS NULL OR assigned_to = ?)
+ ORDER BY priority DESC, created_at ASC
+ LIMIT 1
+ """,
+ (agent,),
+ ).fetchone()
+ if row is None:
+ conn.execute("COMMIT")
+ raise NoPendingTaskError("no pending task available")
+ task_id = row["id"]
+ inserted = conn.execute(
+ """
+ INSERT OR IGNORE INTO claim_locks (
+ task_id, claimed_by, claim_token, lease_version, claimed_at, expires_at
+ ) VALUES (?, ?, ?, 1, ?, ?)
+ """,
+ (task_id, agent, claim_token, now, expires_at),
+ ).rowcount
+ if inserted != 1:
+ conn.execute("COMMIT")
+ raise NoPendingTaskError("claim lost to another worker")
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET status = 'in_progress',
+ claimed_by = ?,
+ claimed_at = ?,
+ active_claim_token = ?,
+ active_lease_version = 1,
+ updated_at = ?
+ WHERE id = ?
+ """,
+ (agent, now, claim_token, now, task_id),
+ )
+ payload_row = conn.execute("SELECT * FROM in_flight_tasks WHERE id = ?", (task_id,)).fetchone()
+ conn.execute("COMMIT")
+ return ClaimedTask(
+ id=task_id,
+ claim_token=claim_token,
+ lease_version=1,
+ expires_at=expires_at,
+ payload=dict(payload_row),
+ )
+
+ def renew_claim(
+ self,
+ task_id: str,
+ agent: str,
+ claim_token: str,
+ *,
+ lease_ttl_ms: int = DEFAULT_LEASE_TTL_MS,
+ now_ms: int | None = None,
+ ) -> ClaimedTask:
+ now = now_ms or utc_ms()
+ with self.connect() as conn:
+ conn.execute("BEGIN IMMEDIATE")
+ lock = self._valid_lock(conn, task_id, agent, claim_token, now)
+ lease_version = int(lock["lease_version"]) + 1
+ expires_at = now + lease_ttl_ms
+ conn.execute(
+ """
+ UPDATE claim_locks
+ SET lease_version = ?, expires_at = ?
+ WHERE task_id = ?
+ """,
+ (lease_version, expires_at, task_id),
+ )
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET active_lease_version = ?, updated_at = ?
+ WHERE id = ?
+ """,
+ (lease_version, now, task_id),
+ )
+ row = conn.execute("SELECT * FROM in_flight_tasks WHERE id = ?", (task_id,)).fetchone()
+ conn.execute("COMMIT")
+ return ClaimedTask(task_id, claim_token, lease_version, expires_at, dict(row))
+
+ def complete_task(
+ self,
+ task_id: str,
+ agent: str,
+ claim_token: str,
+ *,
+ results: dict[str, Any] | None = None,
+ summary: str = "",
+ target_wiki_path: str | None = None,
+ body_hash: str | None = None,
+ now_ms: int | None = None,
+ ) -> None:
+ now = now_ms or utc_ms()
+ with self.connect() as conn:
+ conn.execute("BEGIN IMMEDIATE")
+ self._valid_lock(conn, task_id, agent, claim_token, now)
+ conn.execute("DELETE FROM claim_locks WHERE task_id = ?", (task_id,))
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET status = 'completed',
+ completed_at = ?,
+ results_json = ?,
+ completion_summary = ?,
+ completion_body_hash = ?,
+ target_wiki_path = ?,
+ updated_at = ?
+ WHERE id = ?
+ """,
+ (
+ now,
+ json.dumps(results or {}, sort_keys=True),
+ summary,
+ body_hash,
+ target_wiki_path,
+ now,
+ task_id,
+ ),
+ )
+ conn.execute("COMMIT")
+
+ def sweep_expired_claims(self, *, now_ms: int | None = None) -> int:
+ with self.connect() as conn:
+ conn.execute("BEGIN IMMEDIATE")
+ count = self._sweep_expired_claims(conn, now_ms or utc_ms())
+ conn.execute("COMMIT")
+ return count
+
+ def pending_materializations(self) -> list[dict[str, Any]]:
+ with self.connect() as conn:
+ rows = conn.execute(
+ """
+ SELECT * FROM in_flight_tasks
+ WHERE status = 'completed'
+ AND materialized_at IS NULL
+ AND target_wiki_path IS NOT NULL
+ ORDER BY completed_at ASC
+ """
+ ).fetchall()
+ return [dict(row) for row in rows]
+
+ def mark_materialized(self, task_id: str, wiki_path: str, *, now_ms: int | None = None) -> None:
+ now = now_ms or utc_ms()
+ with self.connect() as conn:
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET wiki_path = ?, materialized_at = ?, updated_at = ?, last_error = NULL
+ WHERE id = ?
+ """,
+ (wiki_path, now, now, task_id),
+ )
+
+ def mark_materialization_error(self, task_id: str, error: str) -> None:
+ with self.connect() as conn:
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET completion_attempt_count = completion_attempt_count + 1,
+ last_error = ?,
+ updated_at = ?
+ WHERE id = ?
+ """,
+ (error[:1000], utc_ms(), task_id),
+ )
+
+ def heartbeat(self, agent: str, status: str = "ok", meta: dict[str, Any] | None = None) -> None:
+ now = utc_ms()
+ with self.connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO agent_state(agent, last_heartbeat, status, meta)
+ VALUES (?, ?, ?, ?)
+ ON CONFLICT(agent) DO UPDATE SET
+ last_heartbeat = excluded.last_heartbeat,
+ status = excluded.status,
+ meta = excluded.meta
+ """,
+ (agent, now, status, json.dumps(meta or {}, sort_keys=True)),
+ )
+
+ def increment_rate_limit(
+ self,
+ agent: str,
+ operation: str,
+ *,
+ window_ms: int = 3_600_000,
+ now_ms: int | None = None,
+ ) -> int:
+ now = now_ms or utc_ms()
+ bucket = (now // window_ms) * window_ms
+ with self.connect() as conn:
+ row = conn.execute(
+ """
+ INSERT INTO rate_limits(agent, operation, window_ms, bucket_start_ms, count, last_hit)
+ VALUES (?, ?, ?, ?, 1, ?)
+ ON CONFLICT(agent, operation, window_ms, bucket_start_ms)
+ DO UPDATE SET count = count + 1, last_hit = excluded.last_hit
+ RETURNING count
+ """,
+ (agent, operation, window_ms, bucket, now),
+ ).fetchone()
+ return int(row["count"])
+
+ def create_notification(self, agent: str, kind: str, body: str) -> str:
+ notification_id = str(uuid.uuid4())
+ with self.connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO notifications(id, agent, kind, body, created_at)
+ VALUES (?, ?, ?, ?, ?)
+ """,
+ (notification_id, agent, kind, body, utc_ms()),
+ )
+ return notification_id
+
+ def list_notifications(self, agent: str, *, unread_only: bool = False) -> list[dict[str, Any]]:
+ clause = "AND read_at IS NULL" if unread_only else ""
+ with self.connect() as conn:
+ rows = conn.execute(
+ f"SELECT * FROM notifications WHERE agent = ? {clause} ORDER BY created_at DESC",
+ (agent,),
+ ).fetchall()
+ return [dict(row) for row in rows]
+
+ def mark_notification_read(self, notification_id: str) -> None:
+ with self.connect() as conn:
+ conn.execute(
+ "UPDATE notifications SET read_at = ? WHERE id = ?",
+ (utc_ms(), notification_id),
+ )
+
+ def list_due_reminders(
+ self,
+ *,
+ now_ms: int | None = None,
+ limit: int = 50,
+ ) -> list[dict[str, Any]]:
+ cutoff = now_ms if now_ms is not None else utc_ms()
+ with self.connect() as conn:
+ rows = conn.execute(
+ """
+ SELECT * FROM reminders
+ WHERE status = 'pending' AND due_at <= ?
+ ORDER BY due_at
+ LIMIT ?
+ """,
+ (cutoff, limit),
+ ).fetchall()
+ return [dict(row) for row in rows]
+
+ def list_reminders(
+ self,
+ *,
+ event_id: str | None = None,
+ status: str | None = None,
+ limit: int = 100,
+ ) -> list[dict[str, Any]]:
+ clauses: list[str] = []
+ params: list[Any] = []
+ if event_id is not None:
+ clauses.append("event_id = ?")
+ params.append(event_id)
+ if status is not None:
+ clauses.append("status = ?")
+ params.append(status)
+ where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
+ params.append(limit)
+ with self.connect() as conn:
+ rows = conn.execute(
+ f"SELECT * FROM reminders{where} ORDER BY due_at ASC LIMIT ?",
+ params,
+ ).fetchall()
+ return [dict(row) for row in rows]
+
+ def create_reminder(
+ self,
+ *,
+ event_id: str,
+ due_at: int,
+ payload: dict[str, Any] | None = None,
+ channel: str | None = None,
+ offset_minutes: int | None = None,
+ agent: str | None = None,
+ reminder_id: str | None = None,
+ status: str = "pending",
+ ) -> str:
+ rid = reminder_id or str(uuid.uuid4())
+ now = utc_ms()
+ with self.connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO reminders
+ (id, agent, event_id, due_at, payload_json, status,
+ channel, offset_minutes, attempt_count, last_error,
+ created_at, updated_at, sent_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, NULL, ?, ?, NULL)
+ """,
+ (
+ rid, agent, event_id, due_at,
+ json.dumps(payload or {}, ensure_ascii=False, sort_keys=True),
+ status, channel, offset_minutes, now, now,
+ ),
+ )
+ return rid
+
+ def cancel_reminder(self, *, reminder_id: str) -> None:
+ with self.connect() as conn:
+ conn.execute(
+ "UPDATE reminders SET status='cancelled', updated_at=? WHERE id=?",
+ (utc_ms(), reminder_id),
+ )
+
+ def mark_reminder_sent(self, *, reminder_id: str, sent_at: int | None = None) -> None:
+ ts = sent_at if sent_at is not None else utc_ms()
+ with self.connect() as conn:
+ conn.execute(
+ "UPDATE reminders SET status='sent', sent_at=?, updated_at=? WHERE id=?",
+ (ts, ts, reminder_id),
+ )
+
+ def record_reminder_error(self, *, reminder_id: str, error: str) -> None:
+ with self.connect() as conn:
+ conn.execute(
+ """
+ UPDATE reminders
+ SET last_error = ?,
+ attempt_count = attempt_count + 1,
+ updated_at = ?
+ WHERE id = ?
+ """,
+ (error[:1000], utc_ms(), reminder_id),
+ )
+
+ def replace_event_reminders(
+ self,
+ *,
+ event_id: str,
+ reminders: list[dict[str, Any]],
+ ) -> list[str]:
+ ids: list[str] = []
+ now = utc_ms()
+ with self.connect() as conn:
+ conn.execute(
+ "UPDATE reminders SET status='cancelled', updated_at=? "
+ "WHERE event_id=? AND status='pending'",
+ (now, event_id),
+ )
+ for r in reminders:
+ rid = r.get("reminder_id") or str(uuid.uuid4())
+ ids.append(rid)
+ conn.execute(
+ """
+ INSERT INTO reminders
+ (id, agent, event_id, due_at, payload_json, status,
+ channel, offset_minutes, attempt_count, last_error,
+ created_at, updated_at, sent_at)
+ VALUES (?, ?, ?, ?, ?, 'pending', ?, ?, 0, NULL, ?, ?, NULL)
+ """,
+ (
+ rid, r.get("agent"), event_id,
+ int(r["due_at"]),
+ json.dumps(r.get("payload") or {}, ensure_ascii=False, sort_keys=True),
+ r.get("channel"), r.get("offset_minutes"),
+ now, now,
+ ),
+ )
+ return ids
+
+ def record_operator_action(
+ self,
+ *,
+ kind: str,
+ agent: str | None = None,
+ task_id: str | None = None,
+ promotion_id: str | None = None,
+ note: str | None = None,
+ payload: dict[str, Any] | None = None,
+ action_id: str | None = None,
+ ) -> str:
+ aid = action_id or str(uuid.uuid4())
+ now = utc_ms()
+ with self.connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO operator_actions
+ (id, kind, agent, task_id, promotion_id, note,
+ action_at, payload_json, created_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(id) DO UPDATE SET
+ kind = excluded.kind,
+ agent = excluded.agent,
+ task_id = excluded.task_id,
+ promotion_id = excluded.promotion_id,
+ note = excluded.note,
+ action_at = excluded.action_at,
+ payload_json = excluded.payload_json
+ """,
+ (
+ aid, kind, agent, task_id, promotion_id, note,
+ now,
+ json.dumps(payload or {}, ensure_ascii=False, sort_keys=True),
+ now,
+ ),
+ )
+ return aid
+
+ def list_operator_actions(
+ self,
+ *,
+ task_id: str | None = None,
+ promotion_id: str | None = None,
+ agent: str | None = None,
+ limit: int = 100,
+ ) -> list[dict[str, Any]]:
+ clauses: list[str] = []
+ params: list[Any] = []
+ if task_id is not None:
+ clauses.append("task_id = ?")
+ params.append(task_id)
+ if promotion_id is not None:
+ clauses.append("promotion_id = ?")
+ params.append(promotion_id)
+ if agent is not None:
+ clauses.append("agent = ?")
+ params.append(agent)
+ where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
+ params.append(limit)
+ with self.connect() as conn:
+ rows = conn.execute(
+ f"SELECT * FROM operator_actions{where} ORDER BY action_at DESC LIMIT ?",
+ params,
+ ).fetchall()
+ return [dict(row) for row in rows]
+
+ def list_tasks(
+ self,
+ *,
+ status: str | None = None,
+ status_in: list[str] | None = None,
+ agent: str | None = None,
+ delegated_by: str | None = None,
+ source: str | None = None,
+ pipeline_id: str | None = None,
+ parent_task: str | None = None,
+ paused: bool | None = None,
+ order_by: str = "priority_created",
+ limit: int = 100,
+ offset: int = 0,
+ include_total: bool = False,
+ ) -> list[dict[str, Any]] | dict[str, Any]:
+ """List tasks with optional filters.
+
+ New params (Phase 3 step 11):
+ - status_in: list of statuses (OR semantics, complements scalar status)
+ - pipeline_id, parent_task, paused: dashboard filters
+ - order_by: 'priority_created' (default, legacy) | 'dashboard'
+ ('dashboard' uses sort_order DESC, priority bucket, created_at DESC
+ to match the Kanban GET in server.js)
+ - include_total: when true, returns {items, total} instead of bare list
+
+ Existing callers using the original scalar params keep working unchanged.
+ """
+ clauses: list[str] = []
+ params: list[Any] = []
+ if status is not None:
+ status = self._validate_status(status)
+ clauses.append("status = ?")
+ params.append(status)
+ if status_in:
+ status_in = [self._validate_status(item) for item in status_in]
+ placeholders = ",".join(["?"] * len(status_in))
+ clauses.append(f"status IN ({placeholders})")
+ params.extend(status_in)
+ if agent is not None:
+ clauses.append("(assigned_to = ? OR claimed_by = ?)")
+ params.append(agent)
+ params.append(agent)
+ if delegated_by is not None:
+ clauses.append("delegated_by = ?")
+ params.append(delegated_by)
+ if source is not None:
+ clauses.append("source = ?")
+ params.append(source)
+ if pipeline_id is not None:
+ clauses.append("pipeline_id = ?")
+ params.append(pipeline_id)
+ if parent_task is not None:
+ clauses.append("parent_task = ?")
+ params.append(parent_task)
+ if paused is not None:
+ clauses.append("paused = ?")
+ params.append(1 if paused else 0)
+ where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
+ if order_by == "dashboard":
+ # Kanban uses string priority labels stored in 'priority' (text) for new
+ # rows but the legacy in_flight_tasks.priority is INTEGER. Order by
+ # sort_order DESC, then numeric priority DESC, then created_at DESC.
+ order_sql = "ORDER BY coalesce(sort_order, 0) DESC, priority DESC, created_at DESC"
+ else:
+ order_sql = "ORDER BY priority DESC, created_at ASC"
+ with self.connect() as conn:
+ if include_total:
+ count_row = conn.execute(
+ f"SELECT count(*) AS c FROM in_flight_tasks{where}",
+ params,
+ ).fetchone()
+ total = int(count_row["c"]) if count_row else 0
+ page_params = list(params) + [limit, offset]
+ rows = conn.execute(
+ f"SELECT * FROM in_flight_tasks{where} {order_sql} LIMIT ? OFFSET ?",
+ page_params,
+ ).fetchall()
+ items = [dict(row) for row in rows]
+ if include_total:
+ return {"items": items, "total": total}
+ return items
+
+ def get_task(self, task_id: str) -> dict[str, Any] | None:
+ with self.connect() as conn:
+ row = conn.execute(
+ "SELECT * FROM in_flight_tasks WHERE id = ?",
+ (task_id,),
+ ).fetchone()
+ if row is None:
+ return None
+ task = dict(row)
+ lock = conn.execute(
+ "SELECT * FROM claim_locks WHERE task_id = ?",
+ (task_id,),
+ ).fetchone()
+ task["claim_lock"] = dict(lock) if lock is not None else None
+ return task
+
+ def list_task_events(
+ self,
+ *,
+ task_id: str | None = None,
+ limit: int = 100,
+ offset: int = 0,
+ ) -> list[dict[str, Any]]:
+ clauses: list[str] = []
+ params: list[Any] = []
+ if task_id is not None:
+ clauses.append("task_id = ?")
+ params.append(task_id)
+ where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
+ params.append(limit)
+ params.append(offset)
+ with self.connect() as conn:
+ rows = conn.execute(
+ f"""
+ SELECT * FROM task_events{where}
+ ORDER BY occurred_at DESC, created_at DESC
+ LIMIT ? OFFSET ?
+ """,
+ params,
+ ).fetchall()
+ return [self._decode_json_fields(dict(row)) for row in rows]
+
+ def append_task_event(
+ self,
+ *,
+ task_id: str,
+ event_type: str,
+ agent: str | None = None,
+ details: dict[str, Any] | None = None,
+ occurred_at: int | None = None,
+ event_id: str | None = None,
+ ) -> str:
+ eid = event_id or str(uuid.uuid4())
+ now = utc_ms()
+ occurred = occurred_at if occurred_at is not None else now
+ with self.connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO task_events
+ (id, task_id, event_type, agent, details_json, occurred_at, created_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?)
+ """,
+ (
+ eid, task_id, event_type, agent,
+ json.dumps(details or {}, sort_keys=True),
+ occurred, now,
+ ),
+ )
+ return eid
+
+ def list_task_outputs(
+ self,
+ *,
+ task_id: str,
+ limit: int = 50,
+ ) -> list[dict[str, Any]]:
+ with self.connect() as conn:
+ rows = conn.execute(
+ """
+ SELECT * FROM task_outputs
+ WHERE task_id = ?
+ ORDER BY created_at DESC
+ LIMIT ?
+ """,
+ (task_id, limit),
+ ).fetchall()
+ return [self._decode_json_fields(dict(row)) for row in rows]
+
+ def append_task_output(
+ self,
+ *,
+ task_id: str,
+ kind: str,
+ summary: str | None = None,
+ payload: dict[str, Any] | None = None,
+ artifact_paths: list[str] | None = None,
+ output_id: str | None = None,
+ ) -> str:
+ oid = output_id or str(uuid.uuid4())
+ now = utc_ms()
+ with self.connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO task_outputs
+ (id, task_id, kind, summary, payload_json, artifact_paths_json, created_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?)
+ """,
+ (
+ oid, task_id, kind, summary,
+ json.dumps(payload or {}, sort_keys=True),
+ json.dumps(list(artifact_paths or []), sort_keys=True),
+ now,
+ ),
+ )
+ return oid
+
+ def list_task_outcomes(
+ self,
+ *,
+ task_id: str,
+ limit: int = 50,
+ ) -> list[dict[str, Any]]:
+ with self.connect() as conn:
+ rows = conn.execute(
+ """
+ SELECT * FROM task_outcomes
+ WHERE task_id = ?
+ ORDER BY occurred_at DESC
+ LIMIT ?
+ """,
+ (task_id, limit),
+ ).fetchall()
+ return [self._decode_json_fields(dict(row)) for row in rows]
+
+ def append_task_outcome(
+ self,
+ *,
+ task_id: str,
+ status: str,
+ reason: str | None = None,
+ payload: dict[str, Any] | None = None,
+ occurred_at: int | None = None,
+ outcome_id: str | None = None,
+ ) -> str:
+ oid = outcome_id or str(uuid.uuid4())
+ now = utc_ms()
+ occurred = occurred_at if occurred_at is not None else now
+ with self.connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO task_outcomes
+ (id, task_id, status, reason, payload_json, occurred_at, created_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?)
+ """,
+ (
+ oid, task_id, status, reason,
+ json.dumps(payload or {}, sort_keys=True),
+ occurred, now,
+ ),
+ )
+ return oid
+
+ def list_failure_reports(
+ self,
+ *,
+ task_id: str | None = None,
+ agent: str | None = None,
+ since_ms: int | None = None,
+ limit: int = 100,
+ ) -> list[dict[str, Any]]:
+ clauses: list[str] = []
+ params: list[Any] = []
+ if task_id is not None:
+ clauses.append("task_id = ?")
+ params.append(task_id)
+ if agent is not None:
+ clauses.append("agent = ?")
+ params.append(agent)
+ if since_ms is not None:
+ clauses.append("occurred_at >= ?")
+ params.append(since_ms)
+ where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
+ params.append(limit)
+ with self.connect() as conn:
+ rows = conn.execute(
+ f"""
+ SELECT * FROM failure_reports{where}
+ ORDER BY occurred_at DESC
+ LIMIT ?
+ """,
+ params,
+ ).fetchall()
+ return [self._decode_json_fields(dict(row)) for row in rows]
+
+ def append_failure_report(
+ self,
+ *,
+ task_id: str | None = None,
+ error_class: str,
+ message: str,
+ agent: str | None = None,
+ stack: str | None = None,
+ recovery: str | None = None,
+ payload: dict[str, Any] | None = None,
+ occurred_at: int | None = None,
+ report_id: str | None = None,
+ ) -> str:
+ rid = report_id or str(uuid.uuid4())
+ now = utc_ms()
+ occurred = occurred_at if occurred_at is not None else now
+ with self.connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO failure_reports
+ (id, task_id, agent, error_class, message, stack, recovery,
+ payload_json, occurred_at, created_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ """,
+ (
+ rid, task_id, agent, error_class, message, stack, recovery,
+ json.dumps(payload or {}, sort_keys=True),
+ occurred, now,
+ ),
+ )
+ return rid
+
+ def task_analytics(
+ self,
+ *,
+ since_ms: int | None = None,
+ agent: str | None = None,
+ group_by: str = "status",
+ ) -> dict[str, Any]:
+ """Aggregated task statistics for Hermes pipeline-analytics queries.
+
+ Returns counts grouped by ``group_by`` (one of 'status', 'agent',
+ 'retry_count'), plus duration and retry summaries across the same
+ filtered slice. Replaces the LLM-driven Cypher analytics cookbook used
+ by ``kurultai-queue-diagnostic/references/pipeline-analytics-queries.md``.
+ """
+ allowed_group_by = {"status", "agent", "retry_count"}
+ if group_by not in allowed_group_by:
+ raise ValueError(f"unsupported group_by: {group_by!r}")
+ clauses: list[str] = []
+ params: list[Any] = []
+ if since_ms is not None:
+ clauses.append("created_at >= ?")
+ params.append(since_ms)
+ if agent is not None:
+ clauses.append("(assigned_to = ? OR claimed_by = ?)")
+ params.append(agent)
+ params.append(agent)
+ where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
+
+ if group_by == "status":
+ group_col = "status"
+ elif group_by == "agent":
+ group_col = "COALESCE(claimed_by, assigned_to, '')"
+ else:
+ group_col = "retry_count"
+
+ with self.connect() as conn:
+ counts_rows = conn.execute(
+ f"""
+ SELECT {group_col} AS bucket, count(*) AS n
+ FROM in_flight_tasks{where}
+ GROUP BY bucket
+ ORDER BY n DESC
+ """,
+ params,
+ ).fetchall()
+ counts = {str(row["bucket"]) if row["bucket"] is not None else "": int(row["n"]) for row in counts_rows}
+
+ # Status counts (always returned, regardless of group_by)
+ status_rows = conn.execute(
+ f"SELECT status, count(*) AS n FROM in_flight_tasks{where} GROUP BY status",
+ params,
+ ).fetchall()
+ counts_by_status = {row["status"]: int(row["n"]) for row in status_rows}
+
+ # Retry summary
+ retry_rows = conn.execute(
+ f"""
+ SELECT
+ count(*) AS total,
+ COALESCE(sum(CASE WHEN retry_count > 0 THEN 1 ELSE 0 END), 0) AS retried,
+ COALESCE(max(retry_count), 0) AS max_retries,
+ COALESCE(avg(retry_count), 0.0) AS mean_retries
+ FROM in_flight_tasks{where}
+ """,
+ params,
+ ).fetchone()
+ total = int(retry_rows["total"]) if retry_rows else 0
+ retried = int(retry_rows["retried"]) if retry_rows else 0
+ failure_clauses = list(clauses)
+ failure_params = list(params)
+ failure_clauses.append("status = 'failed'")
+ failure_where = " WHERE " + " AND ".join(failure_clauses)
+ failed_count_row = conn.execute(
+ f"SELECT count(*) AS n FROM in_flight_tasks{failure_where}",
+ failure_params,
+ ).fetchone()
+ failed_count = int(failed_count_row["n"]) if failed_count_row else 0
+
+ # Durations (in_progress -> completed) using completed_at - claimed_at
+ duration_clauses = list(clauses)
+ duration_params = list(params)
+ duration_clauses.append("status = 'completed'")
+ duration_clauses.append("completed_at IS NOT NULL")
+ duration_clauses.append("claimed_at IS NOT NULL")
+ duration_where = " WHERE " + " AND ".join(duration_clauses)
+ durations = [
+ int(row["d"])
+ for row in conn.execute(
+ f"""
+ SELECT (completed_at - claimed_at) AS d
+ FROM in_flight_tasks{duration_where}
+ ORDER BY d ASC
+ """,
+ duration_params,
+ ).fetchall()
+ if row["d"] is not None and int(row["d"]) >= 0
+ ]
+ duration_summary: dict[str, Any] = {
+ "samples": len(durations),
+ "mean_ms": (sum(durations) / len(durations)) if durations else 0.0,
+ "p50_ms": _percentile(durations, 0.50),
+ "p90_ms": _percentile(durations, 0.90),
+ }
+ return {
+ "group_by": group_by,
+ "counts": counts,
+ "counts_by_status": counts_by_status,
+ "total_tasks": total,
+ "retried_tasks": retried,
+ "max_retries_observed": int(retry_rows["max_retries"]) if retry_rows else 0,
+ "mean_retries": float(retry_rows["mean_retries"]) if retry_rows else 0.0,
+ "failed_count": failed_count,
+ "failure_rate": (failed_count / total) if total else 0.0,
+ "duration": duration_summary,
+ }
+
+ def cap_retry_count(
+ self,
+ *,
+ task_ids: list[str],
+ max_retries: int,
+ ) -> dict[str, Any]:
+ """Cap ``retry_count`` to ``max_retries`` for the given task ids.
+
+ Returns ``{updated, ids}`` listing tasks whose retry_count exceeded
+ ``max_retries`` and were lowered. Used by the codebase-autopilot
+ retry-overshoot remediation; replaces the LLM-emitted ``SET t.retry_count
+ = toInteger(t.max_retries)`` Cypher path.
+ """
+ if not isinstance(max_retries, int) or max_retries < 0:
+ raise ValueError("max_retries must be a non-negative int")
+ if not task_ids:
+ return {"updated": 0, "ids": []}
+ now = utc_ms()
+ updated_ids: list[str] = []
+ with self.connect() as conn:
+ conn.execute("BEGIN IMMEDIATE")
+ placeholders = ",".join("?" * len(task_ids))
+ rows = conn.execute(
+ f"""
+ SELECT id, retry_count FROM in_flight_tasks
+ WHERE id IN ({placeholders}) AND retry_count > ?
+ """,
+ (*task_ids, max_retries),
+ ).fetchall()
+ for row in rows:
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET retry_count = ?, updated_at = ?
+ WHERE id = ?
+ """,
+ (max_retries, now, row["id"]),
+ )
+ conn.execute(
+ """
+ INSERT INTO task_events
+ (id, task_id, event_type, agent, details_json, occurred_at, created_at)
+ VALUES (?, ?, 'retry_capped', NULL, ?, ?, ?)
+ """,
+ (
+ str(uuid.uuid4()),
+ row["id"],
+ json.dumps(
+ {
+ "previous_retry_count": int(row["retry_count"]),
+ "capped_to": max_retries,
+ },
+ sort_keys=True,
+ ),
+ now,
+ now,
+ ),
+ )
+ updated_ids.append(row["id"])
+ conn.execute("COMMIT")
+ return {"updated": len(updated_ids), "ids": updated_ids}
+
+ def cancel_tasks(
+ self,
+ *,
+ task_ids: list[str],
+ reason: str,
+ agent: str | None = None,
+ ) -> dict[str, Any]:
+ """Bulk-cancel tasks. Skips already-terminal tasks.
+
+ Returns ``{cancelled, skipped}`` with per-task disposition. Each
+ cancellation appends a ``task_events`` entry of type ``cancelled``
+ with the reason and the (optional) requesting agent. Replaces the
+ LLM-emitted ``SET t.status = 'CANCELLED'`` Cypher path used by
+ ``kurultai-pipeline-stall-diagnosis`` and the daily queue patrol.
+ """
+ if not reason:
+ raise ValueError("reason is required for cancel_tasks")
+ if not task_ids:
+ return {"cancelled": 0, "skipped": []}
+ now = utc_ms()
+ cancelled: list[str] = []
+ skipped: list[dict[str, Any]] = []
+ with self.connect() as conn:
+ conn.execute("BEGIN IMMEDIATE")
+ placeholders = ",".join("?" * len(task_ids))
+ rows = conn.execute(
+ f"SELECT id, status FROM in_flight_tasks WHERE id IN ({placeholders})",
+ tuple(task_ids),
+ ).fetchall()
+ present = {row["id"]: row["status"] for row in rows}
+ terminal = {"completed", "failed", "cancelled"}
+ for tid in task_ids:
+ status = present.get(tid)
+ if status is None:
+ skipped.append({"id": tid, "reason": "not_found"})
+ continue
+ if status in terminal:
+ skipped.append({"id": tid, "reason": f"already_{status}"})
+ continue
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET status = 'cancelled',
+ failed_at = ?,
+ updated_at = ?,
+ last_error = ?
+ WHERE id = ?
+ """,
+ (now, now, f"cancelled: {reason[:500]}", tid),
+ )
+ conn.execute("DELETE FROM claim_locks WHERE task_id = ?", (tid,))
+ conn.execute(
+ """
+ INSERT INTO task_events
+ (id, task_id, event_type, agent, details_json, occurred_at, created_at)
+ VALUES (?, ?, 'cancelled', ?, ?, ?, ?)
+ """,
+ (
+ str(uuid.uuid4()),
+ tid,
+ agent,
+ json.dumps({"reason": reason, "previous_status": status}, sort_keys=True),
+ now,
+ now,
+ ),
+ )
+ cancelled.append(tid)
+ conn.execute("COMMIT")
+ return {"cancelled": len(cancelled), "ids": cancelled, "skipped": skipped}
+
+ def reset_to_pending(
+ self,
+ *,
+ task_ids: list[str],
+ agent: str | None = None,
+ reason: str | None = None,
+ ) -> dict[str, Any]:
+ """Reset orphaned in_progress tasks back to pending and clear claims.
+
+ Used by the daily queue patrol to recover orphaned WORKING/IN_PROGRESS
+ tasks. Skips tasks already in a terminal state; only acts on tasks
+ currently ``in_progress``. Returns ``{reset, skipped}``.
+ """
+ if not task_ids:
+ return {"reset": 0, "skipped": []}
+ now = utc_ms()
+ reset_ids: list[str] = []
+ skipped: list[dict[str, Any]] = []
+ with self.connect() as conn:
+ conn.execute("BEGIN IMMEDIATE")
+ placeholders = ",".join("?" * len(task_ids))
+ rows = conn.execute(
+ f"SELECT id, status FROM in_flight_tasks WHERE id IN ({placeholders})",
+ tuple(task_ids),
+ ).fetchall()
+ present = {row["id"]: row["status"] for row in rows}
+ for tid in task_ids:
+ status = present.get(tid)
+ if status is None:
+ skipped.append({"id": tid, "reason": "not_found"})
+ continue
+ if status != "in_progress":
+ skipped.append({"id": tid, "reason": f"not_in_progress:{status}"})
+ continue
+ conn.execute("DELETE FROM claim_locks WHERE task_id = ?", (tid,))
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET status = 'pending',
+ claimed_by = NULL,
+ claimed_at = NULL,
+ active_claim_token = NULL,
+ active_lease_version = NULL,
+ updated_at = ?
+ WHERE id = ?
+ """,
+ (now, tid),
+ )
+ conn.execute(
+ """
+ INSERT INTO task_events
+ (id, task_id, event_type, agent, details_json, occurred_at, created_at)
+ VALUES (?, ?, 'reset_to_pending', ?, ?, ?, ?)
+ """,
+ (
+ str(uuid.uuid4()),
+ tid,
+ agent,
+ json.dumps({"reason": reason or "orphaned-claim", "previous_status": status}, sort_keys=True),
+ now,
+ now,
+ ),
+ )
+ reset_ids.append(tid)
+ conn.execute("COMMIT")
+ return {"reset": len(reset_ids), "ids": reset_ids, "skipped": skipped}
+
+ def recover_expired_claims(
+ self,
+ *,
+ grace_minutes: int = 10,
+ limit: int = 100,
+ agent: str | None = None,
+ reason: str | None = None,
+ ) -> list[dict[str, Any]]:
+ """Reset in-progress tasks whose claim lock has expired.
+
+ This replaces the old graph ``recover_orphans`` path used by
+ task-reaper. It returns the recovered task rows with the previous lock
+ metadata needed for logs/alerts.
+ """
+ now = utc_ms()
+ cutoff = now - max(0, int(grace_minutes)) * 60_000
+ clauses = [
+ "t.status IN ('in_progress', 'IN_PROGRESS', 'WORKING')",
+ "l.expires_at < ?",
+ ]
+ params: list[Any] = [cutoff]
+ if agent:
+ clauses.append("(t.assigned_to = ? OR t.claimed_by = ?)")
+ params.extend([agent, agent])
+ params.append(max(1, int(limit)))
+ recovered: list[dict[str, Any]] = []
+ with self.connect() as conn:
+ conn.execute("BEGIN IMMEDIATE")
+ rows = conn.execute(
+ f"""
+ SELECT t.*, l.claimed_by AS lock_claimed_by,
+ l.claim_token AS lock_claim_token,
+ l.lease_version AS lock_lease_version,
+ l.expires_at AS lock_expires_at
+ FROM in_flight_tasks t
+ JOIN claim_locks l ON l.task_id = t.id
+ WHERE {' AND '.join(clauses)}
+ ORDER BY l.expires_at ASC
+ LIMIT ?
+ """,
+ params,
+ ).fetchall()
+ for row in rows:
+ task = dict(row)
+ tid = task["id"]
+ prev = task["status"]
+ conn.execute("DELETE FROM claim_locks WHERE task_id = ?", (tid,))
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET previous_status = ?,
+ status = 'pending',
+ claimed_by = NULL,
+ claimed_at = NULL,
+ active_claim_token = NULL,
+ active_lease_version = NULL,
+ updated_at = ?
+ WHERE id = ?
+ """,
+ (prev, now, tid),
+ )
+ self._append_event_inline(
+ conn,
+ task_id=tid,
+ event_type="expired_claim_recovered",
+ agent=agent or "task-reaper",
+ details={
+ "reason": reason or "expired_claim",
+ "previous_status": prev,
+ "lock_expires_at": task.get("lock_expires_at"),
+ },
+ now=now,
+ )
+ task["previous_status"] = prev
+ task["status"] = "pending"
+ recovered.append(task)
+ conn.execute("COMMIT")
+ return recovered
+
+ def promote_orphaned_tasks(
+ self,
+ *,
+ hold_minutes: int = 5,
+ limit: int = 100,
+ agent: str | None = None,
+ ) -> list[dict[str, Any]]:
+ """Promote legacy ORPHANED rows back to pending after a hold period."""
+ now = utc_ms()
+ cutoff = now - max(0, int(hold_minutes)) * 60_000
+ clauses = ["status = 'ORPHANED'", "updated_at < ?"]
+ params: list[Any] = [cutoff]
+ if agent:
+ clauses.append("assigned_to = ?")
+ params.append(agent)
+ params.append(max(1, int(limit)))
+ promoted: list[dict[str, Any]] = []
+ with self.connect() as conn:
+ conn.execute("BEGIN IMMEDIATE")
+ rows = conn.execute(
+ f"""
+ SELECT * FROM in_flight_tasks
+ WHERE {' AND '.join(clauses)}
+ ORDER BY updated_at ASC
+ LIMIT ?
+ """,
+ params,
+ ).fetchall()
+ for row in rows:
+ task = dict(row)
+ tid = task["id"]
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET previous_status = status, status = 'pending', updated_at = ?
+ WHERE id = ?
+ """,
+ (now, tid),
+ )
+ self._append_event_inline(
+ conn,
+ task_id=tid,
+ event_type="orphan_promoted",
+ agent=agent or "task-reaper",
+ details={"previous_status": task["status"]},
+ now=now,
+ )
+ task["previous_status"] = task["status"]
+ task["status"] = "pending"
+ promoted.append(task)
+ conn.execute("COMMIT")
+ return promoted
+
+ def promote_ready_pipeline_tasks(
+ self,
+ *,
+ pipeline_id: str | None = None,
+ limit: int = 100,
+ agent: str | None = None,
+ ) -> list[dict[str, Any]]:
+ """Move blocked pipeline tasks to pending once all dependencies finish."""
+ now = utc_ms()
+ params: list[Any] = []
+ pipeline_clause = ""
+ if pipeline_id is not None:
+ pipeline_clause = "AND t.pipeline_id = ?"
+ params.append(pipeline_id)
+ params.append(max(1, int(limit)))
+ promoted: list[dict[str, Any]] = []
+ with self.connect() as conn:
+ conn.execute("BEGIN IMMEDIATE")
+ rows = conn.execute(
+ f"""
+ SELECT t.*
+ FROM in_flight_tasks t
+ WHERE t.status = 'blocked'
+ {pipeline_clause}
+ AND EXISTS (
+ SELECT 1 FROM task_dependencies d WHERE d.task_id = t.id
+ )
+ AND NOT EXISTS (
+ SELECT 1
+ FROM task_dependencies d
+ LEFT JOIN in_flight_tasks dep ON dep.id = d.depends_on_task_id
+ WHERE d.task_id = t.id
+ AND COALESCE(dep.status, 'missing') NOT IN ('completed', 'COMPLETED', 'DONE')
+ )
+ ORDER BY t.priority DESC, t.created_at ASC
+ LIMIT ?
+ """,
+ params,
+ ).fetchall()
+ for row in rows:
+ task = dict(row)
+ tid = task["id"]
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET previous_status = status, status = 'pending', updated_at = ?
+ WHERE id = ?
+ """,
+ (now, tid),
+ )
+ self._append_event_inline(
+ conn,
+ task_id=tid,
+ event_type="pipeline_dependencies_satisfied",
+ agent=agent or "pipeline",
+ details={"pipeline_id": task.get("pipeline_id")},
+ now=now,
+ )
+ task["previous_status"] = task["status"]
+ task["status"] = "pending"
+ promoted.append(task)
+ conn.execute("COMMIT")
+ return promoted
+
+ def list_pipeline_status(self, *, pipeline_id: str) -> dict[str, Any]:
+ tasks = self.list_tasks(pipeline_id=pipeline_id, limit=500, order_by="dashboard")
+ summary: dict[str, int] = {}
+ task_ids = [task["id"] for task in tasks]
+ dependencies: dict[str, list[str]] = {tid: [] for tid in task_ids}
+ if task_ids:
+ placeholders = ",".join("?" * len(task_ids))
+ with self.connect() as conn:
+ rows = conn.execute(
+ f"""
+ SELECT task_id, depends_on_task_id
+ FROM task_dependencies
+ WHERE task_id IN ({placeholders})
+ ORDER BY task_id, depends_on_task_id
+ """,
+ tuple(task_ids),
+ ).fetchall()
+ for row in rows:
+ dependencies.setdefault(row["task_id"], []).append(row["depends_on_task_id"])
+ for task in tasks:
+ status = str(task.get("status") or "")
+ summary[status] = summary.get(status, 0) + 1
+ task["depends_on"] = dependencies.get(task["id"], [])
+ task["phase"] = task.get("dispatch_phase")
+ task["agent"] = task.get("assigned_to")
+ return {"pipeline_id": pipeline_id, "tasks": tasks, "summary": summary}
+
+ def cleanup_pipeline(self, *, pipeline_id: str) -> dict[str, Any]:
+ now = utc_ms()
+ with self.connect() as conn:
+ conn.execute("BEGIN IMMEDIATE")
+ rows = conn.execute(
+ "SELECT id FROM in_flight_tasks WHERE pipeline_id = ?",
+ (pipeline_id,),
+ ).fetchall()
+ ids = [row["id"] for row in rows]
+ if ids:
+ placeholders = ",".join("?" * len(ids))
+ conn.execute(
+ f"DELETE FROM task_dependencies WHERE task_id IN ({placeholders})",
+ tuple(ids),
+ )
+ conn.execute(
+ f"DELETE FROM task_dependencies WHERE depends_on_task_id IN ({placeholders})",
+ tuple(ids),
+ )
+ conn.execute(
+ f"DELETE FROM task_events WHERE task_id IN ({placeholders})",
+ tuple(ids),
+ )
+ conn.execute(
+ f"DELETE FROM task_outputs WHERE task_id IN ({placeholders})",
+ tuple(ids),
+ )
+ conn.execute(
+ f"DELETE FROM task_outcomes WHERE task_id IN ({placeholders})",
+ tuple(ids),
+ )
+ conn.execute(
+ f"DELETE FROM failure_reports WHERE task_id IN ({placeholders})",
+ tuple(ids),
+ )
+ conn.execute(
+ "INSERT INTO audit_events (id, actor, action, decision, resource, details_json, created_at) "
+ "VALUES (?, 'pipeline', 'cleanup_pipeline', 'allow', ?, ?, ?)",
+ (
+ str(uuid.uuid4()),
+ pipeline_id,
+ json.dumps({"deleted_task_ids": ids}, sort_keys=True),
+ now,
+ ),
+ )
+ conn.execute(
+ f"DELETE FROM in_flight_tasks WHERE id IN ({placeholders})",
+ tuple(ids),
+ )
+ conn.execute("COMMIT")
+ return {"pipeline_id": pipeline_id, "deleted": len(ids), "ids": ids}
+
+ def list_terminal_tasks(
+ self,
+ *,
+ since_ms: int,
+ status_filter: list[str] | None = None,
+ limit: int = 100,
+ ) -> list[dict[str, Any]]:
+ """Tasks that reached a terminal status since ``since_ms``.
+
+ Replaces the obsolete ``MATCH (t:Task) WHERE t.status IN [...]`` Cypher
+ used by ``kurultai_terminal_task_notifier.py`` (now a SQLite reader,
+ which we wrap so all Hermes consumers traverse the same RPC surface).
+ Ordered by completion/failure time, descending.
+ """
+ if status_filter is None:
+ status_filter = ["completed", "failed", "cancelled"]
+ allowed = {"completed", "failed", "cancelled"}
+ for s in status_filter:
+ if s not in allowed:
+ raise ValueError(f"unsupported status_filter value: {s!r}")
+ placeholders = ",".join("?" * len(status_filter))
+ params: list[Any] = list(status_filter)
+ params.append(since_ms)
+ params.append(limit)
+ with self.connect() as conn:
+ rows = conn.execute(
+ f"""
+ SELECT * FROM in_flight_tasks
+ WHERE status IN ({placeholders})
+ AND COALESCE(completed_at, failed_at, updated_at) >= ?
+ ORDER BY COALESCE(completed_at, failed_at, updated_at) DESC
+ LIMIT ?
+ """,
+ params,
+ ).fetchall()
+ return [dict(row) for row in rows]
+
+ # ------------------------------------------------------------------
+ # Phase 3 step 11: task CRUD RPCs called by the-kurultai dashboard.
+ # These complement (do not replace) create_task / claim_task / complete_task,
+ # which are still used by the Mac-mini executor on the wiki materialization
+ # path. Every mutation appends a task_event row for audit / revert support.
+ # ------------------------------------------------------------------
+
+ _ALLOWED_STATUSES = (
+ "pending", "in_progress", "completed", "failed", "cancelled",
+ "blocked", "awaiting_approval", "obsolete",
+ )
+ _STATUS_ALIASES = {
+ "PENDING": "pending",
+ "WORKING": "in_progress",
+ "IN_PROGRESS": "in_progress",
+ "COMPLETED": "completed",
+ "DONE": "completed",
+ "FAILED": "failed",
+ "ORPHANED": "failed",
+ "STALE": "failed",
+ "CANCELLED": "cancelled",
+ "CANCELED": "cancelled",
+ "BLOCKED": "blocked",
+ "AWAITING_APPROVAL": "awaiting_approval",
+ "OBSOLETE": "obsolete",
+ }
+
+ def _validate_status(self, status: str) -> str:
+ normalized = self._normalize_status(status)
+ if normalized not in self._ALLOWED_STATUSES:
+ raise ValueError(
+ f"invalid status {status!r}; must be one of {self._ALLOWED_STATUSES}"
+ )
+ return normalized
+
+ def _normalize_status(self, status: str) -> str:
+ if not isinstance(status, str):
+ raise ValueError(f"invalid status {status!r}; must be a string")
+ stripped = status.strip()
+ return self._STATUS_ALIASES.get(stripped, stripped.lower())
+
+ def _coerce_priority(self, priority: Any) -> int:
+ # The dashboard sends string priorities (critical/high/normal/low). Map
+ # them to the integer column. Numeric inputs pass through unchanged.
+ if priority is None:
+ return 0
+ if isinstance(priority, int):
+ return priority
+ if isinstance(priority, str):
+ mapping = {"critical": 100, "high": 50, "normal": 10, "low": 1}
+ try:
+ return int(priority)
+ except ValueError:
+ return mapping.get(priority.lower(), 10)
+ try:
+ return int(priority)
+ except (TypeError, ValueError):
+ return 0
+
+ def create_task_full(
+ self,
+ *,
+ task_id: str | None = None,
+ title: str | None = None,
+ prompt: str | None = None,
+ description: str | None = None,
+ delegated_by: str = "system",
+ assigned_to: str | None = None,
+ priority: Any = "normal",
+ status: str = "pending",
+ type: str = "task",
+ domain: str | None = None,
+ source: str | None = None,
+ parent_task: str | None = None,
+ reflection_id: str | None = None,
+ pipeline_id: str | None = None,
+ sort_order: int | None = None,
+ dispatch_phase: str | None = None,
+ max_retries: int = 3,
+ timeout_s: int | None = None,
+ depth: int = 0,
+ requires_computer_use: bool = False,
+ skill_hint: str | None = None,
+ score: float | None = None,
+ results: dict[str, Any] | None = None,
+ trace_id: str | None = None,
+ depends_on: list[str] | None = None,
+ ) -> dict[str, Any]:
+ """Create a task with the full Phase 3 column set (no claim).
+
+ Distinct from create_task: takes the dashboard's string priorities,
+ sets PENDING (uppercase) by default, and accepts every column the
+ the-kurultai Kanban UI exposes.
+ """
+ dependencies = [str(dep) for dep in (depends_on or []) if str(dep)]
+ status = self._validate_status(status)
+ if dependencies and status == "pending":
+ status = "blocked"
+ now = utc_ms()
+ tid = task_id or f"task-{uuid.uuid4()}"
+ trace_id = trace_id or new_trace_id()
+ prio = self._coerce_priority(priority)
+ desc = description or title or prompt or tid
+ with self.connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO in_flight_tasks (
+ id, type, description, delegated_by, assigned_to, priority, status,
+ results_json, trace_id, reliability_state, retry_count,
+ created_at, updated_at,
+ title, prompt, domain, source, parent_task, reflection_id,
+ pipeline_id, sort_order, dispatch_phase, max_retries, timeout_s,
+ depth, requires_computer_use, skill_hint, score, original_prompt
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', 0, ?, ?,
+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ """,
+ (
+ tid, type, desc, delegated_by, assigned_to, prio, status,
+ json.dumps(results or {}, sort_keys=True), trace_id, now, now,
+ title, prompt, domain, source, parent_task, reflection_id,
+ pipeline_id, sort_order, dispatch_phase, max_retries, timeout_s,
+ depth, 1 if requires_computer_use else 0, skill_hint, score,
+ prompt,
+ ),
+ )
+ for dep in dependencies:
+ conn.execute(
+ """
+ INSERT OR IGNORE INTO task_dependencies
+ (task_id, depends_on_task_id, created_at)
+ VALUES (?, ?, ?)
+ """,
+ (tid, dep, now),
+ )
+ self._append_event_inline(
+ conn, task_id=tid, event_type="created", agent=delegated_by,
+ details={
+ "status": status,
+ "title": title,
+ "source": source,
+ "depends_on": dependencies,
+ },
+ now=now,
+ )
+ return {"task_id": tid}
+
+ def _append_event_inline(
+ self,
+ conn: sqlite3.Connection,
+ *,
+ task_id: str,
+ event_type: str,
+ agent: str | None,
+ details: dict[str, Any] | None,
+ now: int,
+ ) -> str:
+ eid = str(uuid.uuid4())
+ conn.execute(
+ """
+ INSERT INTO task_events (id, task_id, event_type, agent, details_json, occurred_at, created_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?)
+ """,
+ (eid, task_id, event_type, agent, json.dumps(details or {}, sort_keys=True), now, now),
+ )
+ return eid
+
+ def set_task_status(
+ self,
+ *,
+ task_id: str,
+ status: str,
+ agent: str | None = None,
+ reason: str | None = None,
+ ) -> dict[str, Any]:
+ status = self._validate_status(status)
+ now = utc_ms()
+ with self.connect() as conn:
+ row = conn.execute(
+ "SELECT status FROM in_flight_tasks WHERE id = ?", (task_id,)
+ ).fetchone()
+ if row is None:
+ raise ValueError(f"task {task_id} not found")
+ prev = row["status"]
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET previous_status = ?, status = ?, updated_at = ?
+ WHERE id = ?
+ """,
+ (prev, status, now, task_id),
+ )
+ self._append_event_inline(
+ conn, task_id=task_id, event_type="status_change", agent=agent,
+ details={"previous_status": prev, "status": status, "reason": reason},
+ now=now,
+ )
+ return {"task_id": task_id, "previous_status": prev, "status": status}
+
+ def retry_task(
+ self,
+ *,
+ task_id: str,
+ agent: str | None = None,
+ ) -> dict[str, Any]:
+ now = utc_ms()
+ with self.connect() as conn:
+ row = conn.execute(
+ "SELECT status, claim_epoch, retry_count FROM in_flight_tasks WHERE id = ?",
+ (task_id,),
+ ).fetchone()
+ if row is None:
+ raise ValueError(f"task {task_id} not found")
+ new_epoch = (row["claim_epoch"] or 0) + 1
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET previous_status = status,
+ status = 'pending',
+ retry_count = 0,
+ claim_epoch = ?,
+ updated_at = ?
+ WHERE id = ?
+ """,
+ (new_epoch, now, task_id),
+ )
+ conn.execute("DELETE FROM claim_locks WHERE task_id = ?", (task_id,))
+ self._append_event_inline(
+ conn, task_id=task_id, event_type="retry", agent=agent,
+ details={"previous_status": row["status"], "claim_epoch": new_epoch},
+ now=now,
+ )
+ return {"task_id": task_id, "claim_epoch": new_epoch, "previous_status": row["status"]}
+
+ def redo_task(
+ self,
+ *,
+ task_id: str,
+ agent: str | None = None,
+ new_prompt: str | None = None,
+ new_task_id: str | None = None,
+ ) -> dict[str, Any]:
+ """Clone the task with a fresh task_id at PENDING. Optionally override prompt."""
+ now = utc_ms()
+ with self.connect() as conn:
+ src = conn.execute(
+ "SELECT * FROM in_flight_tasks WHERE id = ?", (task_id,)
+ ).fetchone()
+ if src is None:
+ raise ValueError(f"task {task_id} not found")
+ new_tid = new_task_id or f"task-{uuid.uuid4()}"
+ prompt = new_prompt if new_prompt is not None else src["prompt"]
+ trace_id = new_trace_id()
+ conn.execute(
+ """
+ INSERT INTO in_flight_tasks (
+ id, type, description, delegated_by, assigned_to, priority, status,
+ results_json, trace_id, reliability_state, retry_count,
+ created_at, updated_at,
+ title, prompt, domain, source, parent_task, reflection_id,
+ pipeline_id, dispatch_phase, max_retries, timeout_s,
+ depth, requires_computer_use, skill_hint, score, original_prompt
+ ) VALUES (?, ?, ?, ?, ?, ?, 'PENDING', '{}', ?, 'pending', 0, ?, ?,
+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ """,
+ (
+ new_tid, src["type"], src["description"], src["delegated_by"],
+ src["assigned_to"], src["priority"], trace_id, now, now,
+ src["title"], prompt, src["domain"], src["source"],
+ src["parent_task"], src["reflection_id"], src["pipeline_id"],
+ src["dispatch_phase"], src["max_retries"], src["timeout_s"],
+ src["depth"], src["requires_computer_use"], src["skill_hint"],
+ src["score"], src["prompt"],
+ ),
+ )
+ self._append_event_inline(
+ conn, task_id=new_tid, event_type="redo", agent=agent,
+ details={"source_task_id": task_id, "new_prompt": new_prompt is not None},
+ now=now,
+ )
+ self._append_event_inline(
+ conn, task_id=task_id, event_type="redo_source", agent=agent,
+ details={"new_task_id": new_tid}, now=now,
+ )
+ return {"new_task_id": new_tid, "source_task_id": task_id}
+
+ def retry_all_tasks(
+ self,
+ *,
+ status: str | None = "FAILED",
+ agent: str | None = None,
+ pipeline_id: str | None = None,
+ limit: int = 50,
+ acting_agent: str | None = None,
+ ) -> dict[str, Any]:
+ now = utc_ms()
+ clauses: list[str] = []
+ params: list[Any] = []
+ if status is not None:
+ clauses.append("status = ?")
+ params.append(status)
+ if agent is not None:
+ clauses.append("assigned_to = ?")
+ params.append(agent)
+ if pipeline_id is not None:
+ clauses.append("pipeline_id = ?")
+ params.append(pipeline_id)
+ where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
+ params.append(limit)
+ with self.connect() as conn:
+ rows = conn.execute(
+ f"SELECT id, status, claim_epoch FROM in_flight_tasks{where} LIMIT ?",
+ params,
+ ).fetchall()
+ retried: list[str] = []
+ for r in rows:
+ new_epoch = (r["claim_epoch"] or 0) + 1
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET previous_status = status, status = 'pending',
+ retry_count = 0, claim_epoch = ?, updated_at = ?
+ WHERE id = ?
+ """,
+ (new_epoch, now, r["id"]),
+ )
+ conn.execute("DELETE FROM claim_locks WHERE task_id = ?", (r["id"],))
+ self._append_event_inline(
+ conn, task_id=r["id"], event_type="retry",
+ agent=acting_agent or agent,
+ details={"previous_status": r["status"], "claim_epoch": new_epoch, "bulk": True},
+ now=now,
+ )
+ retried.append(r["id"])
+ return {"retried": retried, "count": len(retried), "skipped": 0, "ids": retried}
+
+ def set_task_obsolete(
+ self,
+ *,
+ task_id: str,
+ agent: str | None = None,
+ reason: str | None = None,
+ acting_agent: str | None = None,
+ ) -> dict[str, Any]:
+ now = utc_ms()
+ with self.connect() as conn:
+ row = conn.execute(
+ "SELECT status FROM in_flight_tasks WHERE id = ? AND (assigned_to = ? OR ? IS NULL)",
+ (task_id, agent, agent),
+ ).fetchone()
+ if row is None:
+ raise ValueError(f"task {task_id} not found (or assigned_to mismatch)")
+ prev = row["status"]
+ if prev not in ("PENDING", "FAILED", "ORPHANED", "pending", "failed"):
+ raise ValueError(
+ f"task {task_id} cannot transition to OBSOLETE from {prev!r}"
+ )
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET previous_status = ?, status = 'OBSOLETE',
+ obsolete_reason = ?, obsolete_by = ?, obsolete_at = ?,
+ updated_at = ?
+ WHERE id = ?
+ """,
+ (prev, reason, acting_agent, now, now, task_id),
+ )
+ self._append_event_inline(
+ conn, task_id=task_id, event_type="status_change",
+ agent=acting_agent or agent,
+ details={"previous_status": prev, "status": "OBSOLETE", "reason": reason},
+ now=now,
+ )
+ return {"task_id": task_id, "previous_status": prev, "status": "OBSOLETE"}
+
+ def revert_task_status(
+ self,
+ *,
+ task_id: str,
+ agent: str | None = None,
+ ) -> dict[str, Any]:
+ now = utc_ms()
+ with self.connect() as conn:
+ row = conn.execute(
+ "SELECT status FROM in_flight_tasks WHERE id = ?", (task_id,)
+ ).fetchone()
+ if row is None:
+ raise ValueError(f"task {task_id} not found")
+ cur_status = row["status"]
+ event = conn.execute(
+ """
+ SELECT details_json FROM task_events
+ WHERE task_id = ? AND event_type = 'status_change'
+ ORDER BY occurred_at DESC, created_at DESC LIMIT 1
+ """,
+ (task_id,),
+ ).fetchone()
+ if event is None:
+ raise ValueError(f"no prior status_change event for task {task_id}")
+ details = json.loads(event["details_json"]) if event["details_json"] else {}
+ prev = details.get("previous_status")
+ if not prev:
+ raise ValueError(f"prior status_change event for task {task_id} has no previous_status")
+ self._validate_status(prev)
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET previous_status = ?, status = ?, updated_at = ?
+ WHERE id = ?
+ """,
+ (cur_status, prev, now, task_id),
+ )
+ self._append_event_inline(
+ conn, task_id=task_id, event_type="status_change", agent=agent,
+ details={"previous_status": cur_status, "status": prev, "reason": "revert"},
+ now=now,
+ )
+ return {"task_id": task_id, "status": prev, "reverted_from": cur_status}
+
+ def revert_task_prompt(
+ self,
+ *,
+ task_id: str,
+ agent: str | None = None,
+ ) -> dict[str, Any]:
+ now = utc_ms()
+ with self.connect() as conn:
+ row = conn.execute(
+ "SELECT prompt FROM in_flight_tasks WHERE id = ?", (task_id,)
+ ).fetchone()
+ if row is None:
+ raise ValueError(f"task {task_id} not found")
+ cur_prompt = row["prompt"]
+ event = conn.execute(
+ """
+ SELECT details_json FROM task_events
+ WHERE task_id = ? AND event_type = 'prompt_change'
+ ORDER BY occurred_at DESC, created_at DESC LIMIT 1
+ """,
+ (task_id,),
+ ).fetchone()
+ if event is None:
+ raise ValueError(f"no prior prompt_change event for task {task_id}")
+ details = json.loads(event["details_json"]) if event["details_json"] else {}
+ prev = details.get("previous_prompt")
+ if prev is None:
+ raise ValueError(f"prior prompt_change event for task {task_id} has no previous_prompt")
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET previous_prompt = ?, prompt = ?, updated_at = ?
+ WHERE id = ?
+ """,
+ (cur_prompt, prev, now, task_id),
+ )
+ self._append_event_inline(
+ conn, task_id=task_id, event_type="prompt_change", agent=agent,
+ details={"previous_prompt": cur_prompt, "prompt": prev, "reason": "revert"},
+ now=now,
+ )
+ return {"task_id": task_id, "prompt": prev}
+
+ def reassign_task(
+ self,
+ *,
+ task_id: str,
+ new_agent: str,
+ prev_agent: str | None = None,
+ reason: str | None = None,
+ acting_agent: str | None = None,
+ ) -> dict[str, Any]:
+ now = utc_ms()
+ with self.connect() as conn:
+ row = conn.execute(
+ "SELECT assigned_to, claim_epoch, claimed_by FROM in_flight_tasks WHERE id = ?",
+ (task_id,),
+ ).fetchone()
+ if row is None:
+ raise ValueError(f"task {task_id} not found")
+ actual_prev = row["assigned_to"]
+ if prev_agent is not None and actual_prev != prev_agent:
+ raise ValueError(
+ f"prev_agent mismatch: db has {actual_prev!r}, caller said {prev_agent!r}"
+ )
+ new_epoch = (row["claim_epoch"] or 0) + 1
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET previous_agent = ?, assigned_to = ?, reassigned_from = ?,
+ reassign_reason = ?, reassigned_by = ?, reassigned_at = ?,
+ claim_epoch = ?, updated_at = ?
+ WHERE id = ?
+ """,
+ (
+ actual_prev, new_agent, actual_prev, reason, acting_agent, now,
+ new_epoch, now, task_id,
+ ),
+ )
+ if row["claimed_by"] == actual_prev:
+ conn.execute("DELETE FROM claim_locks WHERE task_id = ?", (task_id,))
+ conn.execute(
+ "UPDATE in_flight_tasks SET claimed_by = NULL, claimed_at = NULL, active_claim_token = NULL WHERE id = ?",
+ (task_id,),
+ )
+ self._append_event_inline(
+ conn, task_id=task_id, event_type="reassign",
+ agent=acting_agent,
+ details={
+ "previous_agent": actual_prev, "new_agent": new_agent,
+ "claim_epoch": new_epoch, "reason": reason,
+ },
+ now=now,
+ )
+ return {"task_id": task_id, "previous_agent": actual_prev, "new_agent": new_agent}
+
+ def update_task_prompt(
+ self,
+ *,
+ task_id: str,
+ new_prompt: str,
+ agent: str | None = None,
+ reason: str | None = None,
+ acting_agent: str | None = None,
+ ) -> dict[str, Any]:
+ now = utc_ms()
+ with self.connect() as conn:
+ row = conn.execute(
+ "SELECT prompt, original_prompt, assigned_to FROM in_flight_tasks WHERE id = ?",
+ (task_id,),
+ ).fetchone()
+ if row is None:
+ raise ValueError(f"task {task_id} not found")
+ if agent is not None and row["assigned_to"] != agent:
+ raise ValueError(
+ f"task {task_id} not assigned to {agent!r} (assigned_to={row['assigned_to']!r})"
+ )
+ prev_prompt = row["prompt"]
+ original = row["original_prompt"] if row["original_prompt"] is not None else prev_prompt
+ self._append_event_inline(
+ conn, task_id=task_id, event_type="prompt_change",
+ agent=acting_agent or agent,
+ details={"previous_prompt": prev_prompt, "prompt": new_prompt, "reason": reason},
+ now=now,
+ )
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET previous_prompt = ?, prompt = ?, original_prompt = ?,
+ rewrite_reason = ?, rewrite_by = ?, rewritten_at = ?,
+ updated_at = ?
+ WHERE id = ?
+ """,
+ (prev_prompt, new_prompt, original, reason, acting_agent, now, now, task_id),
+ )
+ return {"task_id": task_id, "previous_prompt": prev_prompt}
+
+ def delete_task(
+ self,
+ *,
+ task_id: str,
+ agent: str | None = None,
+ ) -> dict[str, Any]:
+ now = utc_ms()
+ cascaded = 0
+ with self.connect() as conn:
+ row = conn.execute(
+ "SELECT id, assigned_to FROM in_flight_tasks WHERE id = ?",
+ (task_id,),
+ ).fetchone()
+ if row is None:
+ raise ValueError(f"task {task_id} not found")
+ if agent is not None and row["assigned_to"] != agent:
+ raise ValueError(
+ f"task {task_id} not assigned to {agent!r} (assigned_to={row['assigned_to']!r})"
+ )
+ for table in ("task_events", "task_outputs", "task_outcomes", "failure_reports"):
+ cur = conn.execute(f"DELETE FROM {table} WHERE task_id = ?", (task_id,))
+ cascaded += cur.rowcount or 0
+ cur = conn.execute("DELETE FROM claim_locks WHERE task_id = ?", (task_id,))
+ cascaded += cur.rowcount or 0
+ cur = conn.execute("DELETE FROM in_flight_tasks WHERE id = ?", (task_id,))
+ cascaded += cur.rowcount or 0
+ conn.execute(
+ """
+ INSERT INTO audit_events (id, trace_id, actor, action, decision, resource, details_json, created_at)
+ VALUES (?, NULL, ?, 'task.delete', 'allow', ?, ?, ?)
+ """,
+ (
+ str(uuid.uuid4()), agent or "system", task_id,
+ json.dumps({"task_id": task_id, "cascaded_rows": cascaded}, sort_keys=True),
+ now,
+ ),
+ )
+ return {"deleted": True, "cascaded_rows": cascaded}
+
+ def pause_task(
+ self,
+ *,
+ task_id: str,
+ agent: str | None = None,
+ ) -> dict[str, Any]:
+ now = utc_ms()
+ with self.connect() as conn:
+ row = conn.execute(
+ "SELECT status, paused FROM in_flight_tasks WHERE id = ?", (task_id,)
+ ).fetchone()
+ if row is None:
+ raise ValueError(f"task {task_id} not found")
+ if row["paused"]:
+ return {"task_id": task_id, "paused": True, "already_paused": True}
+ if row["status"] not in ("PENDING", "pending"):
+ raise ValueError(
+ f"task {task_id} not pending (status={row['status']!r}); can only pause PENDING tasks"
+ )
+ conn.execute(
+ "UPDATE in_flight_tasks SET paused = 1, paused_at = ?, updated_at = ? WHERE id = ?",
+ (now, now, task_id),
+ )
+ self._append_event_inline(
+ conn, task_id=task_id, event_type="pause", agent=agent,
+ details={}, now=now,
+ )
+ return {"task_id": task_id, "paused": True}
+
+ def unpause_task(
+ self,
+ *,
+ task_id: str,
+ agent: str | None = None,
+ ) -> dict[str, Any]:
+ now = utc_ms()
+ with self.connect() as conn:
+ row = conn.execute(
+ "SELECT paused FROM in_flight_tasks WHERE id = ?", (task_id,)
+ ).fetchone()
+ if row is None:
+ raise ValueError(f"task {task_id} not found")
+ if not row["paused"]:
+ raise ValueError(f"task {task_id} is not paused")
+ conn.execute(
+ "UPDATE in_flight_tasks SET paused = 0, paused_at = NULL, updated_at = ? WHERE id = ?",
+ (now, task_id),
+ )
+ self._append_event_inline(
+ conn, task_id=task_id, event_type="unpause", agent=agent,
+ details={}, now=now,
+ )
+ return {"task_id": task_id, "paused": False}
+
+ def reorder_tasks(
+ self,
+ *,
+ task_ids: list[str],
+ sort_orders: list[int] | None = None,
+ agent: str | None = None,
+ ) -> dict[str, Any]:
+ """Bulk SET sort_order. If sort_orders omitted, assigns descending order
+ (first task gets highest, last gets 1) so the first id appears at the top
+ of a DESC-by-sort_order Kanban view."""
+ now = utc_ms()
+ if sort_orders is None:
+ sort_orders = list(range(len(task_ids), 0, -1))
+ if len(sort_orders) != len(task_ids):
+ raise ValueError("task_ids and sort_orders must be the same length")
+ updated = 0
+ with self.connect() as conn:
+ for tid, order in zip(task_ids, sort_orders):
+ cur = conn.execute(
+ "UPDATE in_flight_tasks SET sort_order = ?, updated_at = ? WHERE id = ?",
+ (order, now, tid),
+ )
+ if cur.rowcount:
+ updated += 1
+ self._append_event_inline(
+ conn, task_id=tid, event_type="reorder", agent=agent,
+ details={"sort_order": order}, now=now,
+ )
+ return {"updated": updated, "task_ids": list(task_ids)}
+
+ def move_task_to_top(
+ self,
+ *,
+ task_id: str,
+ agent: str | None = None,
+ ) -> dict[str, Any]:
+ now = utc_ms()
+ with self.connect() as conn:
+ row = conn.execute(
+ "SELECT id FROM in_flight_tasks WHERE id = ?", (task_id,)
+ ).fetchone()
+ if row is None:
+ raise ValueError(f"task {task_id} not found")
+ max_row = conn.execute(
+ "SELECT max(coalesce(sort_order, 0)) AS m FROM in_flight_tasks"
+ ).fetchone()
+ new_order = (int(max_row["m"]) if max_row and max_row["m"] is not None else 0) + 1
+ conn.execute(
+ "UPDATE in_flight_tasks SET sort_order = ?, updated_at = ? WHERE id = ?",
+ (new_order, now, task_id),
+ )
+ self._append_event_inline(
+ conn, task_id=task_id, event_type="reorder", agent=agent,
+ details={"sort_order": new_order, "move_to_top": True}, now=now,
+ )
+ return {"task_id": task_id, "sort_order": new_order}
+
+ def bulk_reassign_tasks(
+ self,
+ *,
+ task_ids: list[str],
+ new_agent: str,
+ prev_agent: str | None = None,
+ reason: str | None = None,
+ acting_agent: str | None = None,
+ ) -> dict[str, Any]:
+ moved: list[str] = []
+ skipped: list[str] = []
+ for tid in task_ids:
+ try:
+ self.reassign_task(
+ task_id=tid, new_agent=new_agent,
+ prev_agent=prev_agent, reason=reason, acting_agent=acting_agent,
+ )
+ moved.append(tid)
+ except ValueError:
+ skipped.append(tid)
+ return {"moved": moved, "skipped": skipped, "count": len(moved)}
+
+ def get_task_with_output(
+ self,
+ *,
+ task_id: str,
+ events_limit: int = 20,
+ failures_limit: int = 10,
+ ) -> dict[str, Any] | None:
+ with self.connect() as conn:
+ row = conn.execute(
+ "SELECT * FROM in_flight_tasks WHERE id = ?", (task_id,)
+ ).fetchone()
+ if row is None:
+ return None
+ task = dict(row)
+ outputs = [
+ self._decode_json_fields(dict(r))
+ for r in conn.execute(
+ "SELECT * FROM task_outputs WHERE task_id = ? ORDER BY created_at DESC",
+ (task_id,),
+ ).fetchall()
+ ]
+ outcomes = [
+ self._decode_json_fields(dict(r))
+ for r in conn.execute(
+ "SELECT * FROM task_outcomes WHERE task_id = ? ORDER BY occurred_at DESC",
+ (task_id,),
+ ).fetchall()
+ ]
+ events = [
+ self._decode_json_fields(dict(r))
+ for r in conn.execute(
+ "SELECT * FROM task_events WHERE task_id = ? ORDER BY occurred_at DESC LIMIT ?",
+ (task_id, events_limit),
+ ).fetchall()
+ ]
+ failure_reports = [
+ self._decode_json_fields(dict(r))
+ for r in conn.execute(
+ "SELECT * FROM failure_reports WHERE task_id = ? ORDER BY occurred_at DESC LIMIT ?",
+ (task_id, failures_limit),
+ ).fetchall()
+ ]
+ lock = conn.execute(
+ "SELECT * FROM claim_locks WHERE task_id = ?", (task_id,)
+ ).fetchone()
+ task["claim_lock"] = dict(lock) if lock is not None else None
+ return {
+ "task": task,
+ "outputs": outputs,
+ "outcomes": outcomes,
+ "events": events,
+ "failure_reports": failure_reports,
+ }
+
+ def backup_to(self, destination: str | Path) -> Path:
+ destination = Path(destination)
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ with self.connect() as src, sqlite3.connect(destination) as dst:
+ src.backup(dst)
+ return destination
+
+ def _ensure_column(self, conn: sqlite3.Connection, table: str, column: str, definition: str) -> None:
+ columns = {row["name"] for row in conn.execute(f"PRAGMA table_info({table})").fetchall()}
+ if column not in columns:
+ conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} {definition}")
+
+ def _trace_data(self, trace: Any | None, overrides: dict[str, Any]) -> dict[str, Any]:
+ if trace is None:
+ data: dict[str, Any] = {}
+ elif is_dataclass(trace):
+ data = asdict(trace)
+ elif isinstance(trace, dict):
+ data = dict(trace)
+ else:
+ data = {key: getattr(trace, key) for key in dir(trace) if not key.startswith("_") and not callable(getattr(trace, key))}
+ data.update({key: value for key, value in overrides.items() if value is not None})
+ if "actor" not in data or "operation" not in data:
+ raise ValueError("trace requires actor and operation")
+ return data
+
+ def _decode_json_fields(self, row: dict[str, Any]) -> dict[str, Any]:
+ for key in list(row):
+ if key.endswith("_json"):
+ value = row.pop(key)
+ row[key[:-5]] = json.loads(value) if value else {}
+ return row
+
+ def _sweep_expired_claims(self, conn: sqlite3.Connection, now: int) -> int:
+ rows = conn.execute(
+ "SELECT task_id FROM claim_locks WHERE expires_at <= ?",
+ (now,),
+ ).fetchall()
+ for row in rows:
+ conn.execute("DELETE FROM claim_locks WHERE task_id = ?", (row["task_id"],))
+ conn.execute(
+ """
+ UPDATE in_flight_tasks
+ SET status = 'pending',
+ active_claim_token = NULL,
+ updated_at = ?
+ WHERE id = ? AND status = 'in_progress'
+ """,
+ (now, row["task_id"]),
+ )
+ return len(rows)
+
+ def _valid_lock(
+ self,
+ conn: sqlite3.Connection,
+ task_id: str,
+ agent: str,
+ claim_token: str,
+ now: int,
+ ) -> sqlite3.Row:
+ lock = conn.execute(
+ """
+ SELECT * FROM claim_locks
+ WHERE task_id = ? AND claimed_by = ? AND claim_token = ?
+ """,
+ (task_id, agent, claim_token),
+ ).fetchone()
+ if lock is None or int(lock["expires_at"]) <= now:
+ raise StaleClaimError(f"stale claim for task {task_id}")
+ return lock
diff --git a/kublai/tracing.py b/kublai/tracing.py
new file mode 100644
index 00000000..df29ced3
--- /dev/null
+++ b/kublai/tracing.py
@@ -0,0 +1,45 @@
+"""Lightweight trace helpers for Kublai production-agent flows."""
+
+from __future__ import annotations
+
+import time
+import uuid
+from dataclasses import dataclass, field
+from typing import Any
+
+
+def utc_ms() -> int:
+ """Return the current Unix time in milliseconds (UTC)."""
+ return int(time.time() * 1000)
+
+
+def new_trace_id() -> str:
+ """Create a sortable-enough opaque trace id."""
+ return f"trace-{uuid.uuid4().hex}"
+
+
+def new_span_id() -> str:
+ return f"span-{uuid.uuid4().hex}"
+
+
+@dataclass(frozen=True)
+class TraceContext:
+ """Minimal trace context shared across telemetry, audit, and task rows."""
+
+ actor: str
+ operation: str
+ resource: str | None = None
+ trace_id: str = field(default_factory=new_trace_id)
+ parent_span_id: str | None = None
+ started_at: int = field(default_factory=utc_ms)
+ metadata: dict[str, Any] = field(default_factory=dict)
+
+
+def trace_event(event_type: str, *, span_id: str | None = None, payload: dict[str, Any] | None = None) -> dict[str, Any]:
+ """Build a JSON-serializable trace event payload."""
+ return {
+ "event_type": event_type,
+ "span_id": span_id or new_span_id(),
+ "payload": payload or {},
+ "created_at": utc_ms(),
+ }
diff --git a/kublai/v4.py b/kublai/v4.py
new file mode 100644
index 00000000..50373a21
--- /dev/null
+++ b/kublai/v4.py
@@ -0,0 +1,479 @@
+"""Kublai brain v4 public workflow surface.
+
+The v4 layer is intentionally thin: it reuses the existing SQLite indexes,
+wiki writer, sanitizer, and telemetry store, while making privacy decisions
+explicit at every public boundary.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import time
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from .knowledge import KnowledgeStore, normalize_body, slugify
+from .sanitizer import CANARY_RE, DEFAULT_SANITIZER, SanitizerContext
+
+HARD_PRIVATE_CANARY = "KUBLAI_HARD_PRIVATE_CANARY_SYNTHETIC_DO_NOT_LEAK"
+DEFAULT_PUBLIC_ROOT = Path(os.environ.get("KUBLAI_BRAIN_PUBLIC_ROOT", str(Path.home() / "brain-public")))
+COMMAND_REGISTRY = (
+ "wiki",
+ "save",
+ "ask",
+ "capture",
+ "ingest",
+ "process-inbox",
+ "connect",
+ "brief",
+ "write",
+ "publish",
+ "research",
+ "lint",
+)
+SECRET_PATTERNS = (
+ re.compile(r"(?i)\b(api[_-]?key|secret|token|password)\s*[:=]\s*['\"]?[A-Za-z0-9_\-]{16,}"),
+ re.compile(r"\bsk-[A-Za-z0-9]{20,}\b"),
+ re.compile(r"\bage-secret-key-[A-Za-z0-9]+\b"),
+)
+
+
+class V4WorkflowError(ValueError):
+ """Raised for v4 workflow validation errors."""
+
+
+class V4PrivacyError(V4WorkflowError):
+ """Raised when a request tries to cross a privacy boundary."""
+
+
+def utc_date() -> str:
+ return datetime.now(timezone.utc).date().isoformat()
+
+
+def split_frontmatter(text: str) -> tuple[dict[str, Any], str]:
+ if not text.startswith("---\n"):
+ return {}, text
+ end = text.find("\n---\n", 4)
+ if end < 0:
+ return {}, text
+ parsed = yaml.safe_load(text[4:end]) or {}
+ return (parsed if isinstance(parsed, dict) else {}), text[end + 5 :]
+
+
+def _scrub_public(content: str, *, context: SanitizerContext) -> tuple[str, list[str]]:
+ return DEFAULT_SANITIZER.scrub(content, target_class="public", context=context)
+
+
+def _summary(text: str, *, chars: int = 480) -> str:
+ squashed = re.sub(r"\s+", " ", text).strip()
+ return squashed[:chars]
+
+
+class V4WorkflowService:
+ def __init__(self, service: Any):
+ self.service = service
+ self.wiki_root = Path(service.wiki_root)
+ self.manifest_path = self.wiki_root / "raw" / "v4-source-manifest.jsonl"
+ self.graphify_queue_path = self.wiki_root / "raw" / "graphify-queue.jsonl"
+
+ def command_registry(self) -> list[str]:
+ return list(COMMAND_REGISTRY)
+
+ def public_search(self, *, query: str, limit: int = 10) -> list[dict[str, Any]]:
+ rows = self.service.index.search(query=query, limit=limit)
+ safe_rows: list[dict[str, Any]] = []
+ for row in rows:
+ fm = row.get("frontmatter") or {}
+ if DEFAULT_SANITIZER.classify(str(row.get("rel_path") or ""), fm) != "public":
+ continue
+ snippet, findings = _scrub_public(str(row.get("body_text") or ""), context=SanitizerContext.GATEWAY_RESPONSE)
+ safe_rows.append(
+ {
+ "rel_path": row["rel_path"],
+ "title": row.get("title") or row.get("id"),
+ "type": row.get("type"),
+ "updated": row.get("updated"),
+ "score": row.get("score"),
+ "snippet": _summary(snippet),
+ "sanitizer_findings": findings,
+ }
+ )
+ return safe_rows[:limit]
+
+ def public_get(self, *, rel_path: str) -> dict[str, Any]:
+ target = self._resolve_wiki_path(rel_path)
+ fm, body = split_frontmatter(target.read_text(encoding="utf-8", errors="ignore"))
+ if DEFAULT_SANITIZER.classify(rel_path, fm) != "public":
+ raise V4PrivacyError("HTTPS/public get only serves explicit-public pages")
+ safe_body, findings = _scrub_public(body, context=SanitizerContext.GATEWAY_RESPONSE)
+ return {
+ "rel_path": rel_path,
+ "frontmatter": self._public_frontmatter(fm),
+ "body": safe_body,
+ "sanitizer_findings": findings,
+ }
+
+ def public_pages(self, *, limit: int = 100) -> list[dict[str, Any]]:
+ return [
+ {
+ "rel_path": row["rel_path"],
+ "title": row.get("title") or row.get("id"),
+ "type": row.get("type"),
+ "updated": row.get("updated"),
+ }
+ for row in self.service.index.list_nodes(limit=limit)
+ if DEFAULT_SANITIZER.classify(row["rel_path"], row.get("frontmatter") or {}) == "public"
+ ]
+
+ def public_tags(self, *, limit: int = 200) -> list[dict[str, Any]]:
+ counts: dict[str, int] = {}
+ for row in self.service.index.list_nodes(limit=max(limit * 20, 500)):
+ fm = row.get("frontmatter") or {}
+ if DEFAULT_SANITIZER.classify(row["rel_path"], fm) != "public":
+ continue
+ tags = fm.get("tags") or []
+ if isinstance(tags, str):
+ tags = [tags]
+ for tag in tags:
+ counts[str(tag)] = counts.get(str(tag), 0) + 1
+ return [{"tag": tag, "count": count} for tag, count in sorted(counts.items())[:limit]]
+
+ def public_stub_rebuild(self) -> dict[str, Any]:
+ indexed = self.service.reindex()
+ stats = self.service.verify_index()
+ return {"indexed": indexed, "stats": stats}
+
+ def ask(self, *, query: str, privacy_scope: str = "public", limit: int = 5, request_id: str | None = None) -> dict[str, Any]:
+ self._require_public_scope(privacy_scope)
+ started = time.monotonic()
+ rows = self.public_search(query=query, limit=limit)
+ answer = "No public context matched that query."
+ if rows:
+ citations = ", ".join(f"[[{Path(row['rel_path']).stem}]]" for row in rows[:3])
+ answer = f"Public answer draft from indexed context. Relevant public pages: {citations}."
+ result = {"answer": answer, "citations": rows, "privacy_scope": "public"}
+ self._record_command("ask", privacy_scope="public", dry_run=True, source_count=len(rows), duration_s=time.monotonic() - started, request_id=request_id)
+ return result
+
+ def capture_dry_run(self, **params: Any) -> dict[str, Any]:
+ return self._capture_plan(kind="capture", dry_run=True, **params)
+
+ def capture_apply(self, **params: Any) -> dict[str, Any]:
+ plan = self._capture_plan(kind="capture", dry_run=False, **params)
+ self._write_generated_page(plan["rel_path"], plan["frontmatter"], plan["body"])
+ self._append_manifest(plan)
+ self._enqueue_graphify(plan["rel_path"], reason="capture.apply")
+ if plan["privacy_class"] == "public":
+ self.service.reindex()
+ self._record_command("capture", privacy_scope=plan["privacy_class"], dry_run=False, source_count=1, output_path=plan["rel_path"], request_id=params.get("request_id"))
+ plan["written"] = True
+ return plan
+
+ def ingest_dry_run(self, **params: Any) -> dict[str, Any]:
+ return self._capture_plan(kind="ingest", dry_run=True, **params)
+
+ def ingest_apply(self, **params: Any) -> dict[str, Any]:
+ plan = self._capture_plan(kind="ingest", dry_run=False, **params)
+ self._write_generated_page(plan["rel_path"], plan["frontmatter"], plan["body"])
+ self._append_manifest(plan)
+ self._enqueue_graphify(plan["rel_path"], reason="ingest.apply")
+ if plan["privacy_class"] == "public":
+ self.service.reindex()
+ self._record_command("ingest", privacy_scope=plan["privacy_class"], dry_run=False, source_count=1, output_path=plan["rel_path"], request_id=params.get("request_id"))
+ plan["written"] = True
+ return plan
+
+ def publish_dry_run(self, *, output_root: str | Path | None = None, request_id: str | None = None) -> dict[str, Any]:
+ started = time.monotonic()
+ output_root = Path(output_root) if output_root else DEFAULT_PUBLIC_ROOT
+ files: list[dict[str, Any]] = []
+ findings: list[dict[str, str]] = []
+ for md in sorted(self.wiki_root.rglob("*.md")):
+ if ".git" in md.parts:
+ continue
+ rel = md.relative_to(self.wiki_root).as_posix()
+ fm, body = split_frontmatter(md.read_text(encoding="utf-8", errors="ignore"))
+ if DEFAULT_SANITIZER.classify(rel, fm) != "public":
+ continue
+ safe_body, sanitizer_findings = _scrub_public(body, context=SanitizerContext.PUBLISH)
+ page_findings = self._publish_findings(rel, fm, body, sanitizer_findings)
+ findings.extend({"rel_path": rel, "finding": finding} for finding in page_findings)
+ files.append(
+ {
+ "rel_path": rel,
+ "output_path": str(output_root / rel),
+ "frontmatter": self._public_frontmatter(fm),
+ "body_hash": hashlib.sha256(safe_body.encode("utf-8")).hexdigest(),
+ "findings": page_findings,
+ }
+ )
+ result = {"ok": not findings, "dry_run": True, "output_root": str(output_root), "files": files, "findings": findings}
+ self._record_command("publish", privacy_scope="public", dry_run=True, source_count=len(files), duration_s=time.monotonic() - started, error_code=None if result["ok"] else "sanitizer_findings", request_id=request_id)
+ return result
+
+ def publish_apply(self, *, output_root: str | Path | None = None, request_id: str | None = None) -> dict[str, Any]:
+ dry = self.publish_dry_run(output_root=output_root, request_id=request_id)
+ if not dry["ok"]:
+ raise V4WorkflowError("publish.apply requires zero sanitizer findings")
+ output_root = Path(dry["output_root"])
+ for item in dry["files"]:
+ source = self._resolve_wiki_path(item["rel_path"])
+ fm, body = split_frontmatter(source.read_text(encoding="utf-8", errors="ignore"))
+ safe_body, _ = _scrub_public(body, context=SanitizerContext.PUBLISH)
+ target = output_root / item["rel_path"]
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_text(KnowledgeStore.render(self._public_frontmatter(fm), safe_body), encoding="utf-8")
+ self._record_command("publish", privacy_scope="public", dry_run=False, source_count=len(dry["files"]), output_path=str(output_root), request_id=request_id)
+ return dry | {"dry_run": False, "written": len(dry["files"])}
+
+ def research_public_dossier(self, *, query: str, lenses: list[str] | None = None, limit: int = 6, request_id: str | None = None) -> dict[str, Any]:
+ lenses = lenses or ["evidence", "risk", "architecture"]
+ contexts = []
+ for lens in lenses:
+ rows = self.public_search(query=f"{query} {lens}", limit=limit)
+ contexts.append({"lens": lens, "context_count": len(rows), "citations": rows})
+ dossier = {
+ "query": query,
+ "privacy_scope": "public",
+ "lenses": contexts,
+ "synthesis": "Public-only dossier assembled from explicit-public index rows.",
+ "contradictions": [],
+ "confidence": "medium" if any(item["context_count"] for item in contexts) else "low",
+ "needs_private_review": [],
+ }
+ self._record_command("research", privacy_scope="public", dry_run=True, source_count=sum(item["context_count"] for item in contexts), request_id=request_id)
+ return dossier
+
+ def write_dry_run(self, *, topic: str, request_id: str | None = None) -> dict[str, Any]:
+ exemplars = self._public_voice_exemplars()
+ rows = self.public_search(query=topic, limit=5)
+ body = (
+ f"# {topic}\n\n"
+ "This is a public-context draft. It uses only explicit-public wiki context "
+ "and public voice exemplars, and it needs human review before publication.\n"
+ )
+ lint = self._style_lint(body, citations=rows)
+ result = {"topic": topic, "draft": body, "exemplar_count": len(exemplars), "citations": rows, "lint": lint, "dry_run": True}
+ self._record_command("write", privacy_scope="public", dry_run=True, source_count=len(rows) + len(exemplars), request_id=request_id)
+ return result
+
+ def lint(self) -> dict[str, Any]:
+ issues: list[dict[str, str]] = []
+ counts = {"public": 0, "private": 0, "hard-private": 0}
+ for md in sorted(self.wiki_root.rglob("*.md")):
+ if ".git" in md.parts:
+ continue
+ rel = md.relative_to(self.wiki_root).as_posix()
+ fm, body = split_frontmatter(md.read_text(encoding="utf-8", errors="ignore"))
+ privacy = DEFAULT_SANITIZER.classify(rel, fm)
+ counts[privacy] = counts.get(privacy, 0) + 1
+ if rel not in {"index.md", "home.md", "log.md"} and not fm:
+ issues.append({"rel_path": rel, "issue": "missing_frontmatter"})
+ if privacy == "public":
+ for finding in self._publish_findings(rel, fm, body, []):
+ issues.append({"rel_path": rel, "issue": finding})
+ public_canary_hits = self.public_search(query=HARD_PRIVATE_CANARY, limit=3)
+ if public_canary_hits:
+ issues.append({"rel_path": "public-index", "issue": "hard_private_canary_public_index"})
+ return {"ok": not issues, "counts": counts, "issues": issues, "command_registry": self.command_registry()}
+
+ def audit_private_summary(self, *, limit: int = 20) -> dict[str, Any]:
+ path = Path(self.service.private_access_log)
+ if not path.exists():
+ return {"ok": True, "events": 0, "recent": []}
+ lines = [line for line in path.read_text(encoding="utf-8", errors="ignore").splitlines() if line.strip()]
+ recent: list[dict[str, Any]] = []
+ for line in lines[-limit:]:
+ try:
+ record = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ recent.append({"ts": record.get("ts"), "action": record.get("action"), "requester": record.get("requester"), "result_count": len(record.get("rel_paths") or [])})
+ return {"ok": True, "events": len(lines), "recent": recent}
+
+ def doctor_full(self) -> dict[str, Any]:
+ lint = self.lint()
+ public_stats = self.service.verify_index()
+ private_stats = self.service.verify_private_index()
+ graphify = {"exists": self.graphify_queue_path.exists(), "pending": self._jsonl_count(self.graphify_queue_path)}
+ private_audit = self.audit_private_summary(limit=1)
+ checks = {
+ "command_registry": set(COMMAND_REGISTRY).issubset(set(self.command_registry())),
+ "public_index_ok": bool(public_stats.get("ok")),
+ "private_index_ok": bool(private_stats.get("ok")),
+ "canary_absent_public": not self.public_search(query=HARD_PRIVATE_CANARY, limit=1),
+ "lint_boundary_clean": not any(
+ issue.get("issue") != "missing_frontmatter"
+ for issue in lint.get("issues", [])
+ ),
+ "graphify_queue_writable": self._jsonl_parent_writable(self.graphify_queue_path),
+ "private_audit_log_readable": bool(private_audit.get("ok")),
+ }
+ return {"ok": all(checks.values()), "checks": checks, "lint": lint, "public_index": public_stats, "private_index": private_stats, "graphify_queue": graphify}
+
+ def _capture_plan(self, *, kind: str, content: str, title: str | None = None, source: str | None = None, frontmatter: dict[str, Any] | None = None, dry_run: bool, privacy_scope: str | None = None, request_id: str | None = None) -> dict[str, Any]:
+ if not content.strip():
+ raise V4WorkflowError(f"{kind} requires content")
+ fm = dict(frontmatter or {})
+ date = utc_date()
+ title = title or f"{kind.title()} {date}"
+ fm.setdefault("type", "capture")
+ fm.setdefault("status", "pending-review")
+ fm.setdefault("created", date)
+ fm["updated"] = date
+ fm.setdefault("sources", 1)
+ fm.setdefault("tags", ["kublai", kind])
+ if source:
+ fm.setdefault("source_label", _summary(source, chars=120))
+ requested_scope = privacy_scope or DEFAULT_SANITIZER.classify("", fm)
+ privacy_class = DEFAULT_SANITIZER.classify("", fm)
+ if requested_scope in {"private", "hard-private"} and privacy_class == "public":
+ privacy_class = requested_scope
+ if privacy_class == "private":
+ privacy_class = "hard-private"
+ rel_path = self._generated_rel_path(kind, title, privacy_class)
+ body = (
+ f"# {title}\n\n"
+ f"{normalize_body(content)}\n\n"
+ "## Related\n\n"
+ "- [[brain-service]]\n"
+ "- [[kublai]]\n"
+ )
+ safe_body, findings = DEFAULT_SANITIZER.scrub(body, target_class="public" if privacy_class == "public" else "private", context=SanitizerContext.MCP_RESPONSE)
+ if findings and privacy_class == "public":
+ privacy_class = "hard-private"
+ rel_path = self._generated_rel_path(kind, title, privacy_class)
+ plan = {
+ "ok": True,
+ "kind": kind,
+ "dry_run": dry_run,
+ "privacy_class": privacy_class,
+ "rel_path": rel_path,
+ "frontmatter": fm,
+ "body": safe_body if privacy_class == "public" else body,
+ "source_count": 1 if source or content else 0,
+ "sanitizer_findings": findings,
+ "request_id": request_id or f"v4-{uuid.uuid4().hex}",
+ "written": False,
+ }
+ self._record_command(kind, privacy_scope=privacy_class, dry_run=dry_run, source_count=plan["source_count"], output_path=rel_path, request_id=plan["request_id"])
+ return plan
+
+ def _generated_rel_path(self, kind: str, title: str, privacy_class: str) -> str:
+ prefix = "hard-private/inbox" if privacy_class == "hard-private" else "captures"
+ return f"{prefix}/{utc_date()}-{slugify(title)}.md"
+
+ def _write_generated_page(self, rel_path: str, frontmatter: dict[str, Any], body: str) -> None:
+ target = self._resolve_wiki_path(rel_path, must_exist=False)
+ target.parent.mkdir(parents=True, exist_ok=True)
+ self.service.knowledge.atomic_write(target, KnowledgeStore.render(frontmatter, body))
+
+ def _append_manifest(self, plan: dict[str, Any]) -> None:
+ self.manifest_path.parent.mkdir(parents=True, exist_ok=True)
+ record = {
+ "ts": datetime.now(timezone.utc).isoformat(),
+ "kind": plan["kind"],
+ "rel_path": plan["rel_path"],
+ "privacy_class": plan["privacy_class"],
+ "body_hash": hashlib.sha256(plan["body"].encode("utf-8")).hexdigest(),
+ "request_id": plan.get("request_id"),
+ }
+ with self.manifest_path.open("a", encoding="utf-8") as handle:
+ handle.write(json.dumps(record, sort_keys=True) + "\n")
+
+ def _enqueue_graphify(self, rel_path: str, *, reason: str) -> None:
+ self.graphify_queue_path.parent.mkdir(parents=True, exist_ok=True)
+ record = {"ts": datetime.now(timezone.utc).isoformat(), "rel_path": rel_path, "reason": reason}
+ with self.graphify_queue_path.open("a", encoding="utf-8") as handle:
+ handle.write(json.dumps(record, sort_keys=True) + "\n")
+
+ def _resolve_wiki_path(self, rel_path: str, *, must_exist: bool = True) -> Path:
+ rel = Path(rel_path)
+ if rel.is_absolute() or ".." in rel.parts:
+ raise V4WorkflowError(f"invalid wiki path: {rel_path}")
+ target = (self.wiki_root / rel).resolve()
+ if self.wiki_root != target and self.wiki_root not in target.parents:
+ raise V4WorkflowError(f"path escapes wiki root: {rel_path}")
+ if must_exist and not target.exists():
+ raise FileNotFoundError(rel_path)
+ return target
+
+ def _public_frontmatter(self, frontmatter: dict[str, Any]) -> dict[str, Any]:
+ allowed = {"type", "title", "status", "created", "updated", "sources", "tags", "publish", "public_stub", "voice_exemplar"}
+ return {key: self._json_safe(value) for key, value in frontmatter.items() if key in allowed}
+
+ def _json_safe(self, value: Any) -> Any:
+ if isinstance(value, (str, int, float, bool)) or value is None:
+ return value
+ if isinstance(value, list):
+ return [self._json_safe(item) for item in value]
+ if isinstance(value, dict):
+ return {str(key): self._json_safe(item) for key, item in value.items()}
+ return str(value)
+
+ def _publish_findings(self, rel_path: str, frontmatter: dict[str, Any], body: str, sanitizer_findings: list[str]) -> list[str]:
+ findings = list(sanitizer_findings)
+ if DEFAULT_SANITIZER.classify(rel_path, frontmatter) != "public":
+ findings.append("not_explicit_public")
+ if CANARY_RE.search(body):
+ findings.append("hard_private_canary")
+ if "hard-private/" in body:
+ findings.append("hard_private_path")
+ for pattern in SECRET_PATTERNS:
+ if pattern.search(body):
+ findings.append("secret_like_value")
+ break
+ if not body.strip():
+ findings.append("empty_body")
+ return sorted(set(findings))
+
+ def _public_voice_exemplars(self) -> list[dict[str, Any]]:
+ exemplars = []
+ for md in sorted((self.wiki_root / "published").glob("*.md")) if (self.wiki_root / "published").exists() else []:
+ fm, body = split_frontmatter(md.read_text(encoding="utf-8", errors="ignore"))
+ rel = md.relative_to(self.wiki_root).as_posix()
+ if fm.get("voice_exemplar") is True and DEFAULT_SANITIZER.classify(rel, fm) == "public":
+ exemplars.append({"rel_path": rel, "title": fm.get("title") or md.stem, "excerpt": _summary(body)})
+ return exemplars
+
+ def _style_lint(self, body: str, *, citations: list[dict[str, Any]]) -> dict[str, Any]:
+ banned = ["synergy", "leverage", "game-changer"]
+ hits = [word for word in banned if re.search(rf"\b{re.escape(word)}\b", body, re.I)]
+ return {"ok": not hits and bool(citations), "banned_words": hits, "citation_count": len(citations)}
+
+ def _record_command(self, command: str, *, privacy_scope: str, dry_run: bool, source_count: int, duration_s: float | None = None, output_path: str | None = None, error_code: str | None = None, request_id: str | None = None) -> None:
+ details = {
+ "command": command,
+ "privacy_class": privacy_scope,
+ "dry_run": dry_run,
+ "source_count": source_count,
+ "output_path": output_path,
+ "duration_ms": int((duration_s or 0) * 1000),
+ "error_code": error_code,
+ "llm_destination_class": "external-public-only" if privacy_scope == "public" else "local-only",
+ "request_id": request_id,
+ }
+ try:
+ self.service.telemetry.record_audit_event(actor="kublai.v4", action=f"command.{command}", decision="apply" if not dry_run else "dry_run", details=details, resource=output_path)
+ except Exception:
+ pass
+
+ def _require_public_scope(self, privacy_scope: str) -> None:
+ if privacy_scope != "public":
+ raise V4PrivacyError("external/public workflow accepts only privacy_scope=public")
+
+ def _jsonl_count(self, path: Path) -> int:
+ if not path.exists():
+ return 0
+ return sum(1 for line in path.read_text(encoding="utf-8", errors="ignore").splitlines() if line.strip())
+
+ def _jsonl_parent_writable(self, path: Path) -> bool:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ return os.access(path.parent, os.W_OK)
diff --git a/kublai/v4_cli.py b/kublai/v4_cli.py
new file mode 100644
index 00000000..37579e2a
--- /dev/null
+++ b/kublai/v4_cli.py
@@ -0,0 +1,130 @@
+"""Command-line facade for Kublai brain v4 workflows."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+from .brain_service import BrainService
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(prog="kublai-v4")
+ parser.add_argument("--wiki-root", default=os.getenv("BRAIN_WIKI_ROOT", str(Path.home() / "brain")))
+ parser.add_argument("--telemetry-db", default=os.getenv("KUBLAI_TELEMETRY_DB", str(Path.home() / ".kublai/telemetry.db")))
+ parser.add_argument("--index-db", default=os.getenv("BRAIN_INDEX_DB", str(Path.home() / ".brain-index/brain.db")))
+ parser.add_argument("--json", action="store_true")
+ parser.add_argument("--request-id")
+ sub = parser.add_subparsers(dest="command", required=True)
+
+ ask = sub.add_parser("ask")
+ ask.add_argument("query")
+ ask.add_argument("--privacy-scope", default="public", choices=["public", "private", "hard-private"])
+
+ for name in ("capture", "ingest", "save"):
+ cmd = sub.add_parser(name)
+ cmd.add_argument("--content", required=True)
+ cmd.add_argument("--title")
+ cmd.add_argument("--source")
+ cmd.add_argument("--publish", action="store_true")
+ cmd.add_argument("--public-stub", action="store_true")
+ cmd.add_argument("--privacy-scope", default=None, choices=["public", "private", "hard-private"])
+ cmd.add_argument("--apply", action="store_true")
+ cmd.add_argument("--dry-run", action="store_true")
+
+ publish = sub.add_parser("publish")
+ publish.add_argument("--output-root")
+ publish.add_argument("--apply", action="store_true")
+ publish.add_argument("--dry-run", action="store_true")
+ publish.add_argument("--privacy-scope", default="public", choices=["public", "private", "hard-private"])
+
+ research = sub.add_parser("research")
+ research.add_argument("query")
+ research.add_argument("--privacy-scope", default="public", choices=["public", "private", "hard-private"])
+
+ write = sub.add_parser("write")
+ write.add_argument("topic")
+ write.add_argument("--privacy-scope", default="public", choices=["public", "private", "hard-private"])
+
+ wiki = sub.add_parser("wiki")
+ wiki.add_argument("--query", default="")
+ wiki.add_argument("--rel-path")
+ wiki.add_argument("--privacy-scope", default="public", choices=["public", "private", "hard-private"])
+
+ sub.add_parser("process-inbox")
+ sub.add_parser("connect")
+ sub.add_parser("brief")
+ sub.add_parser("lint")
+ return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+ args = build_parser().parse_args(argv)
+ service = BrainService(args.wiki_root, args.telemetry_db, args.index_db)
+ try:
+ result = dispatch(service, args)
+ if args.json:
+ print(json.dumps(result, sort_keys=True))
+ else:
+ print(json.dumps(result, indent=2, sort_keys=True))
+ return 0
+ except Exception as exc:
+ payload = {"ok": False, "error": type(exc).__name__, "message": str(exc)}
+ print(json.dumps(payload, sort_keys=True), file=sys.stderr)
+ return 1
+
+
+def dispatch(service: BrainService, args: argparse.Namespace) -> dict[str, Any]:
+ request_id = args.request_id
+ if args.command == "ask":
+ return {"ok": True, "result": service.v4.ask(query=args.query, privacy_scope=args.privacy_scope, request_id=request_id)}
+ if args.command in {"capture", "ingest", "save"}:
+ method = "capture" if args.command == "save" else args.command
+ frontmatter: dict[str, Any] = {}
+ if args.publish:
+ frontmatter["publish"] = True
+ if args.public_stub:
+ frontmatter["public_stub"] = True
+ params = {
+ "content": args.content,
+ "title": args.title,
+ "source": args.source,
+ "frontmatter": frontmatter or None,
+ "privacy_scope": args.privacy_scope,
+ "request_id": request_id,
+ }
+ if args.apply:
+ return {"ok": True, "result": getattr(service.v4, f"{method}_apply")(**params)}
+ return {"ok": True, "result": getattr(service.v4, f"{method}_dry_run")(**params)}
+ if args.command == "publish":
+ if args.privacy_scope != "public":
+ raise ValueError("publish is public-only")
+ params = {"output_root": args.output_root, "request_id": request_id}
+ return {"ok": True, "result": service.v4.publish_apply(**params) if args.apply else service.v4.publish_dry_run(**params)}
+ if args.command == "research":
+ if args.privacy_scope != "public":
+ raise ValueError("research external/RSG surface is public-only")
+ return {"ok": True, "result": service.v4.research_public_dossier(query=args.query, request_id=request_id)}
+ if args.command == "write":
+ if args.privacy_scope != "public":
+ raise ValueError("write uses public exemplars and public context only")
+ return {"ok": True, "result": service.v4.write_dry_run(topic=args.topic, request_id=request_id)}
+ if args.command == "wiki":
+ if args.privacy_scope != "public":
+ raise ValueError("wiki CLI defaults to public-safe retrieval")
+ if args.rel_path:
+ return {"ok": True, "result": service.v4.public_get(rel_path=args.rel_path)}
+ return {"ok": True, "result": service.v4.public_search(query=args.query, limit=10) if args.query else service.v4.public_pages()}
+ if args.command in {"process-inbox", "connect", "brief"}:
+ return {"ok": True, "result": {"dry_run": True, "command": args.command, "proposal_required": True, "message": "Workflow registered; mutating actions require explicit --apply through capture/ingest/write/publish surfaces."}}
+ if args.command == "lint":
+ return {"ok": True, "result": service.v4.lint()}
+ raise ValueError(f"unknown command: {args.command}")
+
+
+if __name__ == "__main__":
+ raise SystemExit(main(sys.argv[1:]))
diff --git a/kublai/v4_gateway.py b/kublai/v4_gateway.py
new file mode 100644
index 00000000..ee8e7ab3
--- /dev/null
+++ b/kublai/v4_gateway.py
@@ -0,0 +1,187 @@
+"""Public-only HTTP gateway for Kublai brain v4."""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import os
+import ssl
+import time
+import uuid
+from ipaddress import ip_address
+from http import HTTPStatus
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from typing import Any
+from urllib.parse import parse_qs, urlparse
+
+from .v4 import V4PrivacyError, V4WorkflowError
+
+
+def sign_request(secret: str, method: str, path: str, body: bytes, timestamp: str) -> str:
+ message = b"\n".join([method.upper().encode(), path.encode(), body, timestamp.encode()])
+ return hmac.new(secret.encode(), message, hashlib.sha256).hexdigest()
+
+
+def openapi_spec() -> dict[str, Any]:
+ return {
+ "openapi": "3.1.0",
+ "info": {"title": "Kublai Brain Public Gateway", "version": "4.0"},
+ "paths": {
+ "/health": {"get": {"summary": "Gateway and brain-service health"}},
+ "/api/wiki/search": {"get": {"summary": "Search explicit-public wiki index"}},
+ "/api/wiki/pages": {"get": {"summary": "List or fetch explicit-public wiki pages"}},
+ "/api/tags": {"get": {"summary": "List explicit-public tags"}},
+ "/api/ask": {"post": {"summary": "Public-context question answering"}},
+ "/api/capture": {"post": {"summary": "Capture proposal; private flags rejected"}},
+ "/api/ingest/dry-run": {"post": {"summary": "Public-safe ingest dry run"}},
+ "/api/publish/dry-run": {"post": {"summary": "Public publish dry run"}},
+ },
+ }
+
+
+def contains_private_request(value: Any) -> bool:
+ if isinstance(value, dict):
+ for key, item in value.items():
+ key_l = str(key).lower()
+ if key_l in {"include_private", "private", "hard_private"} and bool(item):
+ return True
+ if key_l == "privacy_scope" and item != "public":
+ return True
+ if contains_private_request(item):
+ return True
+ if isinstance(value, list):
+ return any(contains_private_request(item) for item in value)
+ if isinstance(value, str) and "hard-private/" in value:
+ return True
+ return False
+
+
+def is_loopback_host(host: str) -> bool:
+ if host in {"localhost", "::1"}:
+ return True
+ try:
+ return ip_address(host).is_loopback
+ except ValueError:
+ return False
+
+
+class PublicGatewayHandler(BaseHTTPRequestHandler):
+ service: Any = None
+ hmac_secret: str | None = None
+ server_version = "KublaiBrainGateway/4.0"
+
+ def log_message(self, fmt: str, *args: Any) -> None:
+ return
+
+ def do_GET(self) -> None:
+ request_id = self.headers.get("X-Request-ID") or f"gw-{uuid.uuid4().hex}"
+ try:
+ parsed = urlparse(self.path)
+ params = {key: values[-1] for key, values in parse_qs(parsed.query).items()}
+ if parsed.path == "/health":
+ self._write_json({"ok": True, "request_id": request_id, "gateway": "public", "brain": self.service.health()})
+ return
+ if parsed.path == "/openapi.json":
+ self._write_json(openapi_spec())
+ return
+ self._require_auth("GET", parsed.path, b"")
+ if parsed.path == "/api/wiki/search":
+ self._write_json({"ok": True, "request_id": request_id, "result": self.service.v4.public_search(query=params.get("q", ""), limit=int(params.get("limit", 10)))})
+ return
+ if parsed.path == "/api/wiki/pages":
+ if "rel_path" in params:
+ self._write_json({"ok": True, "request_id": request_id, "result": self.service.v4.public_get(rel_path=params["rel_path"])})
+ else:
+ self._write_json({"ok": True, "request_id": request_id, "result": self.service.v4.public_pages(limit=int(params.get("limit", 100)))})
+ return
+ if parsed.path == "/api/tags":
+ self._write_json({"ok": True, "request_id": request_id, "result": self.service.v4.public_tags(limit=int(params.get("limit", 200)))})
+ return
+ self._write_error(HTTPStatus.NOT_FOUND, "not_found", "route not found", request_id)
+ except V4PrivacyError as exc:
+ self._write_error(HTTPStatus.FORBIDDEN, "privacy_boundary", str(exc), request_id)
+ except Exception as exc:
+ self._write_error(HTTPStatus.BAD_REQUEST, type(exc).__name__, str(exc), request_id)
+
+ def do_POST(self) -> None:
+ request_id = self.headers.get("X-Request-ID") or f"gw-{uuid.uuid4().hex}"
+ body = self.rfile.read(int(self.headers.get("Content-Length") or "0"))
+ try:
+ parsed = urlparse(self.path)
+ self._require_auth("POST", parsed.path, body)
+ payload = json.loads(body.decode("utf-8") or "{}")
+ if contains_private_request(payload):
+ raise V4PrivacyError("HTTPS gateway rejects private and hard-private requests")
+ if parsed.path == "/api/ask":
+ self._write_json({"ok": True, "request_id": request_id, "result": self.service.v4.ask(request_id=request_id, **payload)})
+ return
+ if parsed.path == "/api/capture":
+ self._write_json({"ok": True, "request_id": request_id, "result": self.service.v4.capture_dry_run(request_id=request_id, **payload)})
+ return
+ if parsed.path == "/api/ingest/dry-run":
+ self._write_json({"ok": True, "request_id": request_id, "result": self.service.v4.ingest_dry_run(request_id=request_id, **payload)})
+ return
+ if parsed.path == "/api/publish/dry-run":
+ self._write_json({"ok": True, "request_id": request_id, "result": self.service.v4.publish_dry_run(request_id=request_id, **payload)})
+ return
+ self._write_error(HTTPStatus.NOT_FOUND, "not_found", "route not found", request_id)
+ except V4PrivacyError as exc:
+ self._write_error(HTTPStatus.FORBIDDEN, "privacy_boundary", str(exc), request_id)
+ except (V4WorkflowError, ValueError, json.JSONDecodeError) as exc:
+ self._write_error(HTTPStatus.BAD_REQUEST, type(exc).__name__, str(exc), request_id)
+ except Exception as exc:
+ self._write_error(HTTPStatus.INTERNAL_SERVER_ERROR, type(exc).__name__, str(exc), request_id)
+
+ def _require_auth(self, method: str, path: str, body: bytes) -> None:
+ if not self.hmac_secret:
+ return
+ timestamp = self.headers.get("X-Kublai-Timestamp") or ""
+ signature = self.headers.get("X-Kublai-Signature") or ""
+ if not timestamp or not signature:
+ raise V4PrivacyError("missing HMAC authentication")
+ try:
+ if abs(time.time() - float(timestamp)) > 300:
+ raise V4PrivacyError("stale HMAC timestamp")
+ except ValueError as exc:
+ raise V4PrivacyError("invalid HMAC timestamp") from exc
+ expected = sign_request(self.hmac_secret, method, path, body, timestamp)
+ if not hmac.compare_digest(expected, signature):
+ raise V4PrivacyError("invalid HMAC signature")
+
+ def _write_json(self, payload: dict[str, Any], status: HTTPStatus = HTTPStatus.OK) -> None:
+ data = json.dumps(payload, sort_keys=True).encode("utf-8")
+ self.send_response(status)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(data)))
+ self.end_headers()
+ self.wfile.write(data)
+
+ def _write_error(self, status: HTTPStatus, code: str, message: str, request_id: str) -> None:
+ self._write_json({"ok": False, "request_id": request_id, "error": {"code": code, "message": message}}, status)
+
+
+def serve_gateway(
+ service: Any,
+ *,
+ host: str = "127.0.0.1",
+ port: int = 8765,
+ secret: str | None = None,
+ certfile: str | None = None,
+ keyfile: str | None = None,
+) -> ThreadingHTTPServer:
+ secret = secret if secret is not None else os.getenv("KUBLAI_GATEWAY_HMAC_SECRET")
+ if not secret and not is_loopback_host(host):
+ raise ValueError("brain public gateway requires HMAC secret for non-loopback bind")
+
+ class Handler(PublicGatewayHandler):
+ pass
+
+ Handler.service = service
+ Handler.hmac_secret = secret
+ server = ThreadingHTTPServer((host, port), Handler)
+ if certfile:
+ context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+ context.load_cert_chain(certfile=certfile, keyfile=keyfile)
+ server.socket = context.wrap_socket(server.socket, server_side=True)
+ return server
diff --git a/lints/no_active_graph_store.py b/lints/no_active_graph_store.py
new file mode 100755
index 00000000..ec6c24b5
--- /dev/null
+++ b/lints/no_active_graph_store.py
@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+"""Fail if launchd-active runtime paths depend on the retired graph store."""
+from __future__ import annotations
+
+import ast
+import os
+import plistlib
+import re
+import sys
+from pathlib import Path
+
+HOME = Path(os.environ.get("HOME", "/Users/kublai")).expanduser()
+LAUNCH_AGENTS = HOME / "Library" / "LaunchAgents"
+REPO_ROOT = HOME / "kurultai" / "kublai-repo"
+OPENCLAW_SCRIPTS = HOME / ".openclaw" / "agents" / "main" / "scripts"
+OPENCLAW_ROOT = HOME / ".openclaw"
+
+ACTIVE_LABEL_RE = re.compile(r"(kurultai|openclaw|hermes|brain)", re.IGNORECASE)
+BLOCKED_PATTERNS = [
+ re.compile(r"^\s*from\s+neo4j\b", re.IGNORECASE),
+ re.compile(r"^\s*import\s+neo4j\b", re.IGNORECASE),
+ re.compile(r"^\s*from\s+neo4j_task_tracker\b", re.IGNORECASE),
+ re.compile(r"^\s*import\s+neo4j_task_tracker\b", re.IGNORECASE),
+ re.compile(r"^\s*from\s+neo4j_v2_core\b", re.IGNORECASE),
+ re.compile(r"^\s*import\s+neo4j_v2_core\b", re.IGNORECASE),
+ re.compile(r"^\s*from\s+neo4j_calendar\b", re.IGNORECASE),
+ re.compile(r"^\s*import\s+neo4j_calendar\b", re.IGNORECASE),
+ re.compile(r"\bGraphDatabase(?:\.driver)?\b"),
+ re.compile(r"\bcypher-shell\b"),
+ re.compile(r"bolt://localhost:7687"),
+ re.compile(r"\bNEO4J_[A-Z_]+\b"),
+ re.compile(r"neo4j\.env", re.IGNORECASE),
+]
+
+RUNTIME_EXTS = {".py", ".sh", ".bash", ".zsh", ".js", ".mjs", ".cjs"}
+SEARCH_ROOTS = [
+ OPENCLAW_SCRIPTS,
+ OPENCLAW_ROOT / "scripts",
+ REPO_ROOT,
+ REPO_ROOT / "tools",
+ REPO_ROOT / "kublai",
+ HOME / "Developer" / "brain-sync",
+ HOME / ".hermes" / "hermes-agent",
+]
+
+
+def _read_plist(path: Path) -> dict:
+ try:
+ with path.open("rb") as f:
+ return plistlib.load(f)
+ except Exception:
+ return {}
+
+
+def _is_runtime_file(path: Path) -> bool:
+ return path.exists() and path.is_file() and path.suffix in RUNTIME_EXTS
+
+
+def _resolve_module(module: str, importer: Path | None = None) -> Path | None:
+ parts = module.split(".")
+ roots = []
+ if importer is not None:
+ roots.append(importer.parent)
+ roots.extend(SEARCH_ROOTS)
+ for root in roots:
+ direct = root.joinpath(*parts).with_suffix(".py")
+ package = root.joinpath(*parts, "__init__.py")
+ if _is_runtime_file(direct):
+ return direct.resolve()
+ if _is_runtime_file(package):
+ return package.resolve()
+ return None
+
+
+def _extract_program_files(plist_path: Path) -> set[Path]:
+ data = _read_plist(plist_path)
+ label = str(data.get("Label") or "")
+ if not ACTIVE_LABEL_RE.search(label):
+ return set()
+ if label == "homebrew.mxcl.neo4j":
+ return set()
+ args = [str(a) for a in data.get("ProgramArguments") or []]
+ workdir = Path(str(data.get("WorkingDirectory") or plist_path.parent)).expanduser()
+ files: set[Path] = set()
+ for idx, arg in enumerate(args):
+ candidate = Path(arg).expanduser()
+ if not candidate.is_absolute():
+ candidate = workdir / candidate
+ if _is_runtime_file(candidate):
+ files.add(candidate.resolve())
+ if arg == "-m" and idx + 1 < len(args):
+ module_path = _resolve_module(args[idx + 1], workdir)
+ if module_path:
+ files.add(module_path)
+ return files
+
+
+_ABS_RUNTIME_RE = re.compile(r"(/Users/kublai/[^\s\"'`<>|;&]+(?:\.py|\.sh|\.bash|\.zsh|\.js|\.mjs|\.cjs))")
+_REL_RUNTIME_RE = re.compile(r"(? set[Path]:
+ files: set[Path] = set()
+ for match in _ABS_RUNTIME_RE.finditer(text):
+ candidate = Path(match.group(1)).expanduser()
+ if _is_runtime_file(candidate):
+ files.add(candidate.resolve())
+ # Shell wrappers commonly invoke relative scripts. Python files, by
+ # contrast, often mention script names in docs/help text; only follow
+ # relative names from explicit dynamic-load/invocation lines.
+ if path.suffix in {".sh", ".bash", ".zsh"}:
+ candidate_lines = text.splitlines()
+ else:
+ candidate_lines = [
+ line for line in text.splitlines()
+ if (
+ "spec_from_file_location" in line
+ or "run([" in line
+ or "subprocess." in line
+ or "SCRIPTS_DIR" in line
+ or "Path(__file__)" in line
+ )
+ ]
+ for line in candidate_lines:
+ for match in _REL_RUNTIME_RE.finditer(line):
+ candidate = path.parent / match.group(1)
+ if _is_runtime_file(candidate):
+ files.add(candidate.resolve())
+ return files
+
+
+def _extract_python_imports(path: Path, text: str) -> set[Path]:
+ files: set[Path] = set()
+ try:
+ tree = ast.parse(text)
+ except SyntaxError:
+ return files
+ for node in tree.body:
+ if isinstance(node, ast.Import):
+ for alias in node.names:
+ module_path = _resolve_module(alias.name, path)
+ if module_path:
+ files.add(module_path)
+ elif isinstance(node, ast.ImportFrom) and node.module:
+ module_path = _resolve_module(node.module, path)
+ if module_path:
+ files.add(module_path)
+ return files
+
+
+def discover_active_files() -> set[Path]:
+ pending: list[Path] = []
+ seen: set[Path] = set()
+ for plist in LAUNCH_AGENTS.glob("*.plist"):
+ pending.extend(sorted(_extract_program_files(plist)))
+
+ while pending:
+ path = pending.pop()
+ if path in seen or not _is_runtime_file(path):
+ continue
+ seen.add(path)
+ try:
+ text = path.read_text(encoding="utf-8", errors="ignore")
+ except OSError:
+ continue
+ pending.extend(_extract_referenced_runtime_files(path, text) - seen)
+ if path.suffix == ".py":
+ pending.extend(_extract_python_imports(path, text) - seen)
+ return seen
+
+
+def _is_scannable_line(line: str) -> bool:
+ stripped = line.strip()
+ if not stripped:
+ return False
+ if stripped.startswith("#"):
+ return False
+ if stripped.startswith("//"):
+ return False
+ return True
+
+
+def scan_file(path: Path) -> list[str]:
+ failures: list[str] = []
+ try:
+ text = path.read_text(encoding="utf-8", errors="ignore")
+ except OSError as exc:
+ return [f"{path}: unreadable: {exc}"]
+ for lineno, line in enumerate(text.splitlines(), 1):
+ if not _is_scannable_line(line):
+ continue
+ if any(pattern.search(line) for pattern in BLOCKED_PATTERNS):
+ failures.append(f"{path}:{lineno}: {line.strip()[:180]}")
+ return failures
+
+
+def main() -> int:
+ active_files = discover_active_files()
+ failures: list[str] = []
+ for path in sorted(active_files):
+ failures.extend(scan_file(path))
+ if failures:
+ print("Active graph-store references found:", file=sys.stderr)
+ print("\n".join(failures), file=sys.stderr)
+ return 1
+ print(f"ok: no active graph-store references ({len(active_files)} runtime files scanned)")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/migrations/sql/migration-5-task-lifecycle-extended.sql b/migrations/sql/migration-5-task-lifecycle-extended.sql
new file mode 100644
index 00000000..5c30efc3
--- /dev/null
+++ b/migrations/sql/migration-5-task-lifecycle-extended.sql
@@ -0,0 +1,141 @@
+-- Migration v5: Extend in_flight_tasks with Phase 3 dashboard columns and widen status CHECK.
+--
+-- Reason: server.js (the-kurultai dashboard) reads/writes ~24 task properties via Cypher
+-- that don't exist as columns in the SQLite mirror. The Kanban UI also stores statuses
+-- as uppercase variants (PENDING, WORKING, COMPLETED, FAILED, ORPHANED, OBSOLETE, DONE,
+-- STALE, CANCELLED, IN_PROGRESS) which are not in the original CHECK clause.
+--
+-- Strategy:
+-- 1. Recreate in_flight_tasks (STRICT) with all new columns + widened CHECK (CHECK
+-- cannot be altered in place on a STRICT table).
+-- 2. Copy existing rows into the new table.
+-- 3. Recreate dependent indexes and FK from claim_locks.
+-- 4. Add idx_in_flight_tasks_dashboard covering index for Kanban GET.
+-- 5. Record migration v5.
+
+BEGIN IMMEDIATE;
+
+-- Drop existing dependent indexes (will recreate after rename swap)
+DROP INDEX IF EXISTS idx_inflight_tasks_assigned_status;
+DROP INDEX IF EXISTS idx_inflight_tasks_status_priority;
+DROP INDEX IF EXISTS idx_inflight_tasks_pipeline;
+DROP INDEX IF EXISTS idx_inflight_tasks_parent;
+DROP INDEX IF EXISTS idx_inflight_tasks_paused;
+DROP INDEX IF EXISTS idx_in_flight_tasks_dashboard;
+
+-- Phase A: rename old table out of the way (FK from claim_locks would point to old name
+-- after rename, so we DROP claim_locks FK rows first by deleting all locks; the live
+-- run pauses gateway/executor/agents before this migration so claim_locks is empty)
+ALTER TABLE in_flight_tasks RENAME TO _in_flight_tasks_v4_pre_phase_3;
+
+-- Phase B: create new table with all Phase 3 columns + widened CHECK
+CREATE TABLE in_flight_tasks (
+ id TEXT PRIMARY KEY,
+ type TEXT NOT NULL,
+ description TEXT NOT NULL,
+ delegated_by TEXT NOT NULL,
+ assigned_to TEXT,
+ priority INTEGER NOT NULL,
+ status TEXT NOT NULL CHECK(status IN (
+ 'pending','in_progress','completed','failed','cancelled',
+ 'PENDING','WORKING','COMPLETED','FAILED','ORPHANED',
+ 'OBSOLETE','DONE','STALE','CANCELLED','IN_PROGRESS'
+ )),
+ claimed_by TEXT,
+ claimed_at INTEGER,
+ active_claim_token TEXT,
+ active_lease_version INTEGER,
+ completed_at INTEGER,
+ failed_at INTEGER,
+ results_json TEXT CHECK(results_json IS NULL OR json_valid(results_json)),
+ completion_summary TEXT,
+ completion_body_hash TEXT,
+ target_wiki_path TEXT,
+ wiki_path TEXT,
+ materialized_at INTEGER,
+ completion_attempt_count INTEGER NOT NULL DEFAULT 0,
+ last_error TEXT,
+ created_at INTEGER NOT NULL,
+ updated_at INTEGER NOT NULL,
+ trace_id TEXT,
+ reliability_state TEXT NOT NULL DEFAULT 'pending',
+ retry_count INTEGER NOT NULL DEFAULT 0,
+ -- Phase 3 dashboard columns (server.js Cypher property keys)
+ title TEXT,
+ prompt TEXT,
+ domain TEXT,
+ source TEXT,
+ parent_task TEXT,
+ reflection_id TEXT,
+ pipeline_id TEXT,
+ sort_order INTEGER,
+ paused INTEGER NOT NULL DEFAULT 0,
+ paused_at INTEGER,
+ dispatch_phase TEXT,
+ max_retries INTEGER NOT NULL DEFAULT 3,
+ timeout_s INTEGER,
+ depth INTEGER NOT NULL DEFAULT 0,
+ requires_computer_use INTEGER NOT NULL DEFAULT 0,
+ skill_hint TEXT,
+ reassigned_from TEXT,
+ previous_status TEXT,
+ previous_agent TEXT,
+ original_prompt TEXT,
+ previous_prompt TEXT,
+ cancelled_at INTEGER,
+ started_at INTEGER,
+ claim_epoch INTEGER,
+ score REAL,
+ obsolete_reason TEXT,
+ obsolete_by TEXT,
+ obsolete_at INTEGER,
+ rewrite_reason TEXT,
+ rewrite_by TEXT,
+ rewritten_at INTEGER,
+ reassign_reason TEXT,
+ reassigned_by TEXT,
+ reassigned_at INTEGER,
+ optimized_prompt TEXT
+) STRICT;
+
+-- Phase C: copy existing rows. Older rows have only the legacy columns; new ones get
+-- defaults via NULL or the NOT NULL DEFAULT clauses above.
+INSERT INTO in_flight_tasks (
+ id, type, description, delegated_by, assigned_to, priority, status,
+ claimed_by, claimed_at, active_claim_token, active_lease_version,
+ completed_at, failed_at, results_json, completion_summary, completion_body_hash,
+ target_wiki_path, wiki_path, materialized_at, completion_attempt_count, last_error,
+ created_at, updated_at, trace_id, reliability_state, retry_count
+)
+SELECT
+ id, type, description, delegated_by, assigned_to, priority, status,
+ claimed_by, claimed_at, active_claim_token, active_lease_version,
+ completed_at, failed_at, results_json, completion_summary, completion_body_hash,
+ target_wiki_path, wiki_path, materialized_at, completion_attempt_count, last_error,
+ created_at, updated_at, trace_id, reliability_state, retry_count
+FROM _in_flight_tasks_v4_pre_phase_3;
+
+-- Phase D: drop the renamed table (data fully copied to new in_flight_tasks)
+DROP TABLE _in_flight_tasks_v4_pre_phase_3;
+
+-- Phase E: recreate indexes
+CREATE INDEX IF NOT EXISTS idx_inflight_tasks_assigned_status
+ ON in_flight_tasks(assigned_to, status);
+CREATE INDEX IF NOT EXISTS idx_inflight_tasks_status_priority
+ ON in_flight_tasks(status, priority, created_at);
+CREATE INDEX IF NOT EXISTS idx_inflight_tasks_pipeline
+ ON in_flight_tasks(pipeline_id);
+CREATE INDEX IF NOT EXISTS idx_inflight_tasks_parent
+ ON in_flight_tasks(parent_task);
+CREATE INDEX IF NOT EXISTS idx_inflight_tasks_paused
+ ON in_flight_tasks(paused, status);
+
+-- Phase F: covering index for the Kanban GET (status,sort_order,priority,created_at)
+CREATE INDEX IF NOT EXISTS idx_in_flight_tasks_dashboard
+ ON in_flight_tasks(status, sort_order DESC, priority, created_at DESC);
+
+-- Phase G: record the migration
+INSERT OR IGNORE INTO schema_migrations (version, applied_at, description)
+ VALUES (5, unixepoch() * 1000, 'phase 3 step 11 in_flight_tasks extended');
+
+COMMIT;
diff --git a/openclaw_memory.py b/openclaw_memory.py
index 6cfe4e27..754ea44e 100644
--- a/openclaw_memory.py
+++ b/openclaw_memory.py
@@ -7,16 +7,23 @@
from __future__ import annotations
import logging
+import json
+import os
import uuid
import time
import functools
from datetime import datetime, timezone, timedelta
+from pathlib import Path
from typing import Dict, List, Optional, Tuple, Any, Callable
from contextlib import contextmanager
# Configure logging
logger = logging.getLogger(__name__)
+
+def _env_truthy(name: str) -> bool:
+ return os.getenv(name, "").lower() in {"1", "true", "yes", "sqlite", "telemetry"}
+
# Lazy imports for neo4j to avoid numpy recursion issues during test collection
# The neo4j driver imports numpy which can cause RecursionError when pytest's -W error is enabled
_neo4j_imported = False
@@ -242,7 +249,15 @@ def __init__(
Raises:
ValueError: If password is not provided
"""
- if password is None:
+ self._telemetry_primary = _env_truthy("KUBLAI_TELEMETRY_PRIMARY")
+ self._telemetry_store = None
+ if self._telemetry_primary:
+ from kublai.telemetry import TelemetryStore
+
+ telemetry_db = os.getenv("KUBLAI_TELEMETRY_DB", str(Path.home() / ".kublai" / "telemetry.db"))
+ self._telemetry_store = TelemetryStore(telemetry_db)
+
+ if password is None and not self._telemetry_primary:
raise ValueError("Neo4j password is required. Provide it explicitly or via NEO4J_PASSWORD environment variable.")
self.uri = uri
@@ -256,7 +271,54 @@ def __init__(
"connection_timeout": connection_timeout,
"max_transaction_retry_time": max_retry_time
}
- self._initialize_driver()
+ if not self._telemetry_primary:
+ self._initialize_driver()
+
+ def _using_telemetry(self) -> bool:
+ return self._telemetry_store is not None
+
+ @staticmethod
+ def _priority_to_int(priority: str) -> int:
+ return {"low": 1, "normal": 2, "high": 3, "critical": 4}.get(str(priority).lower(), 0)
+
+ @staticmethod
+ def _priority_from_int(priority: int | None) -> str:
+ mapping = {1: "low", 2: "normal", 3: "high", 4: "critical"}
+ return mapping.get(int(priority or 0), "normal")
+
+ @staticmethod
+ def _dt_to_ms(value: datetime) -> int:
+ return int(value.timestamp() * 1000)
+
+ @staticmethod
+ def _ms_to_dt(value: int | None) -> datetime | None:
+ if value is None:
+ return None
+ return datetime.fromtimestamp(int(value) / 1000, timezone.utc)
+
+ def _telemetry_task_to_legacy(self, row: Dict[str, Any]) -> Dict[str, Any]:
+ results = row.get("results_json")
+ try:
+ parsed_results = json.loads(results) if results else None
+ except (TypeError, json.JSONDecodeError):
+ parsed_results = results
+ return {
+ "id": row.get("id"),
+ "type": row.get("type"),
+ "description": row.get("description"),
+ "status": row.get("status"),
+ "delegated_by": row.get("delegated_by"),
+ "assigned_to": row.get("assigned_to"),
+ "priority": self._priority_from_int(row.get("priority")),
+ "created_at": self._ms_to_dt(row.get("created_at")),
+ "claimed_at": self._ms_to_dt(row.get("claimed_at")),
+ "completed_at": self._ms_to_dt(row.get("completed_at")),
+ "claimed_by": row.get("claimed_by"),
+ "claim_token": row.get("active_claim_token"),
+ "lease_version": row.get("active_lease_version"),
+ "results": parsed_results,
+ "error_message": row.get("materialization_error"),
+ }
def _initialize_driver(self) -> None:
"""Initialize Neo4j driver with connection pooling."""
@@ -351,6 +413,20 @@ def create_task(
else:
assigned_to_value = assigned_to
+ if self._using_telemetry():
+ assert self._telemetry_store is not None
+ task_id = self._telemetry_store.create_task(
+ task_id,
+ type=task_type,
+ description=description,
+ delegated_by=delegated_by,
+ assigned_to=assigned_to_value,
+ priority=self._priority_to_int(priority),
+ results=kwargs,
+ )
+ logger.info(f"Telemetry task created: {task_id} (type: {task_type}, assigned_to: {assigned_to})")
+ return task_id
+
cypher = """
CREATE (t:Task {
id: $task_id,
@@ -411,6 +487,23 @@ def claim_task(self, agent: str) -> Optional[Dict]:
RaceConditionError: If another agent claimed the task simultaneously
NoPendingTaskError: If no pending tasks are available
"""
+ if self._using_telemetry():
+ assert self._telemetry_store is not None
+ from kublai.telemetry import NoPendingTaskError as TelemetryNoPendingTaskError
+
+ try:
+ claimed = self._telemetry_store.claim_task(agent)
+ task = self._telemetry_task_to_legacy(claimed.payload)
+ task["status"] = "in_progress"
+ task["claimed_by"] = agent
+ task["claimed_at"] = self._now()
+ task["claim_token"] = claimed.claim_token
+ task["lease_version"] = claimed.lease_version
+ logger.info(f"Telemetry task claimed: {task['id']} by {agent}")
+ return task
+ except TelemetryNoPendingTaskError as exc:
+ raise NoPendingTaskError(str(exc)) from exc
+
claimed_at = self._now()
# Use explicit locking pattern for atomic claim
@@ -480,6 +573,33 @@ def complete_task(
Returns:
True if successful
"""
+ if self._using_telemetry():
+ assert self._telemetry_store is not None
+ with self._telemetry_store.connect() as conn:
+ row = conn.execute(
+ "SELECT claimed_by, active_claim_token, delegated_by FROM in_flight_tasks WHERE id = ?",
+ (task_id,),
+ ).fetchone()
+ if row is None or not row["claimed_by"] or not row["active_claim_token"]:
+ logger.warning(f"Telemetry task not found or not in_progress: {task_id}")
+ return False
+ self._telemetry_store.complete_task(
+ task_id,
+ row["claimed_by"],
+ row["active_claim_token"],
+ results=results,
+ summary=str(results),
+ )
+ if notify_delegator and row["delegated_by"]:
+ self.create_notification(
+ agent=row["delegated_by"],
+ type="task_completed",
+ summary=f"Task {task_id} completed by {row['claimed_by']}",
+ task_id=task_id,
+ )
+ logger.info(f"Telemetry task completed: {task_id}")
+ return True
+
completed_at = self._now()
cypher = """
@@ -727,6 +847,13 @@ def create_notification(
notification_id = self._generate_id()
created_at = self._now()
+ if self._using_telemetry():
+ assert self._telemetry_store is not None
+ body = summary if task_id is None else json.dumps({"summary": summary, "task_id": task_id}, sort_keys=True)
+ notification_id = self._telemetry_store.create_notification(agent, type, body)
+ logger.info(f"Telemetry notification created: {notification_id} for {agent}")
+ return notification_id
+
cypher = """
CREATE (n:Notification {
id: $notification_id,
@@ -780,6 +907,19 @@ def get_notifications(
Returns:
List of notification dicts
"""
+ if self._using_telemetry():
+ assert self._telemetry_store is not None
+ notifications = self._telemetry_store.list_notifications(agent, unread_only=unread_only)
+ legacy = []
+ for notification in notifications:
+ item = dict(notification)
+ item["type"] = item.pop("kind")
+ item["summary"] = item.pop("body")
+ item["read"] = item.get("read_at") is not None
+ item["created_at"] = self._ms_to_dt(item.get("created_at"))
+ legacy.append(item)
+ return legacy
+
if unread_only:
cypher = """
MATCH (n:Notification {agent: $agent, read: false})
@@ -837,6 +977,15 @@ def mark_notification_read(self, notification_id: str) -> bool:
Returns:
True if successful
"""
+ if self._using_telemetry():
+ assert self._telemetry_store is not None
+ with self._telemetry_store.connect() as conn:
+ row = conn.execute("SELECT id FROM notifications WHERE id = ?", (notification_id,)).fetchone()
+ if row is None:
+ return False
+ self._telemetry_store.mark_notification_read(notification_id)
+ return True
+
cypher = """
MATCH (n:Notification {id: $notification_id})
SET n.read = true
@@ -868,6 +1017,13 @@ def mark_all_notifications_read(self, agent: str) -> int:
Returns:
Number of notifications marked as read
"""
+ if self._using_telemetry():
+ notifications = self.get_notifications(agent, unread_only=True)
+ for notification in notifications:
+ self.mark_notification_read(notification["id"])
+ logger.info(f"Marked {len(notifications)} telemetry notifications as read for {agent}")
+ return len(notifications)
+
cypher = """
MATCH (n:Notification {agent: $agent, read: false})
SET n.read = true
@@ -925,6 +1081,22 @@ def check_rate_limit(
else:
reset_time = int((datetime.combine(current_date, datetime.min.time().replace(hour=current_hour + 1)) - datetime(1970, 1, 1)).total_seconds())
+ if self._using_telemetry():
+ assert self._telemetry_store is not None
+ now_ms = self._dt_to_ms(now)
+ window_ms = 3_600_000
+ bucket_start_ms = (now_ms // window_ms) * window_ms
+ with self._telemetry_store.connect() as conn:
+ row = conn.execute(
+ """
+ SELECT count FROM rate_limits
+ WHERE agent = ? AND operation = ? AND window_ms = ? AND bucket_start_ms = ?
+ """,
+ (agent, operation, window_ms, bucket_start_ms),
+ ).fetchone()
+ count = int(row["count"]) if row else 0
+ return (count < max_requests, count, reset_time)
+
cypher = """
MATCH (r:RateLimit {agent: $agent, operation: $operation, date: $date, hour: $hour})
RETURN r.count as count
@@ -1004,6 +1176,16 @@ def record_rate_limit_hit(self, agent: str, operation: str) -> bool:
current_date = now.date()
current_hour = now.hour
+ if self._using_telemetry():
+ assert self._telemetry_store is not None
+ self._telemetry_store.increment_rate_limit(
+ agent,
+ operation,
+ window_ms=3_600_000,
+ now_ms=self._dt_to_ms(now),
+ )
+ return True
+
cypher = """
MERGE (r:RateLimit {agent: $agent, operation: $operation, date: $date, hour: $hour})
ON CREATE SET r.count = 1, r.last_updated = $last_updated
@@ -1052,6 +1234,12 @@ def update_agent_heartbeat(self, agent: str, status: str = "active") -> bool:
"""
now = self._now()
+ if self._using_telemetry():
+ assert self._telemetry_store is not None
+ self._telemetry_store.heartbeat(agent, status=status)
+ logger.debug(f"Telemetry agent heartbeat updated: {agent} ({status})")
+ return True
+
cypher = """
MERGE (a:Agent {name: $agent})
ON CREATE SET a.created_at = $now
@@ -1091,6 +1279,19 @@ def get_agent_status(self, agent: str) -> Optional[Dict]:
Returns:
Agent status dict if found, None otherwise
"""
+ if self._using_telemetry():
+ assert self._telemetry_store is not None
+ with self._telemetry_store.connect() as conn:
+ row = conn.execute("SELECT * FROM agent_state WHERE agent = ?", (agent,)).fetchone()
+ if not row:
+ return None
+ return {
+ "name": row["agent"],
+ "status": row["status"],
+ "last_heartbeat": self._ms_to_dt(row["last_heartbeat"]),
+ "meta": json.loads(row["meta"] or "{}"),
+ }
+
cypher = """
MATCH (a:Agent {name: $agent})
RETURN a
diff --git a/requirements.txt b/requirements.txt
index f7bfddcc..83da15fb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -37,6 +37,8 @@ cryptography>=41.0.0
# ML/Embeddings (for semantic similarity in dependency analysis)
sentence-transformers>=2.2.0
+sqlite-vec>=0.1.9
+watchdog>=6.0.0
# HTTP (used by delegation protocol)
requests>=2.31.0
diff --git a/scripts/bootstrap_kurultai_runtime.py b/scripts/bootstrap_kurultai_runtime.py
new file mode 100644
index 00000000..2d336c72
--- /dev/null
+++ b/scripts/bootstrap_kurultai_runtime.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import shutil
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+CONFIG = ROOT / "config" / "runtime-config"
+STAGING = Path.home() / ".kurultai-rebuild-staging"
+
+FILES = [
+ "hermes.template.yaml",
+ "profiles.yaml",
+ "kurultai.yaml",
+ "brain.yaml",
+ "cron.manifest.json",
+ "skills.manifest.json",
+ "kanban.schema.json",
+ "brain.manifest.json",
+]
+
+DIRS = [
+ Path.home() / ".hermes",
+ Path.home() / ".hermes" / "profiles",
+ Path.home() / ".hermes" / "skills",
+ Path.home() / ".hermes" / "cron",
+ Path.home() / "brain",
+]
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description="Stage sanitized Kurultai rebuild configuration")
+ parser.add_argument("--dry-run", action="store_true", help="print planned actions only")
+ args = parser.parse_args()
+
+ print("Kurultai rebuild staging")
+ print(f"source={CONFIG}")
+ print(f"staging={STAGING}")
+
+ for d in DIRS:
+ print(f"ensure-dir {d}")
+ if not args.dry_run:
+ d.mkdir(parents=True, exist_ok=True)
+
+ for name in FILES:
+ src = CONFIG / name
+ dst = STAGING / name
+ if not src.exists():
+ print(f"missing {src}")
+ continue
+ print(f"copy {src} -> {dst}")
+ if not args.dry_run:
+ STAGING.mkdir(parents=True, exist_ok=True)
+ shutil.copy2(src, dst)
+
+ marker = STAGING / "README.txt"
+ print(f"write {marker}")
+ if not args.dry_run:
+ marker.write_text(
+ "Sanitized Kurultai rebuild staging area. Review these files before applying live private configuration.\n"
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/export_rebuild_manifests.py b/scripts/export_rebuild_manifests.py
new file mode 100644
index 00000000..90b34e1d
--- /dev/null
+++ b/scripts/export_rebuild_manifests.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parents[1]
+OUT = ROOT / "config" / "runtime-config"
+HOME = Path.home()
+
+
+def write_json(name: str, data: Any) -> None:
+ path = OUT / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n")
+ print(path)
+
+
+def parse_skill_header(path: Path) -> dict[str, Any]:
+ text = path.read_text(errors="ignore")
+ description = ""
+ if text.startswith("---"):
+ end = text.find("\n---", 3)
+ if end != -1:
+ header = text[3:end]
+ for line in header.splitlines():
+ if line.strip().startswith("description:"):
+ description = line.split(":", 1)[1].strip().strip('"')
+ break
+ rel = path.parent.relative_to(HOME / ".hermes" / "skills")
+ return {
+ "name": path.parent.name,
+ "path": str(rel),
+ "description": description,
+ }
+
+
+def export_skills() -> None:
+ root = HOME / ".hermes" / "skills"
+ skills = []
+ if root.exists():
+ for md in sorted(root.glob("**/SKILL.md")):
+ skills.append(parse_skill_header(md))
+ write_json("skills.manifest.json", {
+ "schema": "kurultai.skills-manifest.v1",
+ "source": "~/.hermes/skills/**/SKILL.md",
+ "note": "Names, relative paths, and descriptions only. Skill bodies are intentionally not exported by this manifest.",
+ "skills": skills,
+ })
+
+
+def export_kanban_schema() -> None:
+ db = HOME / ".hermes" / "kanban.db"
+ tables = []
+ if db.exists():
+ con = sqlite3.connect(db)
+ for name, typ in con.execute("select name, type from sqlite_master where type in ('table','view') order by name"):
+ if name == "sqlite_sequence":
+ continue
+ cols = []
+ if typ == "table":
+ for col in con.execute(f"pragma table_info({name})"):
+ cols.append({
+ "name": col[1],
+ "type": col[2],
+ "not_null": bool(col[3]),
+ "default": col[4],
+ "primary_key": bool(col[5]),
+ })
+ tables.append({"name": name, "type": typ, "columns": cols})
+ con.close()
+ write_json("kanban.schema.json", {
+ "schema": "kurultai.kanban-schema.v1",
+ "source": "~/.hermes/kanban.db sqlite schema only",
+ "note": "No task bodies, comments, chat IDs, or run outputs are exported.",
+ "tables": tables,
+ })
+
+
+def export_brain_manifest() -> None:
+ root = HOME / "brain"
+ dirs = []
+ if root.exists():
+ for child in sorted(root.iterdir()):
+ if child.is_dir() and not child.name.startswith("."):
+ count = sum(1 for p in child.rglob("*.md") if p.is_file())
+ dirs.append({"name": child.name, "markdown_files": count})
+ write_json("brain.manifest.json", {
+ "schema": "kurultai.brain-manifest.v1",
+ "wiki_root": str(root),
+ "note": "Directory/file-count inventory only. Brain content and private indexes are not copied into this repo.",
+ "top_level_directories": dirs,
+ })
+
+
+def main() -> None:
+ export_skills()
+ export_kanban_schema()
+ export_brain_manifest()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/export_runtime_config_manifest.py b/scripts/export_runtime_config_manifest.py
new file mode 100644
index 00000000..fcee7b1c
--- /dev/null
+++ b/scripts/export_runtime_config_manifest.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+root = Path(__file__).resolve().parents[1]
+source = Path.home() / ".hermes" / "cron" / "jobs.json"
+out = root / "config" / "runtime-config" / "cron.manifest.json"
+
+def load_jobs():
+ if not source.exists():
+ return []
+ data = json.loads(source.read_text())
+ if isinstance(data, dict):
+ return data.get("jobs", [])
+ if isinstance(data, list):
+ return data
+ return []
+
+def public_delivery(value):
+ if value in (None, "local", "origin"):
+ return value
+ text = str(value)
+ if text.startswith("telegram:"):
+ return "telegram:[REDACTED]"
+ if text.startswith("discord:"):
+ return "discord:[REDACTED]"
+ return "external:[REDACTED]"
+
+
+manifest = []
+for job in load_jobs():
+ manifest.append({
+ "job_id": job.get("job_id") or job.get("id"),
+ "name": job.get("name"),
+ "schedule": job.get("schedule"),
+ "repeat": job.get("repeat"),
+ "deliver": public_delivery(job.get("deliver")),
+ "enabled": job.get("enabled"),
+ "state": job.get("state"),
+ "skills": job.get("skills") or ([job.get("skill")] if job.get("skill") else []),
+ "enabled_toolsets": job.get("enabled_toolsets"),
+ "script": job.get("script"),
+ "workdir": job.get("workdir"),
+ })
+
+out.write_text(json.dumps({"schema": "kurultai.cron-manifest.v1", "jobs": manifest}, indent=2, sort_keys=True) + "\n")
+print(out)
diff --git a/scripts/model_switcher.py b/scripts/model_switcher.py
index 79931ce7..39d49d0b 100644
--- a/scripts/model_switcher.py
+++ b/scripts/model_switcher.py
@@ -10,6 +10,7 @@
- Jochi (analyst): Analysis tasks
- Ögedei (ops): Operations/emergency router
"""
+from __future__ import annotations
import fcntl
import json
diff --git a/scripts/optimize_classifier.py b/scripts/optimize_classifier.py
index 7d34be06..b0adbea5 100644
--- a/scripts/optimize_classifier.py
+++ b/scripts/optimize_classifier.py
@@ -1,4 +1,6 @@
"""Optimize TeamSizeClassifier weights and thresholds via k-fold cross-validation."""
+from __future__ import annotations
+
import sys
sys.path.insert(0, ".")
diff --git a/src/kublai/knowledge-client.js b/src/kublai/knowledge-client.js
new file mode 100644
index 00000000..04cfafda
--- /dev/null
+++ b/src/kublai/knowledge-client.js
@@ -0,0 +1,3 @@
+'use strict';
+
+module.exports = require('../../kublai/knowledge-client');
diff --git a/src/kublai/telemetry-client.js b/src/kublai/telemetry-client.js
new file mode 100644
index 00000000..fa024e50
--- /dev/null
+++ b/src/kublai/telemetry-client.js
@@ -0,0 +1,3 @@
+'use strict';
+
+module.exports = require('../../kublai/telemetry-client');
diff --git a/test_runbooks_integration.py b/test_runbooks_integration.py
index 35b66cd1..522a26a6 100644
--- a/test_runbooks_integration.py
+++ b/test_runbooks_integration.py
@@ -12,6 +12,7 @@
4. ErrorRecoveryManager.load_runbook() functionality
5. Content quality checks (required sections)
"""
+from __future__ import annotations
import sys
import re
diff --git a/test_webchat.py b/test_webchat.py
index ae3d6585..eccb2bd3 100644
--- a/test_webchat.py
+++ b/test_webchat.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""Test webchat WebSocket connection and verify the fix."""
+from __future__ import annotations
from playwright.sync_api import sync_playwright
import sys
diff --git a/test_webchat2.py b/test_webchat2.py
index fab8193d..d0e76ef0 100644
--- a/test_webchat2.py
+++ b/test_webchat2.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""Test webchat WebSocket connection with localStorage clearing."""
+from __future__ import annotations
from playwright.sync_api import sync_playwright
import sys
diff --git a/test_webchat3.py b/test_webchat3.py
index 40bdc03b..33ae9313 100644
--- a/test_webchat3.py
+++ b/test_webchat3.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""Test webchat WebSocket connection with correct localStorage URL."""
+from __future__ import annotations
from playwright.sync_api import sync_playwright
import sys
diff --git a/tests/integration/test_neo4j_operations.py b/tests/integration/test_neo4j_operations.py
index d3e4e2c6..af062611 100644
--- a/tests/integration/test_neo4j_operations.py
+++ b/tests/integration/test_neo4j_operations.py
@@ -10,6 +10,7 @@
Note: These tests require the testcontainers package.
Run with: pytest tests/integration/test_neo4j_operations.py -v
"""
+from __future__ import annotations
import asyncio
from datetime import datetime, timezone
diff --git a/tests/interactive/run_interactive_tests.py b/tests/interactive/run_interactive_tests.py
index 4ff6423f..2ed60a55 100644
--- a/tests/interactive/run_interactive_tests.py
+++ b/tests/interactive/run_interactive_tests.py
@@ -14,6 +14,8 @@
python run_interactive_tests.py compare
"""
+from __future__ import annotations
+
import asyncio
import json
import sys
@@ -263,7 +265,7 @@ def compare_sessions(self, session1_path: str, session2_path: str):
sign = "+" if query_diff >= 0 else ""
print(f"Neo4j query difference: {sign}{query_diff}")
- def run_all_scenarios(self) -> create_scenario_report:
+ async def run_all_scenarios(self) -> create_scenario_report:
"""Run all scenarios sequentially.
Returns:
diff --git a/tests/kurultai/test_buildroom_foundation.py b/tests/kurultai/test_buildroom_foundation.py
new file mode 100644
index 00000000..a3ba97e8
--- /dev/null
+++ b/tests/kurultai/test_buildroom_foundation.py
@@ -0,0 +1,425 @@
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+BUILDROOM = ROOT / "tools" / "kurultai" / "buildroom"
+
+
+def run_script(*args: str) -> subprocess.CompletedProcess[str]:
+ return subprocess.run(
+ [sys.executable, *args],
+ cwd=ROOT,
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+
+
+def read_json(path: Path) -> dict:
+ return json.loads(path.read_text(encoding="utf-8"))
+
+
+def write_json(path: Path, data: dict) -> None:
+ path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+def copy_demo_room(tmp_path: Path, name: str = "room") -> Path:
+ room = tmp_path / name
+ shutil.copytree(BUILDROOM / "rooms" / "demo-room", room)
+ return room
+
+
+def test_demo_room_validates() -> None:
+ result = run_script(
+ "tools/kurultai/buildroom/scripts/validate_room.py",
+ "tools/kurultai/buildroom/rooms/demo-room",
+ )
+ assert result.returncode == 0, result.stderr
+ assert "validation passed" in result.stdout
+
+
+def test_corrupt_fixture_fails_with_path_error(tmp_path: Path) -> None:
+ room = copy_demo_room(tmp_path, "bad-room")
+ artifact = room / "ideas" / "idea-contract.json"
+ data = read_json(artifact)
+ del data["title"]
+ write_json(artifact, data)
+
+ result = run_script("tools/kurultai/buildroom/scripts/validate_room.py", str(room))
+
+ assert result.returncode == 1
+ assert "idea-contract.json" in result.stderr
+ assert "title" in result.stderr
+
+
+def test_validation_fails_when_contract_reference_points_at_missing_artifact(tmp_path: Path) -> None:
+ room = copy_demo_room(tmp_path, "missing-ref-room")
+ idea = read_json(room / "ideas" / "idea-contract.json")
+ idea["evidence_refs"].append("research/nonexistent-evidence.json")
+ write_json(room / "ideas" / "idea-contract.json", idea)
+
+ result = run_script("tools/kurultai/buildroom/scripts/validate_room.py", str(room))
+
+ assert result.returncode == 1
+ assert "idea-contract.json" in result.stderr
+ assert "evidence_refs" in result.stderr
+ assert "research/nonexistent-evidence.json" in result.stderr
+
+
+def test_operator_summary_builder_outputs_valid_summary(tmp_path: Path) -> None:
+ room = copy_demo_room(tmp_path)
+
+ result = run_script("tools/kurultai/buildroom/scripts/build_operator_summary.py", str(room))
+
+ assert result.returncode == 0, result.stderr
+ summary = read_json(room / "operator" / "operator-summary.json")
+ assert summary["room_id"] == "room"
+ assert summary["status"] == "clean"
+ assert any("Trust state" in item for item in summary["operator_needs_to_know"])
+
+ validation = run_script("tools/kurultai/buildroom/scripts/validate_room.py", str(room))
+ assert validation.returncode == 0, validation.stderr
+
+
+def test_operator_summary_includes_watch_followups_and_existing_artifact_links(tmp_path: Path) -> None:
+ room = copy_demo_room(tmp_path, "watch-room")
+ trust_path = room / "trust" / "trust-report.json"
+ trust = read_json(trust_path)
+ trust["state"] = "watch"
+ trust["risk_score"] = 0.42
+ trust["required_followups"] = ["Run independent QA before archive."]
+ write_json(trust_path, trust)
+
+ result = run_script("tools/kurultai/buildroom/scripts/build_operator_summary.py", str(room))
+
+ assert result.returncode == 0, result.stderr
+ summary = read_json(room / "operator" / "operator-summary.json")
+ assert summary["status"] == "watch"
+ assert "Run independent QA before archive." in summary["operator_decisions_needed"]
+ linked_artifacts = [link.removeprefix("buildroom://watch-room/") for link in summary["links"]]
+ assert linked_artifacts == summary["latest_artifacts"]
+ assert all((room / artifact).exists() for artifact in linked_artifacts)
+
+
+def test_sanitized_export_redacts_private_paths_and_secretish_values(tmp_path: Path) -> None:
+ room = copy_demo_room(tmp_path)
+ dest = tmp_path / "export"
+ research = read_json(room / "research" / "research-input.json")
+ sample_secret = "sk-dem" + "..." + "oken"
+ private_path = "/" + "Users" + "/" + "kublai" + "/private/path"
+ research["source_refs"].append(f"token={sample_secret} at {private_path}")
+ write_json(room / "research" / "research-input.json", research)
+
+ result = run_script(
+ "tools/kurultai/buildroom/scripts/export_sanitized_bundle.py",
+ str(room),
+ str(dest),
+ )
+
+ assert result.returncode == 0, result.stderr
+ exported = (dest / "research" / "research-input.json").read_text(encoding="utf-8")
+ assert sample_secret not in exported
+ assert "[REDACTED_ABSOLUTE_PATH]" in exported
+ assert "[REDACTED_SECRET]" in exported
+
+
+def test_sanitized_export_skips_private_artifacts_and_redacts_secret_fields(tmp_path: Path) -> None:
+ room = copy_demo_room(tmp_path)
+ dest = tmp_path / "export"
+ private_artifact = room / "research" / "private-evidence.json"
+ write_json(
+ private_artifact,
+ {
+ "sensitivity": "private",
+ "token": "***",
+ "path": "/private/operator/source.txt",
+ },
+ )
+ research = read_json(room / "research" / "research-input.json")
+ research["api_token"] = "SAMPLE_SECRET_VALUE"
+ write_json(room / "research" / "research-input.json", research)
+
+ result = run_script(
+ "tools/kurultai/buildroom/scripts/export_sanitized_bundle.py",
+ str(room),
+ str(dest),
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "skipped private artifact: research/private-evidence.json" in result.stdout
+ assert not (dest / "research" / "private-evidence.json").exists()
+ exported = read_json(dest / "research" / "research-input.json")
+ assert exported["api_token"] == "[REDACTED_SECRET_FIELD]"
+
+
+def test_approved_build_plan_creates_kanban_task_packet_with_parent_refs(tmp_path: Path) -> None:
+ room = copy_demo_room(tmp_path, "approved-room")
+ build_plan = read_json(room / "plans" / "build-plan.json")
+ build_plan["task_refs"] = ["kanban:t_parent_one", "kanban:t_parent_two", "brain:project-note"]
+ write_json(room / "plans" / "build-plan.json", build_plan)
+ packet_path = tmp_path / "task-packet.json"
+
+ result = run_script(
+ "tools/kurultai/buildroom/scripts/kanban_adapter.py",
+ "task-packet",
+ str(room),
+ str(packet_path),
+ )
+
+ assert result.returncode == 0, result.stderr
+ packet = read_json(packet_path)
+ assert packet["title"] == "buildroom: Buildroom foundation demo"
+ assert packet["assignee"] == "chagatai"
+ assert packet["parents"] == ["t_parent_one", "t_parent_two"]
+ assert packet["idempotency_key"] == "buildroom:build-buildroom-foundation-demo"
+ assert packet["metadata"]["build_id"] == "build-buildroom-foundation-demo"
+ assert "Allowed paths:" in packet["body"]
+ assert "kanban:t_parent_one" in packet["body"]
+
+
+def test_unapproved_idea_refuses_to_generate_kanban_task_packet(tmp_path: Path) -> None:
+ room = copy_demo_room(tmp_path, "unapproved-room")
+ main_review_path = room / "reviews" / "main-review.json"
+ review = read_json(main_review_path)
+ review["decision"] = "needs_more_evidence"
+ review["blocked_reasons"] = ["Missing operator approval."]
+ write_json(main_review_path, review)
+
+ result = run_script(
+ "tools/kurultai/buildroom/scripts/kanban_adapter.py",
+ "task-packet",
+ str(room),
+ str(tmp_path / "task-packet.json"),
+ )
+
+ assert result.returncode == 1
+ assert "main-review decision is not approved_for_planning" in result.stderr
+ assert "Missing operator approval" in result.stderr
+
+
+def test_completed_kanban_task_generates_implementation_receipt_with_evidence_refs(tmp_path: Path) -> None:
+ room = copy_demo_room(tmp_path, "receipt-room")
+ completion_path = tmp_path / "kanban-completion.json"
+ write_json(
+ completion_path,
+ {
+ "id": "t_done123",
+ "status": "done",
+ "assignee": "chagatai",
+ "started_at": "2026-05-10T17:20:00Z",
+ "completed_at": "2026-05-10T17:25:00Z",
+ "summary": "Implemented buildroom adapter with focused tests passing.",
+ "metadata": {
+ "changed_files": ["tools/kurultai/buildroom/scripts/kanban_adapter.py"],
+ "commands_run": ["python -m py_compile tools/kurultai/buildroom/scripts/kanban_adapter.py"],
+ "tests_run": ["pytest tests/kurultai/test_buildroom_foundation.py -q"],
+ "commit_sha": "abc1234",
+ "open_diffs_summary": "adapter script and tests",
+ "deviations_from_plan": [],
+ "blocked_items": [],
+ "evidence_refs": ["kanban-run:151", "artifact:task-packet.json"],
+ },
+ },
+ )
+
+ result = run_script(
+ "tools/kurultai/buildroom/scripts/kanban_adapter.py",
+ "receipt",
+ str(room),
+ str(completion_path),
+ )
+
+ assert result.returncode == 0, result.stderr
+ receipt = read_json(room / "jobs" / "implementation-receipt.json")
+ assert receipt["receipt_id"] == "receipt-t_done123"
+ assert receipt["build_id"] == "build-buildroom-foundation-demo"
+ assert receipt["kanban_task_id"] == "t_done123"
+ assert receipt["kanban_status"] == "done"
+ assert receipt["files_changed"] == ["tools/kurultai/buildroom/scripts/kanban_adapter.py"]
+ assert receipt["tests_run"] == ["pytest tests/kurultai/test_buildroom_foundation.py -q"]
+ assert receipt["evidence_refs"] == ["kanban:t_done123", "kanban-run:151", "artifact:task-packet.json"]
+
+ validation = run_script("tools/kurultai/buildroom/scripts/validate_room.py", str(room))
+ assert validation.returncode == 0, validation.stderr
+
+
+def test_independent_qa_task_packet_uses_build_plan_and_receipt_parent(tmp_path: Path) -> None:
+ room = copy_demo_room(tmp_path, "qa-room")
+ receipt = read_json(room / "jobs" / "implementation-receipt.json")
+ receipt["kanban_task_id"] = "t_build123"
+ write_json(room / "jobs" / "implementation-receipt.json", receipt)
+ packet_path = tmp_path / "qa-task-packet.json"
+
+ result = run_script(
+ "tools/kurultai/buildroom/scripts/qa_trust.py",
+ "qa-packet",
+ str(room),
+ str(packet_path),
+ )
+
+ assert result.returncode == 0, result.stderr
+ packet = read_json(packet_path)
+ assert packet["title"] == "buildroom-qa: Buildroom foundation demo"
+ assert packet["assignee"] == "ogedei"
+ assert packet["parents"] == ["t_build123"]
+ assert packet["idempotency_key"] == "buildroom-qa:build-buildroom-foundation-demo"
+ assert packet["metadata"]["build_id"] == "build-buildroom-foundation-demo"
+ assert packet["metadata"]["independent_of"] == "chagatai"
+ assert "Independent QA only" in packet["body"]
+ assert "python scripts/validate_room.py rooms/demo-room" in packet["body"]
+
+
+def test_verification_delta_confirms_passing_qa_against_receipt(tmp_path: Path) -> None:
+ room = copy_demo_room(tmp_path, "confirmed-room")
+
+ result = run_script("tools/kurultai/buildroom/scripts/qa_trust.py", "delta", str(room))
+
+ assert result.returncode == 0, result.stderr
+ delta = read_json(room / "verification" / "verification-delta.json")
+ assert delta["state"] == "confirmed"
+ assert "QA report passed" in delta["confirmed_claims"]
+ assert delta["unverified_claims"] == []
+ assert delta["regressions"] == []
+
+
+def test_trust_summary_emits_clean_for_passing_confirmed_room(tmp_path: Path) -> None:
+ room = copy_demo_room(tmp_path, "clean-room")
+ delta_result = run_script("tools/kurultai/buildroom/scripts/qa_trust.py", "delta", str(room))
+ assert delta_result.returncode == 0, delta_result.stderr
+
+ result = run_script("tools/kurultai/buildroom/scripts/qa_trust.py", "trust", str(room))
+
+ assert result.returncode == 0, result.stderr
+ trust = read_json(room / "trust" / "trust-report.json")
+ assert trust["state"] == "clean"
+ assert trust["safe_to_archive"] is True
+ assert trust["risk_score"] == 0.1
+ assert "QA report passed" in trust["reasons"]
+
+
+def test_missing_qa_emits_watch_delta_and_trust(tmp_path: Path) -> None:
+ room = copy_demo_room(tmp_path, "missing-qa-room")
+ (room / "verification" / "verification-report.json").unlink()
+
+ delta_result = run_script("tools/kurultai/buildroom/scripts/qa_trust.py", "delta", str(room))
+ trust_result = run_script("tools/kurultai/buildroom/scripts/qa_trust.py", "trust", str(room))
+
+ assert delta_result.returncode == 0, delta_result.stderr
+ assert trust_result.returncode == 0, trust_result.stderr
+ delta = read_json(room / "verification" / "verification-delta.json")
+ trust = read_json(room / "trust" / "trust-report.json")
+ assert delta["state"] == "missing_evidence"
+ assert "verification/verification-report.json missing" in delta["unverified_claims"]
+ assert trust["state"] == "watch"
+ assert trust["safe_to_archive"] is False
+ assert "Run independent QA and regenerate verification report." in trust["required_followups"]
+
+
+def test_scope_expansion_emits_investigate_trust(tmp_path: Path) -> None:
+ room = copy_demo_room(tmp_path, "scope-room")
+ receipt = read_json(room / "jobs" / "implementation-receipt.json")
+ receipt["files_changed"] = ["/etc/passwd", "tools/kurultai/buildroom/scripts/qa_trust.py"]
+ write_json(room / "jobs" / "implementation-receipt.json", receipt)
+
+ delta_result = run_script("tools/kurultai/buildroom/scripts/qa_trust.py", "delta", str(room))
+ trust_result = run_script("tools/kurultai/buildroom/scripts/qa_trust.py", "trust", str(room))
+
+ assert delta_result.returncode == 0, delta_result.stderr
+ assert trust_result.returncode == 0, trust_result.stderr
+ delta = read_json(room / "verification" / "verification-delta.json")
+ trust = read_json(room / "trust" / "trust-report.json")
+ assert delta["state"] == "regression"
+ assert any("outside allowed paths" in item for item in delta["regressions"])
+ assert trust["state"] == "investigate"
+ assert "Investigate scope/protected-path mismatch before archive." in trust["required_followups"]
+
+
+def test_failed_qa_regression_emits_investigate_trust(tmp_path: Path) -> None:
+ room = copy_demo_room(tmp_path, "regression-room")
+ report = read_json(room / "verification" / "verification-report.json")
+ report["pass"] = False
+ report["failures"] = ["pytest regression in buildroom validation"]
+ write_json(room / "verification" / "verification-report.json", report)
+
+ delta_result = run_script("tools/kurultai/buildroom/scripts/qa_trust.py", "delta", str(room))
+ trust_result = run_script("tools/kurultai/buildroom/scripts/qa_trust.py", "trust", str(room))
+
+ assert delta_result.returncode == 0, delta_result.stderr
+ assert trust_result.returncode == 0, trust_result.stderr
+ delta = read_json(room / "verification" / "verification-delta.json")
+ trust = read_json(room / "trust" / "trust-report.json")
+ assert delta["state"] == "regression"
+ assert "pytest regression in buildroom validation" in delta["regressions"]
+ assert trust["state"] == "investigate"
+ assert "Fix failed QA/regressions and rerun verifier." in trust["required_followups"]
+
+
+
+def test_research_to_buildroom_compiles_brain_artifacts(tmp_path: Path) -> None:
+ packet = tmp_path / "packet.md"
+ synthesis = tmp_path / "synthesis.md"
+ review = tmp_path / "review.md"
+ packet.write_text("---\ntitle: Example Research Packet\n---\n\nThis research packet says the system should convert useful research into bounded work.", encoding="utf-8")
+ synthesis.write_text("---\ntitle: Example Auto Think Synthesis\n---\n\nA reusable synthesis should become a buildroom room with review and operator summary.", encoding="utf-8")
+ review.write_text("---\ntitle: Example Review\n---\n\nClassification: durable-change-recommended", encoding="utf-8")
+ out = tmp_path / "rooms"
+ result = run_script(
+ "tools/kurultai/buildroom/scripts/research_to_buildroom.py",
+ "--room-id",
+ "example-research-room",
+ "--packet",
+ str(packet),
+ "--synthesis",
+ str(synthesis),
+ "--review",
+ str(review),
+ "--source-ref",
+ "https://example.com/research",
+ "--output-root",
+ str(out),
+ )
+ assert result.returncode == 0, result.stderr
+ room = out / "example-research-room"
+ assert (room / "research/research-input.json").exists()
+ assert read_json(room / "ideas/idea-contract.json")["title"] == "Example Auto Think Synthesis"
+ validate = run_script("tools/kurultai/buildroom/scripts/validate_room.py", str(room))
+ assert validate.returncode == 0, validate.stderr
+ summary = read_json(room / "operator/operator-summary.json")
+ assert summary["status"] == "watch"
+ assert "buildroom://example-research-room" in summary["links"]
+
+
+def test_control_room_report_summarizes_rooms(tmp_path: Path) -> None:
+ room = copy_demo_room(tmp_path)
+ output = tmp_path / "control-room.md"
+
+ result = run_script("tools/kurultai/buildroom/scripts/control_room.py", "--rooms-root", room.parent, "--output", output)
+
+ assert result.returncode == 0, result.stderr
+ report = output.read_text()
+ assert "# Kurultai Buildroom Control Room" in report
+ assert "### room" in report
+ assert "Trust: clean" in report
+ assert "Missing evidence: none" in report
+ assert "Next action: Ready for PR/merge monitoring or retention follow-up." in report
+
+
+def test_control_room_report_flags_missing_evidence(tmp_path: Path) -> None:
+ room = copy_demo_room(tmp_path)
+ (room / "verification/verification-report.json").unlink()
+ output = tmp_path / "control-room.md"
+
+ result = run_script("tools/kurultai/buildroom/scripts/control_room.py", "--rooms-root", room.parent, "--output", output)
+
+ assert result.returncode == 0, result.stderr
+ report = output.read_text()
+ assert "Rooms with missing evidence: 1" in report
+ assert "Phase: verification" in report
+ assert "Missing evidence: verification/verification-report.json" in report
+ assert "Next action: Fill missing artifact: verification/verification-report.json." in report
+
diff --git a/tests/phase1/test_knowledge_and_service.py b/tests/phase1/test_knowledge_and_service.py
new file mode 100644
index 00000000..dabe8f77
--- /dev/null
+++ b/tests/phase1/test_knowledge_and_service.py
@@ -0,0 +1,380 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import sys
+import threading
+import uuid
+from pathlib import Path
+
+import pytest
+
+from kublai.brain_service import BrainService, classify_page
+from kublai.brain_service_client import call
+from kublai.knowledge import KnowledgeStore, PathPolicyError
+
+
+NODE_BIN = shutil.which("node") or "/opt/homebrew/bin/node"
+
+
+def test_classify_page_marks_hard_private_sources():
+ assert classify_page("hard-private/finance/x.md", {}) == "hard-private"
+ assert classify_page("entities/contact.md", {"type": "human-contact"}) == "hard-private"
+ assert classify_page("raw/tax-note.md", {"tags": ["tax"]}) == "hard-private"
+ assert classify_page("concepts/private-note.md", {"tags": ["research"]}) == "private"
+ assert classify_page("concepts/public-note.md", {"publish": True}) == "public"
+
+
+def test_knowledge_write_is_deterministic_and_path_guarded(tmp_path):
+ store = KnowledgeStore(tmp_path / "brain")
+ path = store.record_completed_task(
+ task_id="abc-123",
+ agent="temujin",
+ delegated_by="kublai",
+ completed_at_ms=1_775_000_000_000,
+ deliverable="done",
+ results={"ok": True},
+ )
+ same = store.record_completed_task(
+ task_id="abc-123",
+ agent="temujin",
+ delegated_by="kublai",
+ completed_at_ms=1_775_000_000_000,
+ deliverable="done again",
+ results={"ok": True},
+ )
+
+ assert path == same
+ assert path.relative_to(tmp_path / "brain").as_posix().startswith("operations/tasks/")
+ assert "task_id: abc-123" in path.read_text()
+ with pytest.raises(PathPolicyError):
+ store.write_page("../escape.md", {}, "bad", typed_field="x", typed_id="y")
+
+
+def test_sweep_materializes_completed_task_once(tmp_path):
+ service = BrainService(
+ tmp_path / "brain",
+ tmp_path / "telemetry.db",
+ tmp_path / "brain-index.db",
+ )
+ service.telemetry.create_task("task-1", description="sweep", delegated_by="kublai")
+ claim = service.telemetry.claim_task("temujin", now_ms=1_000)
+ service.telemetry.complete_task(
+ claim.id,
+ "temujin",
+ claim.claim_token,
+ summary="materialize me",
+ target_wiki_path="operations/tasks/2026/04/task-task-1.md",
+ now_ms=1_100,
+ )
+
+ assert service.sweep() == 1
+ assert service.sweep() == 0
+ task_pages = list((tmp_path / "brain" / "operations/tasks").rglob("*.md"))
+ assert len(task_pages) == 1
+
+
+def test_reindex_and_vector_orphan_check(tmp_path):
+ service = BrainService(
+ tmp_path / "brain",
+ tmp_path / "telemetry.db",
+ tmp_path / "brain-index.db",
+ )
+ for i in range(1_000):
+ page = service.knowledge.record_reflection(
+ agent="jochi",
+ reflection_id=f"reflection-{i}",
+ body=f"reflection body {i}",
+ )
+ page.write_text(page.read_text().replace("updated:", "publish: true\nupdated:", 1))
+
+ assert service.reindex() == 1_000
+ assert service.index.vector_orphans() == 0
+
+ with service.index.connect() as conn:
+ conn.execute("DELETE FROM nodes WHERE node_pk IN (SELECT node_pk FROM nodes LIMIT 10)")
+ assert service.index.vector_orphans() == 10
+
+
+def test_public_reindex_skips_hard_private_before_body_index(tmp_path):
+ brain = tmp_path / "brain"
+ public_dir = brain / "concepts"
+ hard_dir = brain / "hard-private"
+ public_dir.mkdir(parents=True)
+ hard_dir.mkdir(parents=True)
+ (public_dir / "published.md").write_text(
+ "---\ntype: concept\nstatus: active\ncreated: 2026-05-03\nupdated: 2026-05-03\nsources: 1\npublish: true\ntags: [research]\n---\n\npublic sentinel\n",
+ encoding="utf-8",
+ )
+ (public_dir / "private.md").write_text(
+ "---\ntype: concept\nstatus: active\ncreated: 2026-05-03\nupdated: 2026-05-03\nsources: 1\ntags: [research]\n---\n\nordinary private sentinel\n",
+ encoding="utf-8",
+ )
+ (hard_dir / "secret.md").write_text(
+ "---\ntype: note\nstatus: active\ncreated: 2026-05-03\nupdated: 2026-05-03\nsources: 1\ntags: [financial]\n---\n\nhard private sentinel\n",
+ encoding="utf-8",
+ )
+ service = BrainService(brain, tmp_path / "telemetry.db", tmp_path / "brain-index.db")
+
+ assert service.reindex() == 1
+ assert service.verify_index()["ok"] is True
+ assert service.search(query="public")[0]["rel_path"] == "concepts/published.md"
+ assert service.search(query="ordinary") == []
+ assert service.search(query="hard") == []
+
+
+def test_search_treats_hyphenated_query_as_literal_text(tmp_path):
+ service = BrainService(
+ tmp_path / "brain",
+ tmp_path / "telemetry.db",
+ tmp_path / "brain-index.db",
+ )
+
+ assert service.search(query="hard-private") == []
+
+
+def test_verify_index_fails_when_public_db_contains_hard_private_row(tmp_path):
+ service = BrainService(
+ tmp_path / "brain",
+ tmp_path / "telemetry.db",
+ tmp_path / "brain-index.db",
+ )
+ service.knowledge.record_reflection(
+ agent="jochi",
+ reflection_id="reflection-privacy-check",
+ body="fixture reflection",
+ )
+ service.reindex()
+ with service.index.connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO nodes (
+ id, type, rel_path, title, status, agent, typed_id,
+ created, updated, frontmatter, body_hash, body_text, mtime_ns
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ """,
+ (
+ "hard-private/test",
+ "note",
+ "hard-private/test.md",
+ "private",
+ "active",
+ "",
+ None,
+ "2026-05-03",
+ "2026-05-03",
+ json.dumps({"type": "note", "tags": ["financial"]}),
+ "hash",
+ "secret",
+ 1,
+ ),
+ )
+
+ result = service.verify_index()
+ assert result["ok"] is False
+ assert result["hard_private_rows"] == 1
+
+
+def test_verify_index_fails_when_public_db_contains_private_row(tmp_path):
+ service = BrainService(
+ tmp_path / "brain",
+ tmp_path / "telemetry.db",
+ tmp_path / "brain-index.db",
+ )
+ with service.index.connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO nodes (
+ id, type, rel_path, title, status, agent, typed_id,
+ created, updated, frontmatter, body_hash, body_text, mtime_ns
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ """,
+ (
+ "concepts/private",
+ "concept",
+ "concepts/private.md",
+ "private",
+ "active",
+ "",
+ None,
+ "2026-05-03",
+ "2026-05-03",
+ json.dumps({"type": "concept", "tags": ["research"]}),
+ "hash",
+ "private",
+ 1,
+ ),
+ )
+
+ result = service.verify_index()
+ assert result["ok"] is False
+ assert result["private_rows"] == 1
+
+
+def test_list_by_tag_rpc_defaults_to_public_scope(tmp_path):
+ brain = tmp_path / "brain"
+ (brain / "concepts").mkdir(parents=True)
+ (brain / "hard-private").mkdir(parents=True)
+ (brain / "concepts" / "safe.md").write_text(
+ "---\ntype: concept\npublish: true\ntags: [finance]\n---\n\npublic finance\n",
+ encoding="utf-8",
+ )
+ (brain / "hard-private" / "secret.md").write_text(
+ "---\ntype: note\ntags: [finance]\n---\n\nsecret finance\n",
+ encoding="utf-8",
+ )
+ (brain / "concepts" / "sensitive.md").write_text(
+ "---\ntype: note\ntags: [finance, financial]\n---\n\nsensitive finance\n",
+ encoding="utf-8",
+ )
+ service = BrainService(brain, tmp_path / "telemetry.db", tmp_path / "brain-index.db")
+
+ public_default = service.index.list_by_tag_in_wiki(service.knowledge, tag="finance")
+ hard_private = service.index.list_by_tag_in_wiki(
+ service.knowledge,
+ tag="finance",
+ privacy_scope="hard-private",
+ )
+ assert [row["rel_path"] for row in public_default] == ["concepts/safe.md"]
+ assert {row["rel_path"] for row in hard_private} == {
+ "concepts/sensitive.md",
+ "hard-private/secret.md",
+ }
+
+
+def test_private_indexer_indexes_only_hard_private_and_audits_reads(tmp_path, monkeypatch):
+ brain = tmp_path / "brain"
+ (brain / "concepts").mkdir(parents=True)
+ (brain / "hard-private").mkdir(parents=True)
+ (brain / "concepts" / "safe.md").write_text(
+ "---\ntype: concept\npublish: true\ntags: [research]\n---\n\npublic apple\n",
+ encoding="utf-8",
+ )
+ (brain / "hard-private" / "secret.md").write_text(
+ "---\ntype: note\ntags: [financial]\n---\n\nprivate banana\n",
+ encoding="utf-8",
+ )
+ monkeypatch.setenv("BRAIN_PRIVATE_INDEX_DB", str(tmp_path / "private.db"))
+ monkeypatch.setenv("KUBLAI_PRIVATE_ACCESS_LOG", str(tmp_path / "private-access.ndjson"))
+ service = BrainService(brain, tmp_path / "telemetry.db", tmp_path / "brain-index.db")
+
+ assert service.reindex() == 1
+ assert service.reindex_private() == 1
+ assert service.verify_index()["ok"] is True
+ assert service.verify_private_index()["ok"] is True
+ assert service.search(query="banana") == []
+ rows = service.search_private(query="banana")
+ assert [row["rel_path"] for row in rows] == ["hard-private/secret.md"]
+ assert "knowledge.search_private" in (tmp_path / "private-access.ndjson").read_text()
+
+
+def test_unix_socket_rpc_serves_js_clients(tmp_path):
+ service = BrainService(
+ tmp_path / "brain",
+ tmp_path / "telemetry.db",
+ tmp_path / "brain-index.db",
+ )
+ # macOS Unix sockets have a short path limit; keep this in /tmp even when
+ # pytest's tmp_path is deeply nested.
+ socket_path = Path("/tmp") / f"bs-{os.getpid()}-{uuid.uuid4().hex[:8]}.sock"
+ server = service.serve_socket(socket_path)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ created = call(
+ socket_path,
+ "telemetry.create_task",
+ {"task_id": "task-1", "description": "rpc", "delegated_by": "kublai"},
+ )
+ assert created == {"ok": True, "result": "task-1"}
+
+ node_script = f"""
+ const telemetry = require('./kublai/telemetry-client');
+ process.env.BRAIN_SERVICE_SOCKET = {json.dumps(str(socket_path))};
+ telemetry.claimTask({{agent: 'temujin'}})
+ .then(result => {{
+ console.log(JSON.stringify({{id: result.id, token: !!result.claim_token}}));
+ }})
+ .catch(error => {{
+ console.error(error.stack || error.message);
+ process.exit(1);
+ }});
+ """
+ result = subprocess.run(
+ [NODE_BIN, "-e", node_script],
+ cwd=".",
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ assert json.loads(result.stdout) == {"id": "task-1", "token": True}
+
+ found = call(
+ socket_path,
+ "knowledge.search",
+ {"query": "rpc", "limit": 5},
+ )
+ assert found == {"ok": True, "result": []}
+ finally:
+ server.shutdown()
+ server.server_close()
+ if socket_path.exists():
+ socket_path.unlink()
+
+
+def test_brain_service_cli_healthcheck(tmp_path):
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-m",
+ "kublai.brain_service",
+ "--wiki-root",
+ str(tmp_path / "brain"),
+ "--telemetry-db",
+ str(tmp_path / "telemetry.db"),
+ "--index-db",
+ str(tmp_path / "brain-index.db"),
+ "healthcheck",
+ ],
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ payload = json.loads(result.stdout)
+ assert payload["ok"] is True
+
+
+def test_brain_service_verify_index_and_replay_dual_write(tmp_path):
+ service = BrainService(
+ tmp_path / "brain",
+ tmp_path / "telemetry.db",
+ tmp_path / "brain-index.db",
+ )
+ page = service.knowledge.record_reflection(
+ agent="jochi",
+ reflection_id="reflection-1",
+ body="fixture reflection",
+ )
+ page.write_text(page.read_text().replace("updated:", "publish: true\nupdated:", 1))
+ service.reindex()
+
+ assert service.verify_index()["ok"] is True
+ assert service.get_node("reflection", "reflection-1")["rel_path"].startswith("operations/reflections/")
+ assert service.list_nodes(node_type="reflection")[0]["typed_id"] == "reflection-1"
+ assert service.search(query="fixture", node_type="reflection")[0]["typed_id"] == "reflection-1"
+
+ rel_path = page.relative_to(tmp_path / "brain").as_posix()
+ body = page.read_text().split("\n---\n", 1)[1]
+ digest = hashlib.sha256(body.encode("utf-8")).hexdigest()
+ log_path = tmp_path / "dual-write.jsonl"
+ log_path.write_text(json.dumps({"wiki_path": rel_path, "body_hash": digest}) + "\n")
+
+ assert service.replay_dual_write(log_path) == {
+ "ok": True,
+ "checked": 1,
+ "mismatches": 0,
+ "details": [],
+ }
diff --git a/tests/phase1/test_telemetry.py b/tests/phase1/test_telemetry.py
new file mode 100644
index 00000000..7466da29
--- /dev/null
+++ b/tests/phase1/test_telemetry.py
@@ -0,0 +1,113 @@
+import threading
+
+import pytest
+
+from kublai.telemetry import NoPendingTaskError, StaleClaimError, TelemetryStore
+
+
+def test_claim_race_has_single_winner(tmp_path):
+ store = TelemetryStore(tmp_path / "telemetry.db")
+ store.create_task("task-1", description="race", delegated_by="kublai")
+ winners = []
+ errors = []
+
+ def worker(i):
+ try:
+ winners.append(store.claim_task(f"agent-{i}", lease_ttl_ms=10_000))
+ except NoPendingTaskError as exc:
+ errors.append(exc)
+
+ threads = [threading.Thread(target=worker, args=(i,)) for i in range(100)]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join()
+
+ assert len(winners) == 1
+ assert len(errors) == 99
+
+
+def test_fencing_rejects_stale_worker_after_lease_expiry(tmp_path):
+ store = TelemetryStore(tmp_path / "telemetry.db")
+ store.create_task("task-1", description="fenced", delegated_by="kublai")
+
+ first = store.claim_task("agent-a", lease_ttl_ms=100, now_ms=1_000)
+ assert store.sweep_expired_claims(now_ms=1_101) == 1
+ second = store.claim_task("agent-b", lease_ttl_ms=100, now_ms=1_102)
+
+ with pytest.raises(StaleClaimError):
+ store.complete_task(
+ first.id,
+ "agent-a",
+ first.claim_token,
+ summary="stale",
+ now_ms=1_103,
+ )
+
+ store.complete_task(
+ second.id,
+ "agent-b",
+ second.claim_token,
+ summary="fresh",
+ now_ms=1_104,
+ )
+
+
+def test_renew_claim_extends_lease_and_increments_version(tmp_path):
+ store = TelemetryStore(tmp_path / "telemetry.db")
+ store.create_task("task-1", description="renew", delegated_by="kublai")
+ claim = store.claim_task("agent-a", lease_ttl_ms=300, now_ms=1_000)
+
+ for i in range(1, 6):
+ claim = store.renew_claim(
+ claim.id,
+ "agent-a",
+ claim.claim_token,
+ lease_ttl_ms=300,
+ now_ms=1_000 + i * 100,
+ )
+ assert claim.lease_version == i + 1
+ assert claim.expires_at == 1_000 + i * 100 + 300
+
+
+def test_online_backup_restores_task_state(tmp_path):
+ store = TelemetryStore(tmp_path / "telemetry.db")
+ store.create_task("task-1", description="backup", delegated_by="kublai")
+ backup = store.backup_to(tmp_path / "backup.db")
+
+ restored = TelemetryStore(backup)
+ with restored.connect() as conn:
+ row = conn.execute("SELECT id, status FROM in_flight_tasks").fetchone()
+ assert dict(row) == {"id": "task-1", "status": "pending"}
+
+
+def test_dashboard_statuses_normalize_to_claimable_lowercase(tmp_path):
+ store = TelemetryStore(tmp_path / "telemetry.db")
+ created = store.create_task_full(
+ task_id="task-dashboard",
+ title="Dashboard task",
+ prompt="do work",
+ assigned_to="temujin",
+ status="PENDING",
+ )
+ assert created == {"task_id": "task-dashboard"}
+
+ with store.connect() as conn:
+ row = conn.execute("SELECT status FROM in_flight_tasks WHERE id = ?", ("task-dashboard",)).fetchone()
+ assert row["status"] == "pending"
+
+ claim = store.claim_task("temujin", lease_ttl_ms=300, now_ms=1_000)
+ assert claim.id == "task-dashboard"
+
+
+def test_retry_resets_to_claimable_lowercase_pending(tmp_path):
+ store = TelemetryStore(tmp_path / "telemetry.db")
+ store.create_task("task-retry", description="retry", delegated_by="kublai")
+ claim = store.claim_task("temujin", lease_ttl_ms=300, now_ms=1_000)
+ store.complete_task(claim.id, "temujin", claim.claim_token, summary="done", now_ms=1_010)
+
+ result = store.retry_task(task_id="task-retry", agent="temujin")
+ assert result["task_id"] == "task-retry"
+ with store.connect() as conn:
+ row = conn.execute("SELECT status FROM in_flight_tasks WHERE id = ?", ("task-retry",)).fetchone()
+ assert row["status"] == "pending"
diff --git a/tests/phase2/test_knowledge_migration.py b/tests/phase2/test_knowledge_migration.py
new file mode 100644
index 00000000..b2ef46ed
--- /dev/null
+++ b/tests/phase2/test_knowledge_migration.py
@@ -0,0 +1,69 @@
+import json
+from datetime import datetime, timezone
+
+from kublai.brain_service import BrainService
+from kublai.knowledge import KnowledgeStore
+from tools.golden_query_benchmark import run_benchmark
+from tools.neo4j_knowledge_export import materialize_record
+
+
+def test_exporter_materializes_historical_reflection_and_task(tmp_path):
+ store = KnowledgeStore(tmp_path / "brain")
+
+ reflection = materialize_record(
+ store,
+ "reflection",
+ {
+ "id": "reflection-1",
+ "agent": "temujin",
+ "context": "bad deploy",
+ "lesson": "run fixtures first",
+ "created_at": datetime(2026, 4, 1, tzinfo=timezone.utc),
+ },
+ ["Reflection"],
+ )
+ task = materialize_record(
+ store,
+ "task",
+ {
+ "id": "task-1",
+ "status": "completed",
+ "assigned_to": "jochi",
+ "delegated_by": "kublai",
+ "summary": "done",
+ "completed_at": datetime(2026, 4, 2, tzinfo=timezone.utc),
+ "results": {"ok": True},
+ },
+ ["Task"],
+ )
+
+ assert "historical: true" in reflection.read_text()
+ assert "reflection_id: reflection-1" in reflection.read_text()
+ assert task.relative_to(tmp_path / "brain").as_posix() == "operations/tasks/2026/04/task-task-1.md"
+ assert "historical: true" in task.read_text()
+
+
+def test_golden_query_benchmark_writes_and_compares_baseline(tmp_path):
+ service = BrainService(tmp_path / "brain", tmp_path / "telemetry.db", tmp_path / "index.db")
+ service.knowledge.record_reflection(agent="jochi", reflection_id="r1", body="security audit lesson")
+ service.reindex()
+
+ baseline = tmp_path / "golden.json"
+ written = run_benchmark(
+ service=service,
+ queries=["security audit"],
+ baseline_path=baseline,
+ top_k=10,
+ write_baseline=True,
+ )
+ compared = run_benchmark(
+ service=service,
+ queries=["security audit"],
+ baseline_path=baseline,
+ top_k=10,
+ write_baseline=False,
+ )
+
+ assert written["mode"] == "baseline-written"
+ assert compared["average_rank_overlap"] == 1.0
+ assert json.loads(baseline.read_text())["results"]["security audit"] == ["r1"]
diff --git a/tests/phase3/test_telemetry_migration.py b/tests/phase3/test_telemetry_migration.py
new file mode 100644
index 00000000..a5916293
--- /dev/null
+++ b/tests/phase3/test_telemetry_migration.py
@@ -0,0 +1,294 @@
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+import threading
+import time
+import uuid
+from pathlib import Path
+
+import pytest
+
+from kublai.brain_service import BrainService
+from kublai.brain_service_client import call
+from kublai.telemetry import NoPendingTaskError, StaleClaimError, TelemetryStore
+from openclaw_memory import NoPendingTaskError as LegacyNoPendingTaskError
+from openclaw_memory import OperationalMemory
+
+
+NODE_BIN = shutil.which("node") or "/opt/homebrew/bin/node"
+
+
+def test_operational_memory_telemetry_mode_keeps_core_signatures(tmp_path, monkeypatch):
+ monkeypatch.setenv("KUBLAI_TELEMETRY_PRIMARY", "1")
+ monkeypatch.setenv("KUBLAI_TELEMETRY_DB", str(tmp_path / "telemetry.db"))
+
+ memory = OperationalMemory(password=None)
+ task_id = memory.create_task(
+ "analysis",
+ "migrate telemetry",
+ delegated_by="kublai",
+ assigned_to="temujin",
+ priority="high",
+ )
+
+ claimed = memory.claim_task("temujin")
+ assert claimed["id"] == task_id
+ assert claimed["priority"] == "high"
+ assert claimed["claim_token"]
+
+ notification_id = memory.create_notification("kublai", "task_completed", "done", task_id=task_id)
+ assert memory.get_notifications("kublai", unread_only=True)[0]["id"] == notification_id
+ assert memory.mark_notification_read(notification_id) is True
+ assert memory.mark_notification_read("missing-notification") is False
+ assert memory.get_notifications("kublai", unread_only=True) == []
+
+ assert memory.complete_task(task_id, {"ok": True}) is True
+ assert memory.update_agent_heartbeat("temujin", "active") is True
+ assert memory.get_agent_status("temujin")["status"] == "active"
+
+
+def test_operational_memory_telemetry_claim_race_has_single_winner(tmp_path, monkeypatch):
+ monkeypatch.setenv("KUBLAI_TELEMETRY_PRIMARY", "1")
+ monkeypatch.setenv("KUBLAI_TELEMETRY_DB", str(tmp_path / "telemetry.db"))
+ memory = OperationalMemory(password=None)
+ memory.create_task("analysis", "race", delegated_by="kublai", assigned_to="any")
+
+ winners = []
+ errors = []
+
+ def worker(i):
+ try:
+ winners.append(memory.claim_task(f"agent-{i}"))
+ except LegacyNoPendingTaskError as exc:
+ errors.append(exc)
+
+ threads = [threading.Thread(target=worker, args=(i,)) for i in range(100)]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join()
+
+ assert len(winners) == 1
+ assert len(errors) == 99
+
+
+def test_telemetry_lease_fencing_and_renewal(tmp_path):
+ store = TelemetryStore(tmp_path / "telemetry.db")
+ store.create_task("task-1", description="fenced", delegated_by="kublai")
+ first = store.claim_task("agent-a", lease_ttl_ms=90, now_ms=1_000)
+
+ renewed = first
+ for i in range(1, 6):
+ renewed = store.renew_claim(
+ renewed.id,
+ "agent-a",
+ renewed.claim_token,
+ lease_ttl_ms=90,
+ now_ms=1_000 + i * 30,
+ )
+ assert renewed.lease_version == i + 1
+
+ assert store.sweep_expired_claims(now_ms=1_000 + 6 * 90) == 1
+ second = store.claim_task("agent-b", lease_ttl_ms=90, now_ms=1_000 + 6 * 90 + 1)
+ with pytest.raises(StaleClaimError):
+ store.complete_task(first.id, "agent-a", first.claim_token, summary="stale", now_ms=1_000 + 6 * 90 + 2)
+ store.complete_task(second.id, "agent-b", second.claim_token, summary="fresh", now_ms=1_000 + 6 * 90 + 3)
+
+
+def test_telemetry_mid_claim_process_death_releases_task(tmp_path):
+ db_path = tmp_path / "telemetry.db"
+ store = TelemetryStore(db_path)
+ store.create_task("task-1", description="kill", delegated_by="kublai")
+
+ script = f"""
+import os, signal
+from kublai.telemetry import TelemetryStore
+store = TelemetryStore({str(db_path)!r})
+store.claim_task("agent-a", lease_ttl_ms=50, now_ms=1000)
+os.kill(os.getpid(), signal.SIGKILL)
+"""
+ result = subprocess.run([sys.executable, "-c", script], env={**os.environ, "PYTHONPATH": "."})
+ assert result.returncode != 0
+
+ recovered = TelemetryStore(db_path)
+ assert recovered.sweep_expired_claims(now_ms=1_051) == 1
+ assert recovered.claim_task("agent-b", lease_ttl_ms=50, now_ms=1_052).id == "task-1"
+
+
+def test_heartbeat_and_rate_limit_under_load(tmp_path):
+ store = TelemetryStore(tmp_path / "telemetry.db")
+ agents = ["kublai", "temujin", "jochi", "mongke", "ogedei", "chagatai"]
+ start = int(time.time() * 1000)
+ for tick in range(120):
+ for agent in agents:
+ store.heartbeat(agent, status="active", meta={"tick": tick})
+ with store.connect() as conn:
+ assert conn.execute("SELECT count(*) FROM agent_state").fetchone()[0] == len(agents)
+
+ for i in range(500):
+ assert store.increment_rate_limit("temujin", "api", window_ms=60_000, now_ms=start + i) == i + 1
+ with store.connect() as conn:
+ row = conn.execute("SELECT count, bucket_start_ms FROM rate_limits").fetchone()
+ assert row["count"] == 500
+ assert row["bucket_start_ms"] == (start // 60_000) * 60_000
+
+
+def test_single_host_atomicity_stress_has_no_sqlite_busy(tmp_path):
+ store = TelemetryStore(tmp_path / "telemetry.db")
+ busy_errors = []
+
+ for round_id in range(20):
+ store.create_task(f"task-{round_id}", description="stress", delegated_by="kublai")
+ winners = []
+
+ def worker(i):
+ try:
+ winners.append(store.claim_task(f"agent-{i}", lease_ttl_ms=10_000))
+ except NoPendingTaskError:
+ pass
+ except Exception as exc: # pragma: no cover - assertion reports the concrete leak
+ busy_errors.append(exc)
+
+ threads = [threading.Thread(target=worker, args=(i,)) for i in range(100)]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join()
+ assert len(winners) == 1
+
+ assert busy_errors == []
+
+
+def test_wiki_write_failure_injection_recovers_without_duplicate_page(tmp_path, monkeypatch):
+ service = BrainService(tmp_path / "brain", tmp_path / "telemetry.db", tmp_path / "brain-index.db")
+ service.telemetry.create_task("task-1", description="recover", delegated_by="kublai")
+ claim = service.telemetry.claim_task("temujin", now_ms=1_000)
+ service.telemetry.complete_task(
+ claim.id,
+ "temujin",
+ claim.claim_token,
+ summary="recover me",
+ target_wiki_path="operations/tasks/2026/04/task-task-1.md",
+ now_ms=1_100,
+ )
+
+ original = service.knowledge.record_completed_task
+ failures = {"count": 0}
+
+ def fail_once(**kwargs):
+ failures["count"] += 1
+ if failures["count"] == 1:
+ raise OSError("injected wiki write failure")
+ return original(**kwargs)
+
+ monkeypatch.setattr(service.knowledge, "record_completed_task", fail_once)
+
+ assert service.sweep() == 0
+ with service.telemetry.connect() as conn:
+ row = conn.execute(
+ "SELECT completion_attempt_count, last_error, materialized_at FROM in_flight_tasks WHERE id = ?",
+ ("task-1",),
+ ).fetchone()
+ assert row["completion_attempt_count"] == 1
+ assert "injected wiki write failure" in row["last_error"]
+ assert row["materialized_at"] is None
+
+ assert service.sweep() == 1
+ assert service.sweep() == 0
+ task_pages = list((tmp_path / "brain" / "operations/tasks").rglob("*.md"))
+ assert len(task_pages) == 1
+
+
+def test_disk_full_injection_surfaces_materialization_alert(tmp_path, monkeypatch):
+ service = BrainService(tmp_path / "brain", tmp_path / "telemetry.db", tmp_path / "brain-index.db")
+ service.telemetry.create_task("task-1", description="disk full", delegated_by="kublai")
+ claim = service.telemetry.claim_task("temujin", now_ms=1_000)
+ service.telemetry.complete_task(
+ claim.id,
+ "temujin",
+ claim.claim_token,
+ summary="cannot write",
+ target_wiki_path="operations/tasks/2026/04/task-task-1.md",
+ now_ms=1_100,
+ )
+
+ def disk_full(*_args, **_kwargs):
+ raise OSError("No space left on device")
+
+ monkeypatch.setattr(service.knowledge, "atomic_write", disk_full)
+
+ for _ in range(3):
+ assert service.sweep() == 0
+
+ health = service.health()
+ assert health["ok"] is False
+ assert health["materialization_errors"] == 1
+ assert health["materialization_alerts"] == 1
+
+
+def test_js_clients_use_local_unix_socket_for_telemetry_and_knowledge(tmp_path):
+ service = BrainService(tmp_path / "brain", tmp_path / "telemetry.db", tmp_path / "brain-index.db")
+ service.knowledge.record_reflection(agent="kublai", reflection_id="reflection-1", body="socket lookup")
+ service.reindex()
+
+ socket_path = Path("/tmp") / f"bs-phase3-{os.getpid()}-{uuid.uuid4().hex[:8]}.sock"
+ server = service.serve_socket(socket_path)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ assert call(
+ socket_path,
+ "telemetry.create_task",
+ {"task_id": "task-1", "description": "rpc", "delegated_by": "kublai"},
+ ) == {"ok": True, "result": "task-1"}
+
+ node_script = f"""
+ const telemetry = require('./kublai/telemetry-client');
+ const knowledge = require('./kublai/knowledge-client');
+ process.env.BRAIN_SERVICE_SOCKET = {json.dumps(str(socket_path))};
+
+ async function main() {{
+ const samples = [];
+ for (let i = 0; i < 20; i++) {{
+ const started = process.hrtime.bigint();
+ await telemetry.heartbeat({{agent: 'temujin', status: 'active'}});
+ samples.push(Number(process.hrtime.bigint() - started) / 1e6);
+ }}
+ const claimed = await telemetry.claimTask({{agent: 'temujin'}});
+ const found = await knowledge.get({{node_type: 'reflection', typed_id: 'reflection-1'}});
+ samples.sort((a, b) => a - b);
+ const p99 = samples[samples.length - 1];
+ console.log(JSON.stringify({{
+ claimed: claimed.id,
+ hasToken: !!claimed.claim_token,
+ reflection: found && found.typed_id,
+ p99
+ }}));
+ }}
+
+ main().catch(error => {{
+ console.error(error.stack || error.message);
+ process.exit(1);
+ }});
+ """
+ result = subprocess.run(
+ [NODE_BIN, "-e", node_script],
+ cwd=".",
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ payload = json.loads(result.stdout)
+ assert payload["claimed"] == "task-1"
+ assert payload["hasToken"] is True
+ assert payload["reflection"] == "reflection-1"
+ assert payload["p99"] < 50
+ finally:
+ server.shutdown()
+ server.server_close()
+ if socket_path.exists():
+ socket_path.unlink()
diff --git a/tests/privacy/test_phase0_foundation.py b/tests/privacy/test_phase0_foundation.py
new file mode 100644
index 00000000..4902af10
--- /dev/null
+++ b/tests/privacy/test_phase0_foundation.py
@@ -0,0 +1,52 @@
+import json
+
+import pytest
+
+from kublai.cache import cache_key
+from kublai.graph_adapter import GraphifyAdapter
+from kublai.schema_registry import SchemaError, SchemaRegistry
+
+
+def test_cache_key_ignores_frontmatter_when_body_same():
+ prompt_hash = "prompt"
+ assert cache_key("body", "claude-opus", prompt_hash) == cache_key("body", "claude-opus", prompt_hash)
+ assert cache_key("body", "claude-opus", prompt_hash) != cache_key("body", "claude-haiku", prompt_hash)
+
+
+def test_graph_adapter_normalizes_nodes_and_edges(tmp_path):
+ graph = tmp_path / "graph.json"
+ graph.write_text(
+ json.dumps(
+ {
+ "nodes": [{"id": "a", "label": "A", "community": 1}],
+ "links": [{"source": "a", "target": "b", "relation": "mentions", "weight": 0.5}],
+ }
+ ),
+ encoding="utf-8",
+ )
+ normalized = GraphifyAdapter().normalize(graph)
+ assert normalized.nodes[0].id == "a"
+ assert normalized.edges[0].relation == "mentions"
+
+
+def test_schema_registry_validates_required_flat_frontmatter(tmp_path):
+ schema_dir = tmp_path / ".schemas" / "default"
+ schema_dir.mkdir(parents=True)
+ (schema_dir / "v1.schema.json").write_text(
+ json.dumps({"required": ["type", "status", "updated", "created", "sources", "tags"], "flat_only": ["tags"]}),
+ encoding="utf-8",
+ )
+ registry = SchemaRegistry(tmp_path / ".schemas")
+ registry.validate_frontmatter(
+ "concept",
+ {
+ "type": "concept",
+ "status": "active",
+ "updated": "2026-05-03",
+ "created": "2026-05-03",
+ "sources": 1,
+ "tags": ["x"],
+ },
+ )
+ with pytest.raises(SchemaError):
+ registry.validate_frontmatter("concept", {"type": "concept"})
diff --git a/tests/privacy/test_sanitizer_and_llm.py b/tests/privacy/test_sanitizer_and_llm.py
new file mode 100644
index 00000000..9fd4c72f
--- /dev/null
+++ b/tests/privacy/test_sanitizer_and_llm.py
@@ -0,0 +1,166 @@
+import asyncio
+import json
+import subprocess
+import sys
+
+import pytest
+
+from kublai.context_bundle import ContextBundleError, Source
+from kublai.llm import (
+ ContextOverflowError,
+ LlmTimeoutError,
+ MalformedError,
+ RateLimitError,
+ RefusedError,
+ ensure_external_safe,
+ llm_call_fresh_context,
+ llm_call_structured,
+ model_for,
+)
+from kublai.prompt_registry import PromptRegistry
+from kublai.sanitizer import PrivacyBoundaryError, Sanitizer, SanitizerContext
+
+
+def test_sanitizer_classifies_path_type_tags_and_public_marker():
+ sanitizer = Sanitizer()
+ assert sanitizer.classify("hard-private/finance/x.md", {}) == "hard-private"
+ assert sanitizer.classify("concepts/x.md", {"type": "human-contact"}) == "hard-private"
+ assert sanitizer.classify("concepts/x.md", {"tags": ["pii"]}) == "hard-private"
+ assert sanitizer.classify("concepts/x.md", {}) == "private"
+ assert sanitizer.classify("concepts/x.md", {"publish": True}) == "public"
+
+
+def test_sanitizer_scrubs_canary_and_hard_private_paths_for_llm():
+ scrubbed, findings = Sanitizer().scrub(
+ "see hard-private/finance/x.md KUBLAI_HARD_PRIVATE_CANARY_TEST",
+ target_class="public",
+ context=SanitizerContext.LLM_PROMPT,
+ )
+ assert "hard-private/finance/x.md" not in scrubbed
+ assert "KUBLAI_HARD_PRIVATE_CANARY_TEST" not in scrubbed
+ assert set(findings) == {"hard_private_path", "hard_private_canary"}
+
+
+def test_context_bundle_requires_resolvable_rel_path():
+ with pytest.raises(ContextBundleError):
+ Source(rel_path="../secret.md", privacy_class="public")
+
+
+def test_context_bundle_blocks_private_external_llm():
+ bundle = Sanitizer().build_bundle(
+ {"rel_path": "concepts/internal.md", "frontmatter": {}, "content": "private"}
+ )
+ with pytest.raises(PrivacyBoundaryError):
+ ensure_external_safe(bundle, destination="external")
+
+
+def test_llm_error_taxonomy_exposes_expected_failure_modes():
+ assert issubclass(RateLimitError, Exception)
+ assert issubclass(RefusedError, Exception)
+ assert issubclass(MalformedError, Exception)
+ assert issubclass(LlmTimeoutError, Exception)
+ assert issubclass(ContextOverflowError, Exception)
+
+
+def test_llm_structured_retries_with_pushback():
+ responses = iter([
+ json.dumps({"tags": ["a", "b", "c", "d", "e"]}),
+ json.dumps({"tags": ["a", "b", "c"]}),
+ ])
+ prompts = []
+
+ def transport(**request):
+ prompts.append(request["user"])
+ return next(responses)
+
+ def validator(payload):
+ if len(payload["tags"]) != 3:
+ raise ValueError("expected exactly 3 tags")
+
+ result = asyncio.run(
+ llm_call_structured(
+ command="ask",
+ user="return tags",
+ schema={"required": ["tags"]},
+ validator=validator,
+ transport=transport,
+ )
+ )
+
+ assert result == {"tags": ["a", "b", "c"]}
+ assert "Previous response was invalid" in prompts[-1]
+
+
+def test_llm_fresh_context_logs_single_user_message(tmp_path, monkeypatch):
+ log_path = tmp_path / "llm.jsonl"
+ monkeypatch.setenv("KUBLAI_LLM_REQUEST_LOG", str(log_path))
+
+ asyncio.run(
+ llm_call_fresh_context(
+ command="research_lens",
+ system="system prompt",
+ user="single fresh prompt",
+ transport=lambda **request: "ok",
+ )
+ )
+
+ request = json.loads(log_path.read_text().splitlines()[0])
+ assert "user" not in request
+ assert "system" not in request
+ assert request["user_chars"] == len("single fresh prompt")
+ assert request["system_chars"] == len("system prompt")
+ assert len(request["user_sha256"]) == 64
+ assert len(request["system_sha256"]) == 64
+ assert request["model"] == model_for("research_lens")
+
+
+def test_prompt_registry_hash_is_deterministic_and_content_sensitive(tmp_path):
+ prompt_root = tmp_path / ".prompts"
+ prompt_root.mkdir()
+ prompt = prompt_root / "ask.md"
+ prompt.write_text("first", encoding="utf-8")
+ registry = PromptRegistry(prompt_root)
+ first_hash = registry.hash("ask")
+ assert registry.hash("ask") == first_hash
+ prompt.write_text("second", encoding="utf-8")
+ assert registry.hash("ask") != first_hash
+
+
+def test_ast_lint_flags_ungrounded_brain_read(tmp_path):
+ fixture = tmp_path / "bad.py"
+ fixture.write_text(
+ "def bad(brain_service):\n"
+ " return brain_service.search('private')\n",
+ encoding="utf-8",
+ )
+
+ result = subprocess.run(
+ [sys.executable, "lints/no_ungrounded_brain_read.py", str(fixture)],
+ cwd=".",
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+
+ assert result.returncode == 1
+ assert "not sanitizer-grounded" in result.stdout
+
+
+def test_ast_lint_allows_nearby_sanitizer_call(tmp_path):
+ fixture = tmp_path / "good.py"
+ fixture.write_text(
+ "def good(brain_service, sanitizer):\n"
+ " rows = brain_service.search('public')\n"
+ " return sanitizer.scrub(str(rows), target_class='public', context=None)\n",
+ encoding="utf-8",
+ )
+
+ result = subprocess.run(
+ [sys.executable, "lints/no_ungrounded_brain_read.py", str(fixture)],
+ cwd=".",
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+
+ assert result.returncode == 0
diff --git a/tests/test_priority_commands.py b/tests/test_priority_commands.py
index d122793a..3e4b5456 100644
--- a/tests/test_priority_commands.py
+++ b/tests/test_priority_commands.py
@@ -10,6 +10,7 @@
Location: /Users/kurultai/molt/tests/test_priority_commands.py
"""
+from __future__ import annotations
import os
import sys
diff --git a/tests/test_reflection_memory.py b/tests/test_reflection_memory.py
index 6da20fdb..00cf9049 100644
--- a/tests/test_reflection_memory.py
+++ b/tests/test_reflection_memory.py
@@ -136,6 +136,35 @@ def test_record_mistake_success(self, mock_memory):
assert call_args[1]["lesson"] == "Always use hash_password()"
assert "embedding" in call_args[1]
+ def test_record_mistake_dual_writes_to_brain(self, mock_memory, tmp_path, monkeypatch):
+ """Test optional Phase 2 dual-write to markdown knowledge store."""
+ memory, mock_session = mock_memory
+ mock_result = Mock()
+ mock_result.single.return_value = {"reflection_id": "test-reflection-id"}
+ mock_session.run.return_value = mock_result
+ log_path = tmp_path / "dual-write.jsonl"
+
+ monkeypatch.setenv("KUBLAI_KNOWLEDGE_DUAL_WRITE", "1")
+ monkeypatch.setenv("BRAIN_WIKI_ROOT", str(tmp_path / "brain"))
+ monkeypatch.setenv("BRAIN_DUAL_WRITE_LOG", str(log_path))
+
+ rm = AgentReflectionMemory(memory)
+ reflection_id = rm.record_mistake(
+ agent="developer",
+ mistake_type="security",
+ context="Implementing authentication",
+ expected_behavior="Password should be hashed",
+ actual_behavior="Password stored in plaintext",
+ root_cause="Forgot to call hash function",
+ lesson="Always use hash_password()"
+ )
+
+ assert reflection_id == "test-reflection-id"
+ pages = list((tmp_path / "brain" / "operations/reflections").glob("*.md"))
+ assert len(pages) == 1
+ assert "reflection_id: test-reflection-id" in pages[0].read_text()
+ assert '"idempotency_key": "reflection:test-reflection-id"' in log_path.read_text()
+
def test_record_mistake_invalid_type(self, reflection_memory):
"""Test that invalid mistake_type raises ValueError."""
with pytest.raises(ValueError) as exc_info:
diff --git a/tests/v4/test_v4_roadmap.py b/tests/v4/test_v4_roadmap.py
new file mode 100644
index 00000000..d4a158b9
--- /dev/null
+++ b/tests/v4/test_v4_roadmap.py
@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import json
+import threading
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+from kublai.brain_service import BrainService
+from kublai.knowledge import KnowledgeStore
+from kublai.v4 import HARD_PRIVATE_CANARY
+from kublai.v4_gateway import is_loopback_host, serve_gateway, sign_request
+
+
+def make_service(tmp_path: Path, monkeypatch) -> BrainService:
+ monkeypatch.setenv("BRAIN_PRIVATE_INDEX_DB", str(tmp_path / "private-index.db"))
+ wiki = tmp_path / "brain"
+ wiki.mkdir()
+ KnowledgeStore(wiki).atomic_write(
+ wiki / "public.md",
+ KnowledgeStore.render(
+ {
+ "type": "concept",
+ "title": "Public Page",
+ "status": "active",
+ "created": "2026-05-03",
+ "updated": "2026-05-03",
+ "sources": 1,
+ "tags": ["public"],
+ "publish": True,
+ },
+ "# Public Page\n\nBrain-service public context.",
+ ),
+ )
+ (wiki / "hard-private").mkdir()
+ KnowledgeStore(wiki).atomic_write(
+ wiki / "hard-private" / "canary.md",
+ KnowledgeStore.render(
+ {
+ "type": "concept",
+ "title": "Canary",
+ "status": "active",
+ "created": "2026-05-03",
+ "updated": "2026-05-03",
+ "sources": 1,
+ "tags": ["pii"],
+ },
+ f"# Canary\n\n{HARD_PRIVATE_CANARY}",
+ ),
+ )
+ service = BrainService(wiki, tmp_path / "telemetry.db", tmp_path / "index.db")
+ service.reindex()
+ service.reindex_private()
+ return service
+
+
+def test_public_search_and_publish_never_expose_canary(tmp_path: Path, monkeypatch) -> None:
+ service = make_service(tmp_path, monkeypatch)
+ assert service.verify_index()["hard_private_rows"] == 0
+ assert service.v4.public_search(query=HARD_PRIVATE_CANARY) == []
+ dry = service.v4.publish_dry_run(output_root=tmp_path / "public-out")
+ assert dry["ok"] is True
+ assert all("hard-private" not in json.dumps(item) for item in dry["files"])
+ applied = service.v4.publish_apply(output_root=tmp_path / "public-out")
+ assert applied["written"] == 1
+ assert HARD_PRIVATE_CANARY not in (tmp_path / "public-out" / "public.md").read_text()
+
+
+def test_capture_defaults_ambiguous_content_to_hard_private(tmp_path: Path, monkeypatch) -> None:
+ service = make_service(tmp_path, monkeypatch)
+ plan = service.v4.capture_dry_run(content="Ambiguous MCP-derived material", title="Inbox Thing")
+ assert plan["privacy_class"] == "hard-private"
+ assert plan["rel_path"].startswith("hard-private/inbox/")
+
+
+def test_gateway_hmac_and_private_rejection(tmp_path: Path, monkeypatch) -> None:
+ service = make_service(tmp_path, monkeypatch)
+ server = serve_gateway(service, host="127.0.0.1", port=0, secret="test-secret")
+ port = server.server_address[1]
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ body = json.dumps({"query": "Brain-service", "privacy_scope": "public"}).encode()
+ ts = str(time.time())
+ sig = sign_request("test-secret", "POST", "/api/ask", body, ts)
+ req = urllib.request.Request(
+ f"http://127.0.0.1:{port}/api/ask",
+ data=body,
+ method="POST",
+ headers={"Content-Type": "application/json", "X-Kublai-Timestamp": ts, "X-Kublai-Signature": sig},
+ )
+ assert json.loads(urllib.request.urlopen(req, timeout=5).read())["ok"] is True
+
+ private_body = json.dumps({"query": "x", "privacy_scope": "hard-private"}).encode()
+ ts = str(time.time())
+ sig = sign_request("test-secret", "POST", "/api/ask", private_body, ts)
+ req = urllib.request.Request(
+ f"http://127.0.0.1:{port}/api/ask",
+ data=private_body,
+ method="POST",
+ headers={"Content-Type": "application/json", "X-Kublai-Timestamp": ts, "X-Kublai-Signature": sig},
+ )
+ try:
+ urllib.request.urlopen(req, timeout=5)
+ raise AssertionError("private request should fail")
+ except urllib.error.HTTPError as exc:
+ assert exc.code == 403
+ finally:
+ server.shutdown()
+ thread.join(timeout=5)
+
+
+def test_gateway_requires_secret_for_non_loopback_bind(tmp_path: Path, monkeypatch) -> None:
+ service = make_service(tmp_path, monkeypatch)
+ assert is_loopback_host("127.0.0.1")
+ try:
+ serve_gateway(service, host="0.0.0.0", port=0, secret="")
+ raise AssertionError("non-loopback gateway without HMAC secret should fail")
+ except ValueError as exc:
+ assert "requires HMAC secret" in str(exc)
+
+
+def test_llm_request_log_omits_raw_prompt(tmp_path: Path, monkeypatch) -> None:
+ from kublai import llm
+
+ log = tmp_path / "llm.ndjson"
+ monkeypatch.setenv("KUBLAI_LLM_REQUEST_LOG", str(log))
+ monkeypatch.setenv("KUBLAI_LLM_MOCK_RESPONSE", "ok")
+ import asyncio
+
+ assert asyncio.run(llm.llm_call(command="ask", user="raw prompt text")) == "ok"
+ record = json.loads(log.read_text())
+ assert "user" not in record
+ assert record["user_chars"] == len("raw prompt text")
diff --git a/tools/error_recovery.py b/tools/error_recovery.py
index 693800bf..e6cc81d0 100644
--- a/tools/error_recovery.py
+++ b/tools/error_recovery.py
@@ -13,6 +13,7 @@
- RTL-001: Rate Limit Exceeded
- MIG-001: Database Migration Failure
"""
+from __future__ import annotations
import asyncio
import json
diff --git a/tools/golden_query_benchmark.py b/tools/golden_query_benchmark.py
new file mode 100644
index 00000000..768bdadc
--- /dev/null
+++ b/tools/golden_query_benchmark.py
@@ -0,0 +1,165 @@
+"""Golden-query benchmark for brain-service index parity.
+
+The Phase 2 soak keeps a fixed query set and compares rank overlap against a
+checked-in or captured baseline. The baseline is JSON so it can be generated on
+the Mac Mini before cutover and replayed during the 7-day soak.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+from kublai.brain_service import BrainService
+
+
+DEFAULT_QUERIES = [
+ "agent reflection mistake lesson",
+ "capability acquisition pipeline",
+ "completed task deliverable",
+ "architecture decision rationale",
+ "recursive self improvement cycle",
+ "Kublai orchestrator delegation",
+ "Mongke research synthesis",
+ "Temujin implementation testing",
+ "Jochi analysis security audit",
+ "Ogedei operations health",
+ "Signal integration failure",
+ "Neo4j retry overshoot",
+ "provider health patrol",
+ "gateway stderr warning",
+ "proposal decree backlog",
+ "Tailscale Mac mini",
+ "Hermes gateway restore",
+ "brain wiki schema",
+ "knowledge migration",
+ "telemetry claim lock",
+ "rate limit counter",
+ "notification unread",
+ "heartbeat sidecar",
+ "online backup snapshot",
+ "vector orphan check",
+ "Obsidian wikilink",
+ "human curation branch",
+ "Railway out of scope",
+ "AuraDB out of scope",
+ "Homebrew Neo4j",
+ "agent profile capabilities",
+ "learned capability mastery",
+ "decision supersedes",
+ "historical completed task",
+ "dual write reconciliation",
+ "operation id idempotency",
+ "completion body hash",
+ "file watcher reindex",
+ "WAL checkpoint monitor",
+ "brain service healthcheck",
+ "reflection consolidation metarule",
+ "security mistake root cause",
+ "architecture introspection",
+ "proactive reflection opportunity",
+ "delegation protocol",
+ "task dependency engine",
+ "capability based access control",
+ "knowledge source of truth",
+ "Mac mini canonical brain",
+ "this Mac curation replica",
+]
+
+
+def main(argv: list[str] | None = None) -> int:
+ args = build_parser().parse_args(argv)
+ service = BrainService(args.wiki_root, args.telemetry_db, args.index_db)
+ if args.reindex:
+ service.reindex()
+ report = run_benchmark(
+ service=service,
+ queries=load_queries(args.queries),
+ baseline_path=args.baseline,
+ top_k=args.top_k,
+ write_baseline=args.write_baseline,
+ )
+ print(json.dumps(report, indent=2, sort_keys=True))
+ threshold = args.min_overlap
+ return 0 if report["average_rank_overlap"] >= threshold else 1
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(prog="golden-query-benchmark")
+ parser.add_argument("--wiki-root", default=str(Path.home() / "brain"))
+ parser.add_argument("--telemetry-db", default=str(Path.home() / ".kublai/telemetry.db"))
+ parser.add_argument("--index-db", default=str(Path.home() / ".brain-index/brain.db"))
+ parser.add_argument("--queries", type=Path)
+ parser.add_argument("--baseline", type=Path, required=True)
+ parser.add_argument("--top-k", type=int, default=10)
+ parser.add_argument("--min-overlap", type=float, default=0.8)
+ parser.add_argument("--write-baseline", action="store_true")
+ parser.add_argument("--reindex", action="store_true")
+ return parser
+
+
+def run_benchmark(
+ *,
+ service: BrainService,
+ queries: list[str],
+ baseline_path: Path,
+ top_k: int,
+ write_baseline: bool,
+) -> dict[str, Any]:
+ current = {
+ query: [row.get("typed_id") or row["id"] for row in service.search(query=query, limit=top_k)]
+ for query in queries
+ }
+ metadata = {
+ "embedding_model": "all-MiniLM-L6-v2",
+ "embedding_dim": 384,
+ "preprocessing_hash": hashlib.sha256("\n".join(queries).encode("utf-8")).hexdigest(),
+ "top_k": top_k,
+ }
+ if write_baseline or not baseline_path.exists():
+ baseline_path.parent.mkdir(parents=True, exist_ok=True)
+ baseline_path.write_text(json.dumps({"metadata": metadata, "results": current}, indent=2, sort_keys=True))
+ return {
+ "ok": True,
+ "mode": "baseline-written",
+ "queries": len(queries),
+ "average_rank_overlap": 1.0,
+ "metadata": metadata,
+ }
+
+ baseline = json.loads(baseline_path.read_text())
+ expected = baseline["results"]
+ overlaps = []
+ details = []
+ for query in queries:
+ old = expected.get(query, [])
+ new = current.get(query, [])
+ if not old and not new:
+ overlap = 1.0
+ else:
+ denom = max(1, min(len(old), top_k))
+ overlap = len(set(old[:top_k]) & set(new[:top_k])) / denom
+ overlaps.append(overlap)
+ details.append({"query": query, "rank_overlap": overlap, "expected": old[:top_k], "actual": new[:top_k]})
+ average = sum(overlaps) / max(1, len(overlaps))
+ return {
+ "ok": average >= 0.8,
+ "mode": "compare",
+ "queries": len(queries),
+ "average_rank_overlap": average,
+ "metadata": metadata,
+ "details": details,
+ }
+
+
+def load_queries(path: Path | None) -> list[str]:
+ if path is None:
+ return DEFAULT_QUERIES
+ return [line.strip() for line in path.read_text().splitlines() if line.strip() and not line.startswith("#")]
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/kurultai/agent_gemini.py b/tools/kurultai/agent_gemini.py
index d7707bdd..f5026a75 100644
--- a/tools/kurultai/agent_gemini.py
+++ b/tools/kurultai/agent_gemini.py
@@ -5,6 +5,7 @@
Provides each agent with their own Gemini CLI context.
All agents use Gemini 3.1 Pro Preview through separate configurations.
"""
+from __future__ import annotations
import os
import json
diff --git a/tools/kurultai/agent_tasks.py b/tools/kurultai/agent_tasks.py
index c196de42..35f551e0 100644
--- a/tools/kurultai/agent_tasks.py
+++ b/tools/kurultai/agent_tasks.py
@@ -17,6 +17,7 @@
hb = get_heartbeat(driver)
await register_all_tasks(hb)
"""
+from __future__ import annotations
import asyncio
import json
diff --git a/tools/kurultai/api/main.py b/tools/kurultai/api/main.py
index 81f0f4c8..8bdbfbbb 100644
--- a/tools/kurultai/api/main.py
+++ b/tools/kurultai/api/main.py
@@ -4,6 +4,8 @@
Replaces Express.js server with FastAPI
"""
+from __future__ import annotations
+
import os
import sys
from pathlib import Path
diff --git a/tools/kurultai/buildroom/README.md b/tools/kurultai/buildroom/README.md
new file mode 100644
index 00000000..5461ccf9
--- /dev/null
+++ b/tools/kurultai/buildroom/README.md
@@ -0,0 +1,30 @@
+# Kurultai Buildroom
+
+Buildroom is the public-safe contract surface for the Auto-think -> Auto-build chain. It stores typed JSON artifacts that show why an idea exists, who approved it, what Coder was allowed to build, what changed, what QA verified, and what the operator should know.
+
+Canonical repo mirror: `tools/kurultai/buildroom/`. The long-term brain location can mirror these same docs/schemas/fixtures when the brain-side task runs.
+
+## Quick start
+
+```bash
+cd tools/kurultai/buildroom
+python scripts/validate_room.py rooms/demo-room
+python scripts/build_operator_summary.py rooms/demo-room
+python scripts/kanban_adapter.py task-packet rooms/demo-room /tmp/buildroom-task-packet.json
+python scripts/kanban_adapter.py receipt rooms/demo-room /tmp/kanban-completion.json
+python scripts/qa_trust.py qa-packet rooms/demo-room /tmp/buildroom-qa-task-packet.json
+python scripts/qa_trust.py delta rooms/demo-room
+python scripts/qa_trust.py trust rooms/demo-room
+python scripts/export_sanitized_bundle.py rooms/demo-room /tmp/kurultai-buildroom-export
+```
+
+## Directory map
+
+- `docs/` — lifecycle, architecture, operator model, safety, retention notes.
+- `schemas/` — JSON Schemas for every lifecycle artifact.
+- `rooms/demo-room/` — complete demo fixture covering every artifact stage.
+- `scripts/` — stdlib-friendly validation, room creation, operator summary, Kanban adapter, QA/trust, sanitized export.
+
+## Safety boundary
+
+Buildroom bundles are designed to be public-safe after sanitization, but source rooms can contain internal references. Do not place secrets, tokens, raw private logs, or private absolute paths in artifacts unless the sanitizer has a specific redaction rule and the source is never exported directly.
diff --git a/tools/kurultai/buildroom/control-room.md b/tools/kurultai/buildroom/control-room.md
new file mode 100644
index 00000000..11df716d
--- /dev/null
+++ b/tools/kurultai/buildroom/control-room.md
@@ -0,0 +1,37 @@
+# Kurultai Buildroom Control Room
+
+Generated: 2026-05-11T01:37:39Z
+
+## Summary
+
+- Rooms: 2
+- Trust clean: 1
+- Trust watch: 1
+- Trust investigate: 0
+- Rooms with missing evidence: 0
+
+## Rooms
+
+### 2026-05-10-gkisokay-auto-think-auto-build
+
+- Headline: gkisokay Auto Think Auto Build
+- Phase: complete
+- Trust: watch (risk_score=2)
+- Verification pass: True
+- Verification delta: confirmed
+- Retention: keep
+- Missing evidence: none
+- Operator decisions needed: Run validate_room, Review operator summary
+- Next action: Review room state and refresh operator summary.
+
+### demo-room
+
+- Headline: Install a typed buildroom foundation for autonomous improvement work
+- Phase: complete
+- Trust: clean (risk_score=0.1)
+- Verification pass: True
+- Verification delta: confirmed
+- Retention: keep
+- Missing evidence: none
+- Operator decisions needed: none
+- Next action: Ready for PR/merge monitoring or retention follow-up.
diff --git a/tools/kurultai/buildroom/docs/architecture.md b/tools/kurultai/buildroom/docs/architecture.md
new file mode 100644
index 00000000..308da3df
--- /dev/null
+++ b/tools/kurultai/buildroom/docs/architecture.md
@@ -0,0 +1,7 @@
+# Buildroom Architecture
+
+Buildroom is a filesystem-backed contract chain, not a scheduler. Native Hermes Kanban, profiles, cron, receipts, and brain-service remain the runtime organs.
+
+Flow: research input -> idea contract -> intent review -> main review -> product/build plans -> implementation receipt -> verification report/delta -> trust report -> retention review -> operator summary.
+
+Public-safe contract files reference private runtime data by IDs, hashes, or sanitized paths. Private adapters may read live Kanban/receipt/brain state, but they should not copy private logs into export bundles.
diff --git a/tools/kurultai/buildroom/docs/lifecycle.md b/tools/kurultai/buildroom/docs/lifecycle.md
new file mode 100644
index 00000000..fdb7dbe9
--- /dev/null
+++ b/tools/kurultai/buildroom/docs/lifecycle.md
@@ -0,0 +1,5 @@
+# Buildroom Lifecycle
+
+States are artifact-derived: `candidate`, `planned`, `building`, `verifying`, `clean`, `watch`, `investigate`, or `parked`.
+
+Every transition should be recoverable from JSON artifacts alone. Humans and future agents should not need chat history to understand why the room is in its current state.
diff --git a/tools/kurultai/buildroom/docs/operator-model.md b/tools/kurultai/buildroom/docs/operator-model.md
new file mode 100644
index 00000000..04eee14e
--- /dev/null
+++ b/tools/kurultai/buildroom/docs/operator-model.md
@@ -0,0 +1,5 @@
+# Operator Model
+
+The operator sees concise room cards: headline, owner, lifecycle status, trust state, latest verification, retention recommendation, and decisions needed.
+
+Operator decisions are required for approval gates, high-risk blast-radius changes, destructive retention actions, live-capital or external-public actions, and secrets/security surfaces.
diff --git a/tools/kurultai/buildroom/docs/retention.md b/tools/kurultai/buildroom/docs/retention.md
new file mode 100644
index 00000000..9319ff10
--- /dev/null
+++ b/tools/kurultai/buildroom/docs/retention.md
@@ -0,0 +1,5 @@
+# Retention
+
+Retention review recommends `keep`, `improve`, `park`, or `prune`. It does not perform destructive actions.
+
+A prune recommendation must set `requires_human_before_destructive_action: true` and list concrete artifacts to prune so a human can review the blast radius.
diff --git a/tools/kurultai/buildroom/docs/safety.md b/tools/kurultai/buildroom/docs/safety.md
new file mode 100644
index 00000000..1a1b1e9d
--- /dev/null
+++ b/tools/kurultai/buildroom/docs/safety.md
@@ -0,0 +1,7 @@
+# Safety Rules
+
+- Dreamer proposes; it never approves its own work.
+- Coder only works from an approved product/build plan.
+- Retention only recommends; it never deletes or rewrites live state.
+- Sanitized export redacts private paths and secrets-like strings and excludes artifacts marked `sensitivity: private`.
+- Protected paths and stop conditions in build plans are constraints, not suggestions.
diff --git a/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/ideas/idea-contract.json b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/ideas/idea-contract.json
new file mode 100644
index 00000000..1364b7d9
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/ideas/idea-contract.json
@@ -0,0 +1,42 @@
+{
+ "beneficiaries": [
+ "operator",
+ "Kurultai agents"
+ ],
+ "created_at": "2026-05-11T01:17:17Z",
+ "evidence_refs": [
+ "https://x.com/gkisokay/status/2053449921554960545",
+ "https://x.com/gkisokay/status/2053467830155427942",
+ "https://x.com/gkisokay/status/2053613051182772461",
+ "brain:queue/2026-05-10-gkisokay-hermes-auto-think-auto-build.md",
+ "brain:generated/2026-05-10-gkisokay-auto-think-auto-build-synthesis.md",
+ "brain:generated/reviews/2026-05-10-gkisokay-auto-think-auto-build-review.md"
+ ],
+ "expected_artifacts": [
+ "research-input",
+ "idea-contract",
+ "intent-review",
+ "operator-summary"
+ ],
+ "idea_id": "idea-2026-05-10-gkisokay-auto-think-auto-build",
+ "non_goals": [
+ "production deployment",
+ "runtime mutation"
+ ],
+ "problem": "Useful research should move through a structured path from insight to bounded work.",
+ "proposed_by": "research-to-buildroom",
+ "proposed_owner": "kublai",
+ "risk_notes": "Review before promotion.",
+ "scope_boundaries": [
+ "curated artifacts only",
+ "separate review before implementation"
+ ],
+ "status": "candidate",
+ "suggested_workspace": "tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build",
+ "title": "gkisokay Auto Think Auto Build",
+ "verification_hint": [
+ "Validate room",
+ "Review source refs"
+ ],
+ "why_now": "The buildroom contract exists and can now accept real research inputs."
+}
diff --git a/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/jobs/implementation-receipt.json b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/jobs/implementation-receipt.json
new file mode 100644
index 00000000..5216e1e9
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/jobs/implementation-receipt.json
@@ -0,0 +1,28 @@
+{
+ "assignee": "kublai",
+ "blocked_items": [],
+ "build_id": "build-2026-05-10-gkisokay-auto-think-auto-build",
+ "commands_run": [
+ "research_to_buildroom.py"
+ ],
+ "commit_sha": "pending",
+ "completed_at": "2026-05-11T01:17:17Z",
+ "deviations_from_plan": [],
+ "evidence_refs": [
+ "https://x.com/gkisokay/status/2053449921554960545",
+ "https://x.com/gkisokay/status/2053467830155427942",
+ "https://x.com/gkisokay/status/2053613051182772461",
+ "brain:queue/2026-05-10-gkisokay-hermes-auto-think-auto-build.md",
+ "brain:generated/2026-05-10-gkisokay-auto-think-auto-build-synthesis.md",
+ "brain:generated/reviews/2026-05-10-gkisokay-auto-think-auto-build-review.md"
+ ],
+ "files_changed": [
+ "tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build"
+ ],
+ "kanban_status": "not-applicable",
+ "kanban_task_id": "not-created",
+ "open_diffs_summary": "Generated buildroom room from research artifacts.",
+ "receipt_id": "receipt-2026-05-10-gkisokay-auto-think-auto-build",
+ "started_at": "2026-05-11T01:17:17Z",
+ "tests_run": []
+}
diff --git a/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/operator/operator-summary.json b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/operator/operator-summary.json
new file mode 100644
index 00000000..614025b4
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/operator/operator-summary.json
@@ -0,0 +1,45 @@
+{
+ "current_owner": "kublai",
+ "generated_at": "2026-05-11T01:17:25Z",
+ "headline": "gkisokay Auto Think Auto Build",
+ "latest_artifacts": [
+ "research/research-input.json",
+ "ideas/idea-contract.json",
+ "reviews/intent-review.json",
+ "reviews/main-review.json",
+ "plans/product-plan.json",
+ "plans/build-plan.json",
+ "jobs/implementation-receipt.json",
+ "verification/verification-report.json",
+ "verification/verification-delta.json",
+ "trust/trust-report.json",
+ "retention/retention-review.json",
+ "operator/operator-summary.json"
+ ],
+ "links": [
+ "buildroom://2026-05-10-gkisokay-auto-think-auto-build/research/research-input.json",
+ "buildroom://2026-05-10-gkisokay-auto-think-auto-build/ideas/idea-contract.json",
+ "buildroom://2026-05-10-gkisokay-auto-think-auto-build/reviews/intent-review.json",
+ "buildroom://2026-05-10-gkisokay-auto-think-auto-build/reviews/main-review.json",
+ "buildroom://2026-05-10-gkisokay-auto-think-auto-build/plans/product-plan.json",
+ "buildroom://2026-05-10-gkisokay-auto-think-auto-build/plans/build-plan.json",
+ "buildroom://2026-05-10-gkisokay-auto-think-auto-build/jobs/implementation-receipt.json",
+ "buildroom://2026-05-10-gkisokay-auto-think-auto-build/verification/verification-report.json",
+ "buildroom://2026-05-10-gkisokay-auto-think-auto-build/verification/verification-delta.json",
+ "buildroom://2026-05-10-gkisokay-auto-think-auto-build/trust/trust-report.json",
+ "buildroom://2026-05-10-gkisokay-auto-think-auto-build/retention/retention-review.json",
+ "buildroom://2026-05-10-gkisokay-auto-think-auto-build/operator/operator-summary.json"
+ ],
+ "operator_decisions_needed": [
+ "Run validate_room",
+ "Review operator summary"
+ ],
+ "operator_needs_to_know": [
+ "Trust state: watch (risk_score=2).",
+ "Verification pass: True via static_analysis.",
+ "Retention recommendation: keep."
+ ],
+ "room_id": "2026-05-10-gkisokay-auto-think-auto-build",
+ "status": "watch",
+ "summary_id": "summary-2026-05-10-gkisokay-auto-think-auto-build"
+}
diff --git a/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/plans/build-plan.json b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/plans/build-plan.json
new file mode 100644
index 00000000..c3a62313
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/plans/build-plan.json
@@ -0,0 +1,39 @@
+{
+ "assignee": "kublai",
+ "build_id": "build-2026-05-10-gkisokay-auto-think-auto-build",
+ "commands_allowed": [
+ "python3 tools/kurultai/buildroom/scripts/validate_room.py"
+ ],
+ "created_at": "2026-05-11T01:17:17Z",
+ "files_expected": [
+ "tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build"
+ ],
+ "out_of_scope": [
+ "production deploy"
+ ],
+ "plan_id": "plan-2026-05-10-gkisokay-auto-think-auto-build",
+ "steps": [
+ {
+ "action": "Generate artifacts",
+ "id": "generate"
+ },
+ {
+ "action": "Validate schema chain",
+ "id": "validate"
+ },
+ {
+ "action": "Record receipt",
+ "id": "receipt"
+ }
+ ],
+ "stop_conditions": [
+ "validation failure",
+ "scope expansion"
+ ],
+ "task_refs": [
+ "buildroom://2026-05-10-gkisokay-auto-think-auto-build"
+ ],
+ "verification_commands": [
+ "python3 tools/kurultai/buildroom/scripts/validate_room.py tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build"
+ ]
+}
diff --git a/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/plans/product-plan.json b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/plans/product-plan.json
new file mode 100644
index 00000000..0b02abeb
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/plans/product-plan.json
@@ -0,0 +1,25 @@
+{
+ "acceptance_criteria": [
+ "room validates",
+ "operator summary exists",
+ "next action is explicit"
+ ],
+ "allowed_paths": [
+ "tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build"
+ ],
+ "created_at": "2026-05-11T01:17:17Z",
+ "idea_id": "idea-2026-05-10-gkisokay-auto-think-auto-build",
+ "non_goals": [
+ "unrelated implementation"
+ ],
+ "owner": "kublai",
+ "plan_id": "plan-2026-05-10-gkisokay-auto-think-auto-build",
+ "protected_paths": [
+ "runtime state",
+ "raw sessions"
+ ],
+ "rollback_plan": "Remove the generated room or revert its commit.",
+ "rollout_plan": "Generate room, validate it, then review the summary.",
+ "user_value": "Research can become bounded work without being buried in chat history.",
+ "ux_or_operator_surface": "operator/operator-summary.json"
+}
diff --git a/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/research/research-input.json b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/research/research-input.json
new file mode 100644
index 00000000..9fcbd2f3
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/research/research-input.json
@@ -0,0 +1,31 @@
+{
+ "created_at": "2026-05-11T01:17:17Z",
+ "packet_id": "research-2026-05-10-gkisokay-auto-think-auto-build",
+ "sanitization_notes": "Generated from curated references only.",
+ "schema_version": "1.0",
+ "sensitivity": "internal",
+ "signals": [
+ {
+ "kind": "research",
+ "summary": "Auto Think should generate idea contracts, not self-approve builds."
+ },
+ {
+ "kind": "research",
+ "summary": "Auto Build should move through Main, Coder, QA, trust, retention, and operator summary."
+ },
+ {
+ "kind": "research",
+ "summary": "Public buildroom should separate reusable workflow contracts from private runtime state."
+ }
+ ],
+ "source_refs": [
+ "https://x.com/gkisokay/status/2053449921554960545",
+ "https://x.com/gkisokay/status/2053467830155427942",
+ "https://x.com/gkisokay/status/2053613051182772461",
+ "brain:queue/2026-05-10-gkisokay-hermes-auto-think-auto-build.md",
+ "brain:generated/2026-05-10-gkisokay-auto-think-auto-build-synthesis.md",
+ "brain:generated/reviews/2026-05-10-gkisokay-auto-think-auto-build-review.md"
+ ],
+ "source_type": "brain",
+ "summary": "Graeme gkisokay Hermes Auto Think Auto Build Research Packet Danny asked Kublai to research three public X posts by Graeme / `@gkisokay`: Visible engagement when researched: about 41.6K views, 279 likes, 24 reposts, 11 replies Visible engagement when researched: about 2,210 views and 17 likes"
+}
diff --git a/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/retention/retention-review.json b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/retention/retention-review.json
new file mode 100644
index 00000000..9b735118
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/retention/retention-review.json
@@ -0,0 +1,15 @@
+{
+ "artifacts_to_improve": [
+ "operator summary after follow-on work"
+ ],
+ "artifacts_to_keep": [
+ "all generated room artifacts"
+ ],
+ "artifacts_to_prune": [],
+ "rationale": "Keep as a real research-to-buildroom conversion.",
+ "recommendation": "keep",
+ "requires_human_before_destructive_action": true,
+ "retention_id": "retention-2026-05-10-gkisokay-auto-think-auto-build",
+ "reviewed_at": "2026-05-11T01:17:17Z",
+ "room_id": "2026-05-10-gkisokay-auto-think-auto-build"
+}
diff --git a/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/reviews/intent-review.json b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/reviews/intent-review.json
new file mode 100644
index 00000000..8b1e0ac0
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/reviews/intent-review.json
@@ -0,0 +1,11 @@
+{
+ "decision": "needs_main_review",
+ "idea_id": "idea-2026-05-10-gkisokay-auto-think-auto-build",
+ "reason": "System-relevant research maps to the buildroom contract.",
+ "related_existing_work": [
+ "tools/kurultai/buildroom"
+ ],
+ "review_id": "intent-2026-05-10-gkisokay-auto-think-auto-build",
+ "reviewed_at": "2026-05-11T01:17:17Z",
+ "reviewer": "kublai"
+}
diff --git a/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/reviews/main-review.json b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/reviews/main-review.json
new file mode 100644
index 00000000..f4d09e59
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/reviews/main-review.json
@@ -0,0 +1,15 @@
+{
+ "allowed_workspaces": [
+ "tools/kurultai/buildroom"
+ ],
+ "approval_notes": "Approved for autonomous continuation.",
+ "blast_radius": "local",
+ "blocked_reasons": [],
+ "decision": "approved_for_planning",
+ "human_approval_required": false,
+ "idea_id": "idea-2026-05-10-gkisokay-auto-think-auto-build",
+ "review_id": "main-2026-05-10-gkisokay-auto-think-auto-build",
+ "reviewed_at": "2026-05-11T01:17:17Z",
+ "reviewer": "kublai",
+ "risk_band": "low"
+}
diff --git a/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/trust/trust-report.json b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/trust/trust-report.json
new file mode 100644
index 00000000..a582b48a
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/trust/trust-report.json
@@ -0,0 +1,18 @@
+{
+ "generated_at": "2026-05-11T01:17:17Z",
+ "open_questions": [
+ "Which follow-on work should be promoted?"
+ ],
+ "reasons": [
+ "Generated room is safe to inspect."
+ ],
+ "required_followups": [
+ "Run validate_room",
+ "Review operator summary"
+ ],
+ "risk_score": 2,
+ "room_id": "2026-05-10-gkisokay-auto-think-auto-build",
+ "safe_to_archive": false,
+ "state": "watch",
+ "trust_id": "trust-2026-05-10-gkisokay-auto-think-auto-build"
+}
diff --git a/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/verification/verification-delta.json b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/verification/verification-delta.json
new file mode 100644
index 00000000..4c977072
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/verification/verification-delta.json
@@ -0,0 +1,13 @@
+{
+ "build_id": "build-2026-05-10-gkisokay-auto-think-auto-build",
+ "confirmed_claims": [
+ "Generated artifacts are present."
+ ],
+ "delta_id": "delta-2026-05-10-gkisokay-auto-think-auto-build",
+ "implementation_receipt_ref": "jobs/implementation-receipt.json",
+ "next_action": "validate room and select follow-on work",
+ "regressions": [],
+ "state": "confirmed",
+ "unverified_claims": [],
+ "verification_report_ref": "verification/verification-report.json"
+}
diff --git a/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/verification/verification-report.json b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/verification/verification-report.json
new file mode 100644
index 00000000..71f85840
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/2026-05-10-gkisokay-auto-think-auto-build/verification/verification-report.json
@@ -0,0 +1,16 @@
+{
+ "build_id": "build-2026-05-10-gkisokay-auto-think-auto-build",
+ "commands_run": [],
+ "evidence_refs": [
+ "buildroom://2026-05-10-gkisokay-auto-think-auto-build"
+ ],
+ "failures": [],
+ "method": "static_analysis",
+ "observations": [
+ "Room generated."
+ ],
+ "pass": true,
+ "verification_id": "verify-2026-05-10-gkisokay-auto-think-auto-build",
+ "verified_at": "2026-05-11T01:17:17Z",
+ "verified_by": "research-to-buildroom"
+}
diff --git a/tools/kurultai/buildroom/rooms/demo-room/ideas/idea-contract.json b/tools/kurultai/buildroom/rooms/demo-room/ideas/idea-contract.json
new file mode 100644
index 00000000..33cfd71f
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/demo-room/ideas/idea-contract.json
@@ -0,0 +1,44 @@
+{
+ "beneficiaries": [
+ "operator",
+ "dreamer",
+ "coder",
+ "qa",
+ "trust",
+ "retention"
+ ],
+ "created_at": "2026-05-10T17:01:00Z",
+ "evidence_refs": [
+ "research/research-input.json"
+ ],
+ "expected_artifacts": [
+ "schemas",
+ "demo-room",
+ "validation scripts",
+ "operator summary"
+ ],
+ "idea_id": "idea-buildroom-foundation-demo",
+ "non_goals": [
+ "Dashboard integration",
+ "Kanban adapter implementation",
+ "Automated deletion"
+ ],
+ "problem": "Autonomous work loses auditability when evidence, approval, implementation, and QA live only in chat.",
+ "proposed_by": "dreamer",
+ "proposed_owner": "chagatai",
+ "risk_notes": "Low-risk additive repo mirror; privacy risk is export leakage, controlled by sanitizer.",
+ "scope_boundaries": [
+ "Additive docs/schemas/scripts/fixture only",
+ "No dispatcher replacement",
+ "No private runtime export"
+ ],
+ "status": "candidate",
+ "suggested_workspace": "tools/kurultai/buildroom",
+ "title": "Install a typed buildroom foundation for autonomous improvement work",
+ "verification_hint": [
+ "Validate demo room",
+ "Corrupt a fixture and confirm validation fails",
+ "Export sanitized bundle and scan for private paths"
+ ],
+ "why_now": "The Auto-think/Auto-build contract chain has operator approval and a bounded foundation scope."
+}
diff --git a/tools/kurultai/buildroom/rooms/demo-room/jobs/implementation-receipt.json b/tools/kurultai/buildroom/rooms/demo-room/jobs/implementation-receipt.json
new file mode 100644
index 00000000..54bdabd9
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/demo-room/jobs/implementation-receipt.json
@@ -0,0 +1,26 @@
+{
+ "assignee": "temujin",
+ "blocked_items": [],
+ "build_id": "build-buildroom-foundation-demo",
+ "commands_run": [
+ "python3 tools/kurultai/buildroom/scripts/validate_room.py tools/kurultai/buildroom/rooms/demo-room"
+ ],
+ "commit_sha": "",
+ "completed_at": "2026-05-10T20:10:00Z",
+ "deviations_from_plan": [],
+ "evidence_refs": [
+ "kanban:t_demo_buildroom",
+ "brain:receipt/demo"
+ ],
+ "files_changed": [
+ "tools/kurultai/buildroom/README.md"
+ ],
+ "kanban_status": "done",
+ "kanban_task_id": "t_demo_buildroom",
+ "open_diffs_summary": "Demo adapter validation only.",
+ "receipt_id": "receipt-t_demo_buildroom",
+ "started_at": "2026-05-10T20:00:00Z",
+ "tests_run": [
+ "buildroom validation passed"
+ ]
+}
diff --git a/tools/kurultai/buildroom/rooms/demo-room/operator/operator-summary.json b/tools/kurultai/buildroom/rooms/demo-room/operator/operator-summary.json
new file mode 100644
index 00000000..511e3e45
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/demo-room/operator/operator-summary.json
@@ -0,0 +1,42 @@
+{
+ "current_owner": "chagatai",
+ "generated_at": "2026-05-11T00:52:19Z",
+ "headline": "Install a typed buildroom foundation for autonomous improvement work",
+ "latest_artifacts": [
+ "research/research-input.json",
+ "ideas/idea-contract.json",
+ "reviews/intent-review.json",
+ "reviews/main-review.json",
+ "plans/product-plan.json",
+ "plans/build-plan.json",
+ "jobs/implementation-receipt.json",
+ "verification/verification-report.json",
+ "verification/verification-delta.json",
+ "trust/trust-report.json",
+ "retention/retention-review.json",
+ "operator/operator-summary.json"
+ ],
+ "links": [
+ "buildroom://demo-room/research/research-input.json",
+ "buildroom://demo-room/ideas/idea-contract.json",
+ "buildroom://demo-room/reviews/intent-review.json",
+ "buildroom://demo-room/reviews/main-review.json",
+ "buildroom://demo-room/plans/product-plan.json",
+ "buildroom://demo-room/plans/build-plan.json",
+ "buildroom://demo-room/jobs/implementation-receipt.json",
+ "buildroom://demo-room/verification/verification-report.json",
+ "buildroom://demo-room/verification/verification-delta.json",
+ "buildroom://demo-room/trust/trust-report.json",
+ "buildroom://demo-room/retention/retention-review.json",
+ "buildroom://demo-room/operator/operator-summary.json"
+ ],
+ "operator_decisions_needed": [],
+ "operator_needs_to_know": [
+ "Trust state: clean (risk_score=0.1).",
+ "Verification pass: True via mixed.",
+ "Retention recommendation: keep."
+ ],
+ "room_id": "demo-room",
+ "status": "clean",
+ "summary_id": "summary-demo-room"
+}
diff --git a/tools/kurultai/buildroom/rooms/demo-room/plans/build-plan.json b/tools/kurultai/buildroom/rooms/demo-room/plans/build-plan.json
new file mode 100644
index 00000000..878b7284
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/demo-room/plans/build-plan.json
@@ -0,0 +1,37 @@
+{
+ "assignee": "chagatai",
+ "build_id": "build-buildroom-foundation-demo",
+ "commands_allowed": [
+ "python scripts/validate_room.py rooms/demo-room"
+ ],
+ "created_at": "2026-05-10T17:05:00Z",
+ "files_expected": [
+ "tools/kurultai/buildroom/README.md",
+ "tools/kurultai/buildroom/schemas/*.schema.json",
+ "tools/kurultai/buildroom/scripts/*.py"
+ ],
+ "out_of_scope": [
+ "Editing dispatcher",
+ "Posting externally",
+ "Deleting runtime state"
+ ],
+ "plan_id": "product-buildroom-foundation-demo",
+ "steps": [
+ {
+ "description": "Create README and lifecycle/operator/safety docs.",
+ "expected_output": "Docs explain public-safe contract chain.",
+ "step_id": "docs"
+ }
+ ],
+ "stop_conditions": [
+ "Need secrets",
+ "Protected path edit required",
+ "Validation cannot be made deterministic"
+ ],
+ "task_refs": [
+ "kanban:t_faf9a811"
+ ],
+ "verification_commands": [
+ "pytest tests/kurultai/test_buildroom_foundation.py"
+ ]
+}
diff --git a/tools/kurultai/buildroom/rooms/demo-room/plans/product-plan.json b/tools/kurultai/buildroom/rooms/demo-room/plans/product-plan.json
new file mode 100644
index 00000000..1d4428be
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/demo-room/plans/product-plan.json
@@ -0,0 +1,30 @@
+{
+ "acceptance_criteria": [
+ "Every lifecycle artifact has a schema",
+ "Demo room contains one complete artifact chain",
+ "Validation catches missing or invalid fields",
+ "Sanitized export redacts local absolute paths"
+ ],
+ "allowed_paths": [
+ "tools/kurultai/buildroom/**",
+ "tests/kurultai/test_buildroom_foundation.py"
+ ],
+ "created_at": "2026-05-10T17:04:00Z",
+ "idea_id": "idea-buildroom-foundation-demo",
+ "non_goals": [
+ "Live Kanban adapter",
+ "Dashboard UI",
+ "Retention deletion"
+ ],
+ "owner": "chagatai",
+ "plan_id": "product-buildroom-foundation-demo",
+ "protected_paths": [
+ ".env",
+ "**/*secret*",
+ "Hermes dispatcher internals"
+ ],
+ "rollback_plan": "Remove buildroom directory and test file if foundation is rejected.",
+ "rollout_plan": "Land as additive foundation, then downstream tasks consume schemas/scripts.",
+ "user_value": "Operators can audit autonomous improvement lifecycle state from typed files.",
+ "ux_or_operator_surface": "JSON operator summary and future dashboard cards."
+}
diff --git a/tools/kurultai/buildroom/rooms/demo-room/research/research-input.json b/tools/kurultai/buildroom/rooms/demo-room/research/research-input.json
new file mode 100644
index 00000000..24fe7fde
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/demo-room/research/research-input.json
@@ -0,0 +1,19 @@
+{
+ "created_at": "2026-05-10T17:00:00Z",
+ "packet_id": "research-demo-auto-think-001",
+ "sanitization_notes": "Source refs may contain local absolute paths and must be redacted for export.",
+ "schema_version": "1.0",
+ "sensitivity": "internal",
+ "signals": [
+ {
+ "description": "Danny approved the buildroom foundation task.",
+ "signal_id": "sig-approved-plan",
+ "weight": "high"
+ }
+ ],
+ "source_refs": [
+ "plan:${KURULTAI_HOME}/plans/2026-05-10-auto-think-auto-build-contract-chain.md"
+ ],
+ "source_type": "operator",
+ "summary": "Operator-approved plan calls for an auditable Auto-think to Auto-build contract chain."
+}
diff --git a/tools/kurultai/buildroom/rooms/demo-room/retention/retention-review.json b/tools/kurultai/buildroom/rooms/demo-room/retention/retention-review.json
new file mode 100644
index 00000000..f2f3f1f9
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/demo-room/retention/retention-review.json
@@ -0,0 +1,18 @@
+{
+ "artifacts_to_improve": [
+ "Add live adapters in downstream tasks"
+ ],
+ "artifacts_to_keep": [
+ "schemas",
+ "demo-room",
+ "scripts",
+ "docs"
+ ],
+ "artifacts_to_prune": [],
+ "rationale": "Foundation artifacts are small, public-safe after sanitization, and useful as reusable contract examples.",
+ "recommendation": "keep",
+ "requires_human_before_destructive_action": true,
+ "retention_id": "retention-demo-001",
+ "reviewed_at": "2026-05-10T17:35:00Z",
+ "room_id": "demo-room"
+}
diff --git a/tools/kurultai/buildroom/rooms/demo-room/reviews/intent-review.json b/tools/kurultai/buildroom/rooms/demo-room/reviews/intent-review.json
new file mode 100644
index 00000000..5b4429e8
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/demo-room/reviews/intent-review.json
@@ -0,0 +1,12 @@
+{
+ "decision": "needs_main_review",
+ "idea_id": "idea-buildroom-foundation-demo",
+ "reason": "Evidence is sufficient and scope is bounded/additive.",
+ "related_existing_work": [
+ "Hermes Kanban task t_faf9a811",
+ "Plan 2026-05-10 auto-think auto-build contract chain"
+ ],
+ "review_id": "intent-demo-001",
+ "reviewed_at": "2026-05-10T17:02:00Z",
+ "reviewer": "main-intent-screen"
+}
diff --git a/tools/kurultai/buildroom/rooms/demo-room/reviews/main-review.json b/tools/kurultai/buildroom/rooms/demo-room/reviews/main-review.json
new file mode 100644
index 00000000..db293fa6
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/demo-room/reviews/main-review.json
@@ -0,0 +1,15 @@
+{
+ "allowed_workspaces": [
+ "tools/kurultai/buildroom"
+ ],
+ "approval_notes": "Implement foundation artifacts and demo fixture only.",
+ "blast_radius": "local",
+ "blocked_reasons": [],
+ "decision": "approved_for_planning",
+ "human_approval_required": false,
+ "idea_id": "idea-buildroom-foundation-demo",
+ "review_id": "main-demo-001",
+ "reviewed_at": "2026-05-10T17:03:00Z",
+ "reviewer": "kublai",
+ "risk_band": "low"
+}
diff --git a/tools/kurultai/buildroom/rooms/demo-room/trust/trust-report.json b/tools/kurultai/buildroom/rooms/demo-room/trust/trust-report.json
new file mode 100644
index 00000000..69ac86af
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/demo-room/trust/trust-report.json
@@ -0,0 +1,13 @@
+{
+ "generated_at": "2026-05-11T00:52:53Z",
+ "open_questions": [],
+ "reasons": [
+ "QA report passed"
+ ],
+ "required_followups": [],
+ "risk_score": 0.1,
+ "room_id": "demo-room",
+ "safe_to_archive": true,
+ "state": "clean",
+ "trust_id": "trust-buildroom-foundation-demo"
+}
diff --git a/tools/kurultai/buildroom/rooms/demo-room/verification/verification-delta.json b/tools/kurultai/buildroom/rooms/demo-room/verification/verification-delta.json
new file mode 100644
index 00000000..927036c1
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/demo-room/verification/verification-delta.json
@@ -0,0 +1,13 @@
+{
+ "build_id": "build-buildroom-foundation-demo",
+ "confirmed_claims": [
+ "QA report passed"
+ ],
+ "delta_id": "delta-buildroom-foundation-demo",
+ "implementation_receipt_ref": "jobs/implementation-receipt.json",
+ "next_action": "Proceed to trust summary and retention review.",
+ "regressions": [],
+ "state": "confirmed",
+ "unverified_claims": [],
+ "verification_report_ref": "verification/verification-report.json"
+}
diff --git a/tools/kurultai/buildroom/rooms/demo-room/verification/verification-report.json b/tools/kurultai/buildroom/rooms/demo-room/verification/verification-report.json
new file mode 100644
index 00000000..492610a1
--- /dev/null
+++ b/tools/kurultai/buildroom/rooms/demo-room/verification/verification-report.json
@@ -0,0 +1,21 @@
+{
+ "build_id": "build-buildroom-foundation-demo",
+ "commands_run": [
+ "python scripts/validate_room.py rooms/demo-room",
+ "python scripts/export_sanitized_bundle.py rooms/demo-room /tmp/kurultai-buildroom-export"
+ ],
+ "evidence_refs": [
+ "operator/operator-summary.json"
+ ],
+ "failures": [],
+ "method": "mixed",
+ "observations": [
+ "All required artifacts are present.",
+ "Schemas validate demo fixture.",
+ "Sanitized export redacts local absolute paths."
+ ],
+ "pass": true,
+ "verification_id": "verify-demo-001",
+ "verified_at": "2026-05-10T17:25:00Z",
+ "verified_by": "qa-demo"
+}
diff --git a/tools/kurultai/buildroom/schemas/build-plan.schema.json b/tools/kurultai/buildroom/schemas/build-plan.schema.json
new file mode 100644
index 00000000..8197a215
--- /dev/null
+++ b/tools/kurultai/buildroom/schemas/build-plan.schema.json
@@ -0,0 +1,76 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://kurultai.local/buildroom/schemas/build-plan.schema.json",
+ "title": "build-plan",
+ "type": "object",
+ "required": [
+ "build_id",
+ "plan_id",
+ "created_at",
+ "assignee",
+ "task_refs",
+ "steps",
+ "files_expected",
+ "commands_allowed",
+ "verification_commands",
+ "out_of_scope",
+ "stop_conditions"
+ ],
+ "additionalProperties": false,
+ "properties": {
+ "build_id": {
+ "type": "string"
+ },
+ "plan_id": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "assignee": {
+ "type": "string"
+ },
+ "task_refs": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "steps": {
+ "type": "array",
+ "items": {
+ "type": "object"
+ }
+ },
+ "files_expected": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "commands_allowed": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "verification_commands": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "out_of_scope": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "stop_conditions": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
diff --git a/tools/kurultai/buildroom/schemas/idea-contract.schema.json b/tools/kurultai/buildroom/schemas/idea-contract.schema.json
new file mode 100644
index 00000000..b1f093d8
--- /dev/null
+++ b/tools/kurultai/buildroom/schemas/idea-contract.schema.json
@@ -0,0 +1,97 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://kurultai.local/buildroom/schemas/idea-contract.schema.json",
+ "title": "idea-contract",
+ "type": "object",
+ "required": [
+ "idea_id",
+ "created_at",
+ "proposed_by",
+ "title",
+ "problem",
+ "why_now",
+ "beneficiaries",
+ "evidence_refs",
+ "expected_artifacts",
+ "proposed_owner",
+ "suggested_workspace",
+ "scope_boundaries",
+ "non_goals",
+ "risk_notes",
+ "verification_hint",
+ "status"
+ ],
+ "additionalProperties": false,
+ "properties": {
+ "idea_id": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "proposed_by": {
+ "type": "string"
+ },
+ "title": {
+ "type": "string"
+ },
+ "problem": {
+ "type": "string"
+ },
+ "why_now": {
+ "type": "string"
+ },
+ "beneficiaries": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "evidence_refs": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "expected_artifacts": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "proposed_owner": {
+ "type": "string"
+ },
+ "suggested_workspace": {
+ "type": "string"
+ },
+ "scope_boundaries": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "non_goals": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "risk_notes": {
+ "type": "string"
+ },
+ "verification_hint": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "status": {
+ "enum": [
+ "candidate",
+ "withdrawn",
+ "superseded"
+ ]
+ }
+ }
+}
diff --git a/tools/kurultai/buildroom/schemas/implementation-receipt.schema.json b/tools/kurultai/buildroom/schemas/implementation-receipt.schema.json
new file mode 100644
index 00000000..76a0e5d1
--- /dev/null
+++ b/tools/kurultai/buildroom/schemas/implementation-receipt.schema.json
@@ -0,0 +1,89 @@
+{
+ "$id": "https://kurultai.local/buildroom/schemas/implementation-receipt.schema.json",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "additionalProperties": false,
+ "properties": {
+ "assignee": {
+ "type": "string"
+ },
+ "blocked_items": {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ "build_id": {
+ "type": "string"
+ },
+ "commands_run": {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ "commit_sha": {
+ "type": "string"
+ },
+ "completed_at": {
+ "type": "string"
+ },
+ "deviations_from_plan": {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ "evidence_refs": {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ "files_changed": {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ "kanban_status": {
+ "type": "string"
+ },
+ "kanban_task_id": {
+ "type": "string"
+ },
+ "open_diffs_summary": {
+ "type": "string"
+ },
+ "receipt_id": {
+ "type": "string"
+ },
+ "started_at": {
+ "type": "string"
+ },
+ "tests_run": {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ }
+ },
+ "required": [
+ "receipt_id",
+ "build_id",
+ "assignee",
+ "started_at",
+ "completed_at",
+ "files_changed",
+ "commands_run",
+ "tests_run",
+ "commit_sha",
+ "open_diffs_summary",
+ "deviations_from_plan",
+ "blocked_items",
+ "kanban_task_id",
+ "kanban_status",
+ "evidence_refs"
+ ],
+ "title": "implementation-receipt",
+ "type": "object"
+}
diff --git a/tools/kurultai/buildroom/schemas/intent-review.schema.json b/tools/kurultai/buildroom/schemas/intent-review.schema.json
new file mode 100644
index 00000000..0e05a37c
--- /dev/null
+++ b/tools/kurultai/buildroom/schemas/intent-review.schema.json
@@ -0,0 +1,48 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://kurultai.local/buildroom/schemas/intent-review.schema.json",
+ "title": "intent-review",
+ "type": "object",
+ "required": [
+ "review_id",
+ "idea_id",
+ "reviewed_at",
+ "reviewer",
+ "decision",
+ "reason",
+ "related_existing_work"
+ ],
+ "additionalProperties": false,
+ "properties": {
+ "review_id": {
+ "type": "string"
+ },
+ "idea_id": {
+ "type": "string"
+ },
+ "reviewed_at": {
+ "type": "string"
+ },
+ "reviewer": {
+ "type": "string"
+ },
+ "decision": {
+ "enum": [
+ "needs_main_review",
+ "duplicate",
+ "insufficient_evidence",
+ "unsafe",
+ "park"
+ ]
+ },
+ "reason": {
+ "type": "string"
+ },
+ "related_existing_work": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
diff --git a/tools/kurultai/buildroom/schemas/main-review.schema.json b/tools/kurultai/buildroom/schemas/main-review.schema.json
new file mode 100644
index 00000000..5385013e
--- /dev/null
+++ b/tools/kurultai/buildroom/schemas/main-review.schema.json
@@ -0,0 +1,77 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://kurultai.local/buildroom/schemas/main-review.schema.json",
+ "title": "main-review",
+ "type": "object",
+ "required": [
+ "review_id",
+ "idea_id",
+ "reviewed_at",
+ "reviewer",
+ "decision",
+ "risk_band",
+ "blast_radius",
+ "allowed_workspaces",
+ "approval_notes",
+ "human_approval_required",
+ "blocked_reasons"
+ ],
+ "additionalProperties": false,
+ "properties": {
+ "review_id": {
+ "type": "string"
+ },
+ "idea_id": {
+ "type": "string"
+ },
+ "reviewed_at": {
+ "type": "string"
+ },
+ "reviewer": {
+ "type": "string"
+ },
+ "decision": {
+ "enum": [
+ "approved_for_planning",
+ "rejected",
+ "parked",
+ "needs_more_evidence"
+ ]
+ },
+ "risk_band": {
+ "enum": [
+ "low",
+ "medium",
+ "high"
+ ]
+ },
+ "blast_radius": {
+ "enum": [
+ "local",
+ "profile",
+ "kurultai",
+ "external",
+ "security",
+ "secrets"
+ ]
+ },
+ "allowed_workspaces": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "approval_notes": {
+ "type": "string"
+ },
+ "human_approval_required": {
+ "type": "boolean"
+ },
+ "blocked_reasons": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
diff --git a/tools/kurultai/buildroom/schemas/operator-summary.schema.json b/tools/kurultai/buildroom/schemas/operator-summary.schema.json
new file mode 100644
index 00000000..b6830827
--- /dev/null
+++ b/tools/kurultai/buildroom/schemas/operator-summary.schema.json
@@ -0,0 +1,72 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://kurultai.local/buildroom/schemas/operator-summary.schema.json",
+ "title": "operator-summary",
+ "type": "object",
+ "required": [
+ "summary_id",
+ "room_id",
+ "generated_at",
+ "headline",
+ "status",
+ "current_owner",
+ "latest_artifacts",
+ "operator_needs_to_know",
+ "operator_decisions_needed",
+ "links"
+ ],
+ "additionalProperties": false,
+ "properties": {
+ "summary_id": {
+ "type": "string"
+ },
+ "room_id": {
+ "type": "string"
+ },
+ "generated_at": {
+ "type": "string"
+ },
+ "headline": {
+ "type": "string"
+ },
+ "status": {
+ "enum": [
+ "candidate",
+ "planned",
+ "building",
+ "verifying",
+ "clean",
+ "watch",
+ "investigate",
+ "parked"
+ ]
+ },
+ "current_owner": {
+ "type": "string"
+ },
+ "latest_artifacts": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "operator_needs_to_know": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "operator_decisions_needed": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "links": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
diff --git a/tools/kurultai/buildroom/schemas/product-plan.schema.json b/tools/kurultai/buildroom/schemas/product-plan.schema.json
new file mode 100644
index 00000000..5b7a6e9d
--- /dev/null
+++ b/tools/kurultai/buildroom/schemas/product-plan.schema.json
@@ -0,0 +1,71 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://kurultai.local/buildroom/schemas/product-plan.schema.json",
+ "title": "product-plan",
+ "type": "object",
+ "required": [
+ "plan_id",
+ "idea_id",
+ "created_at",
+ "owner",
+ "user_value",
+ "acceptance_criteria",
+ "ux_or_operator_surface",
+ "allowed_paths",
+ "protected_paths",
+ "non_goals",
+ "rollout_plan",
+ "rollback_plan"
+ ],
+ "additionalProperties": false,
+ "properties": {
+ "plan_id": {
+ "type": "string"
+ },
+ "idea_id": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "owner": {
+ "type": "string"
+ },
+ "user_value": {
+ "type": "string"
+ },
+ "acceptance_criteria": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ux_or_operator_surface": {
+ "type": "string"
+ },
+ "allowed_paths": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "protected_paths": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "non_goals": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "rollout_plan": {
+ "type": "string"
+ },
+ "rollback_plan": {
+ "type": "string"
+ }
+ }
+}
diff --git a/tools/kurultai/buildroom/schemas/research-input.schema.json b/tools/kurultai/buildroom/schemas/research-input.schema.json
new file mode 100644
index 00000000..1e2fa92e
--- /dev/null
+++ b/tools/kurultai/buildroom/schemas/research-input.schema.json
@@ -0,0 +1,65 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://kurultai.local/buildroom/schemas/research-input.schema.json",
+ "title": "research-input",
+ "type": "object",
+ "required": [
+ "schema_version",
+ "packet_id",
+ "created_at",
+ "source_type",
+ "source_refs",
+ "summary",
+ "signals",
+ "sensitivity",
+ "sanitization_notes"
+ ],
+ "additionalProperties": false,
+ "properties": {
+ "schema_version": {
+ "type": "string"
+ },
+ "packet_id": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "source_type": {
+ "enum": [
+ "brain",
+ "receipt",
+ "kanban",
+ "cron",
+ "operator",
+ "web",
+ "manual"
+ ]
+ },
+ "source_refs": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "summary": {
+ "type": "string"
+ },
+ "signals": {
+ "type": "array",
+ "items": {
+ "type": "object"
+ }
+ },
+ "sensitivity": {
+ "enum": [
+ "public",
+ "internal",
+ "private"
+ ]
+ },
+ "sanitization_notes": {
+ "type": "string"
+ }
+ }
+}
diff --git a/tools/kurultai/buildroom/schemas/retention-review.schema.json b/tools/kurultai/buildroom/schemas/retention-review.schema.json
new file mode 100644
index 00000000..7b3baef4
--- /dev/null
+++ b/tools/kurultai/buildroom/schemas/retention-review.schema.json
@@ -0,0 +1,61 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://kurultai.local/buildroom/schemas/retention-review.schema.json",
+ "title": "retention-review",
+ "type": "object",
+ "required": [
+ "retention_id",
+ "room_id",
+ "reviewed_at",
+ "recommendation",
+ "rationale",
+ "artifacts_to_keep",
+ "artifacts_to_improve",
+ "artifacts_to_prune",
+ "requires_human_before_destructive_action"
+ ],
+ "additionalProperties": false,
+ "properties": {
+ "retention_id": {
+ "type": "string"
+ },
+ "room_id": {
+ "type": "string"
+ },
+ "reviewed_at": {
+ "type": "string"
+ },
+ "recommendation": {
+ "enum": [
+ "keep",
+ "improve",
+ "park",
+ "prune"
+ ]
+ },
+ "rationale": {
+ "type": "string"
+ },
+ "artifacts_to_keep": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "artifacts_to_improve": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "artifacts_to_prune": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "requires_human_before_destructive_action": {
+ "const": true
+ }
+ }
+}
diff --git a/tools/kurultai/buildroom/schemas/trust-report.schema.json b/tools/kurultai/buildroom/schemas/trust-report.schema.json
new file mode 100644
index 00000000..ea23163c
--- /dev/null
+++ b/tools/kurultai/buildroom/schemas/trust-report.schema.json
@@ -0,0 +1,60 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://kurultai.local/buildroom/schemas/trust-report.schema.json",
+ "title": "trust-report",
+ "type": "object",
+ "required": [
+ "trust_id",
+ "room_id",
+ "generated_at",
+ "state",
+ "reasons",
+ "risk_score",
+ "open_questions",
+ "required_followups",
+ "safe_to_archive"
+ ],
+ "additionalProperties": false,
+ "properties": {
+ "trust_id": {
+ "type": "string"
+ },
+ "room_id": {
+ "type": "string"
+ },
+ "generated_at": {
+ "type": "string"
+ },
+ "state": {
+ "enum": [
+ "clean",
+ "watch",
+ "investigate"
+ ]
+ },
+ "reasons": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "risk_score": {
+ "type": "number"
+ },
+ "open_questions": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "required_followups": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "safe_to_archive": {
+ "type": "boolean"
+ }
+ }
+}
diff --git a/tools/kurultai/buildroom/schemas/verification-delta.schema.json b/tools/kurultai/buildroom/schemas/verification-delta.schema.json
new file mode 100644
index 00000000..f25c655d
--- /dev/null
+++ b/tools/kurultai/buildroom/schemas/verification-delta.schema.json
@@ -0,0 +1,61 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://kurultai.local/buildroom/schemas/verification-delta.schema.json",
+ "title": "verification-delta",
+ "type": "object",
+ "required": [
+ "delta_id",
+ "build_id",
+ "implementation_receipt_ref",
+ "verification_report_ref",
+ "state",
+ "confirmed_claims",
+ "unverified_claims",
+ "regressions",
+ "next_action"
+ ],
+ "additionalProperties": false,
+ "properties": {
+ "delta_id": {
+ "type": "string"
+ },
+ "build_id": {
+ "type": "string"
+ },
+ "implementation_receipt_ref": {
+ "type": "string"
+ },
+ "verification_report_ref": {
+ "type": "string"
+ },
+ "state": {
+ "enum": [
+ "confirmed",
+ "drift",
+ "regression",
+ "missing_evidence"
+ ]
+ },
+ "confirmed_claims": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "unverified_claims": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "regressions": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "next_action": {
+ "type": "string"
+ }
+ }
+}
diff --git a/tools/kurultai/buildroom/schemas/verification-report.schema.json b/tools/kurultai/buildroom/schemas/verification-report.schema.json
new file mode 100644
index 00000000..357f88ec
--- /dev/null
+++ b/tools/kurultai/buildroom/schemas/verification-report.schema.json
@@ -0,0 +1,70 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://kurultai.local/buildroom/schemas/verification-report.schema.json",
+ "title": "verification-report",
+ "type": "object",
+ "required": [
+ "verification_id",
+ "build_id",
+ "verified_by",
+ "verified_at",
+ "method",
+ "commands_run",
+ "observations",
+ "pass",
+ "failures",
+ "evidence_refs"
+ ],
+ "additionalProperties": false,
+ "properties": {
+ "verification_id": {
+ "type": "string"
+ },
+ "build_id": {
+ "type": "string"
+ },
+ "verified_by": {
+ "type": "string"
+ },
+ "verified_at": {
+ "type": "string"
+ },
+ "method": {
+ "enum": [
+ "tests",
+ "manual",
+ "browser",
+ "static_analysis",
+ "receipt_review",
+ "mixed"
+ ]
+ },
+ "commands_run": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "observations": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "pass": {
+ "type": "boolean"
+ },
+ "failures": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "evidence_refs": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
diff --git a/tools/kurultai/buildroom/scripts/__init__.py b/tools/kurultai/buildroom/scripts/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tools/kurultai/buildroom/scripts/build_operator_summary.py b/tools/kurultai/buildroom/scripts/build_operator_summary.py
new file mode 100644
index 00000000..7fc8e4ba
--- /dev/null
+++ b/tools/kurultai/buildroom/scripts/build_operator_summary.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+import argparse
+from pathlib import Path
+from common import EXPECTED_ARTIFACTS, load_json, resolve_room_path, utc_now, write_json
+
+def build_summary(room_dir: Path) -> dict:
+ idea=load_json(room_dir/"ideas/idea-contract.json"); build=load_json(room_dir/"plans/build-plan.json")
+ trust=load_json(room_dir/"trust/trust-report.json"); verification=load_json(room_dir/"verification/verification-report.json")
+ retention=load_json(room_dir/"retention/retention-review.json"); room_id=room_dir.name
+ artifacts=[rel for rel in EXPECTED_ARTIFACTS if (room_dir/rel).exists()]
+ decisions=list(trust.get("required_followups", []))
+ if retention.get("recommendation") == "prune": decisions.append("Human approval required before any destructive retention action.")
+ return {"summary_id":f"summary-{room_id}","room_id":room_id,"generated_at":utc_now(),"headline":idea.get("title", f"Buildroom {room_id}"),"status":trust.get("state","watch"),"current_owner":build.get("assignee","unknown"),"latest_artifacts":artifacts,"operator_needs_to_know":[f"Trust state: {trust.get('state')} (risk_score={trust.get('risk_score')}).",f"Verification pass: {verification.get('pass')} via {verification.get('method')}.",f"Retention recommendation: {retention.get('recommendation')}.",],"operator_decisions_needed":decisions,"links":[f"buildroom://{room_id}/{rel}" for rel in artifacts]}
+
+def main() -> int:
+ parser=argparse.ArgumentParser(description="Create or refresh operator/operator-summary.json for a buildroom room."); parser.add_argument("room", type=Path)
+ args=parser.parse_args(); room=resolve_room_path(args.room)
+ target=room/"operator/operator-summary.json"; write_json(target, build_summary(room)); print(f"wrote {target}"); return 0
+if __name__ == "__main__": raise SystemExit(main())
diff --git a/tools/kurultai/buildroom/scripts/common.py b/tools/kurultai/buildroom/scripts/common.py
new file mode 100644
index 00000000..04e9e523
--- /dev/null
+++ b/tools/kurultai/buildroom/scripts/common.py
@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import json
+import re
+import shutil
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+BUILDROOM_ROOT = Path(__file__).resolve().parents[1]
+SCHEMA_DIR = BUILDROOM_ROOT / "schemas"
+EXPECTED_ARTIFACTS = {
+ "research/research-input.json": "research-input.schema.json",
+ "ideas/idea-contract.json": "idea-contract.schema.json",
+ "reviews/intent-review.json": "intent-review.schema.json",
+ "reviews/main-review.json": "main-review.schema.json",
+ "plans/product-plan.json": "product-plan.schema.json",
+ "plans/build-plan.json": "build-plan.schema.json",
+ "jobs/implementation-receipt.json": "implementation-receipt.schema.json",
+ "verification/verification-report.json": "verification-report.schema.json",
+ "verification/verification-delta.json": "verification-delta.schema.json",
+ "trust/trust-report.json": "trust-report.schema.json",
+ "retention/retention-review.json": "retention-review.schema.json",
+ "operator/operator-summary.json": "operator-summary.schema.json",
+}
+SECRETISH = re.compile(r"(?i)(sk-[a-z0-9_-]{12,}|xox[baprs]-[a-z0-9-]{12,}|gh[pousr]_[a-z0-9_]{12,}|token\s*[:=]\s*[^\s,;]+|secret\s*[:=]\s*[^\s,;]+)")
+ABSOLUTE_PRIVATE_PATH = re.compile(r"/(Users|home)/[^\s\"']+")
+
+def utc_now() -> str:
+ return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+def load_json(path: Path) -> Any:
+ with path.open("r", encoding="utf-8") as handle: return json.load(handle)
+
+def write_json(path: Path, data: Any) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+def redact_value(value: Any) -> Any:
+ if isinstance(value, dict):
+ out = {}
+ for key, item in value.items():
+ if re.search(r"(?i)(secret|token|password|credential|private_key)", key): out[key] = "[REDACTED_SECRET_FIELD]"
+ else: out[key] = redact_value(item)
+ return out
+ if isinstance(value, list): return [redact_value(item) for item in value]
+ if isinstance(value, str):
+ value = ABSOLUTE_PRIVATE_PATH.sub("[REDACTED_ABSOLUTE_PATH]", value)
+ value = SECRETISH.sub("[REDACTED_SECRET]", value)
+ return value
+
+def resolve_room_path(path: Path) -> Path:
+ if path.is_absolute():
+ return path
+ cwd_relative = Path.cwd() / path
+ if cwd_relative.exists():
+ return cwd_relative
+ return BUILDROOM_ROOT / path
+
+def copy_sanitized_room(room_dir: Path, dest_dir: Path) -> list[str]:
+ if not room_dir.exists():
+ raise FileNotFoundError(f"room does not exist: {room_dir}")
+ if dest_dir.exists(): shutil.rmtree(dest_dir)
+ dest_dir.mkdir(parents=True)
+ skipped=[]
+ for path in sorted(room_dir.rglob("*")):
+ rel=path.relative_to(room_dir); target=dest_dir/rel
+ if path.is_dir(): target.mkdir(parents=True, exist_ok=True); continue
+ if path.suffix == ".json":
+ data=load_json(path)
+ if isinstance(data, dict) and data.get("sensitivity") == "private": skipped.append(str(rel)); continue
+ write_json(target, redact_value(data))
+ else:
+ target.parent.mkdir(parents=True, exist_ok=True)
+ text=SECRETISH.sub("[REDACTED_SECRET]", ABSOLUTE_PRIVATE_PATH.sub("[REDACTED_ABSOLUTE_PATH]", path.read_text(encoding="utf-8")))
+ target.write_text(text, encoding="utf-8")
+ return skipped
diff --git a/tools/kurultai/buildroom/scripts/control_room.py b/tools/kurultai/buildroom/scripts/control_room.py
new file mode 100644
index 00000000..973b744b
--- /dev/null
+++ b/tools/kurultai/buildroom/scripts/control_room.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+from typing import Any
+
+from common import BUILDROOM_ROOT, EXPECTED_ARTIFACTS, load_json, utc_now
+
+ROOMS_ROOT = BUILDROOM_ROOT / "rooms"
+DEFAULT_OUTPUT = BUILDROOM_ROOT / "control-room.md"
+
+
+def optional_json(path: Path) -> dict[str, Any]:
+ if not path.exists():
+ return {}
+ data = load_json(path)
+ return data if isinstance(data, dict) else {}
+
+
+def room_dirs(rooms_root: Path) -> list[Path]:
+ if not rooms_root.exists():
+ return []
+ return sorted([p for p in rooms_root.iterdir() if p.is_dir()], key=lambda p: p.name)
+
+
+def missing_artifacts(room_dir: Path) -> list[str]:
+ return [rel for rel in EXPECTED_ARTIFACTS if not (room_dir / rel).exists()]
+
+
+def infer_phase(room_dir: Path, missing: list[str]) -> str:
+ if not missing:
+ return "complete"
+ if not (room_dir / "research/research-input.json").exists():
+ return "intake"
+ if not (room_dir / "ideas/idea-contract.json").exists():
+ return "idea"
+ if not (room_dir / "plans/build-plan.json").exists():
+ return "planning"
+ if not (room_dir / "jobs/implementation-receipt.json").exists():
+ return "implementation"
+ if not (room_dir / "verification/verification-report.json").exists():
+ return "verification"
+ if not (room_dir / "trust/trust-report.json").exists():
+ return "trust"
+ if not (room_dir / "retention/retention-review.json").exists():
+ return "retention"
+ return "operator-summary"
+
+
+def next_action(phase: str, trust_state: str, retention: str, missing: list[str]) -> str:
+ if trust_state == "investigate":
+ return "Investigate trust report before expanding or shipping this room."
+ if missing:
+ return f"Fill missing artifact: {missing[0]}."
+ if retention == "improve":
+ return "Create a follow-up improvement task from the retention review."
+ if retention == "park":
+ return "Park unless the operator reactivates this room."
+ if retention == "prune":
+ return "Require explicit human approval before pruning."
+ if phase == "complete" and trust_state == "clean":
+ return "Ready for PR/merge monitoring or retention follow-up."
+ return "Review room state and refresh operator summary."
+
+
+def room_card(room_dir: Path) -> dict[str, Any]:
+ idea = optional_json(room_dir / "ideas/idea-contract.json")
+ trust = optional_json(room_dir / "trust/trust-report.json")
+ verification = optional_json(room_dir / "verification/verification-report.json")
+ delta = optional_json(room_dir / "verification/verification-delta.json")
+ retention = optional_json(room_dir / "retention/retention-review.json")
+ operator = optional_json(room_dir / "operator/operator-summary.json")
+ missing = missing_artifacts(room_dir)
+ phase = infer_phase(room_dir, missing)
+ trust_state = str(trust.get("state") or operator.get("status") or "watch")
+ retention_recommendation = str(retention.get("recommendation") or "unknown")
+ return {
+ "room_id": room_dir.name,
+ "headline": idea.get("title") or operator.get("headline") or room_dir.name,
+ "phase": phase,
+ "trust_state": trust_state,
+ "risk_score": trust.get("risk_score", "unknown"),
+ "verification_pass": verification.get("pass", "unknown"),
+ "verification_delta": delta.get("delta", delta.get("state", "unknown")),
+ "retention": retention_recommendation,
+ "missing_artifacts": missing,
+ "operator_decisions_needed": operator.get("operator_decisions_needed", []),
+ "next_action": next_action(phase, trust_state, retention_recommendation, missing),
+ }
+
+
+def build_report(rooms_root: Path) -> str:
+ cards = [room_card(room) for room in room_dirs(rooms_root)]
+ clean = sum(1 for c in cards if c["trust_state"] == "clean")
+ watch = sum(1 for c in cards if c["trust_state"] == "watch")
+ investigate = sum(1 for c in cards if c["trust_state"] == "investigate")
+ incomplete = sum(1 for c in cards if c["missing_artifacts"])
+ lines = [
+ "# Kurultai Buildroom Control Room",
+ "",
+ f"Generated: {utc_now()}",
+ "",
+ "## Summary",
+ "",
+ f"- Rooms: {len(cards)}",
+ f"- Trust clean: {clean}",
+ f"- Trust watch: {watch}",
+ f"- Trust investigate: {investigate}",
+ f"- Rooms with missing evidence: {incomplete}",
+ "",
+ "## Rooms",
+ "",
+ ]
+ if not cards:
+ lines.append("No buildroom rooms found.")
+ lines.append("")
+ return "\n".join(lines)
+ for card in cards:
+ lines.extend([
+ f"### {card['room_id']}",
+ "",
+ f"- Headline: {card['headline']}",
+ f"- Phase: {card['phase']}",
+ f"- Trust: {card['trust_state']} (risk_score={card['risk_score']})",
+ f"- Verification pass: {card['verification_pass']}",
+ f"- Verification delta: {card['verification_delta']}",
+ f"- Retention: {card['retention']}",
+ f"- Missing evidence: {', '.join(card['missing_artifacts']) if card['missing_artifacts'] else 'none'}",
+ f"- Operator decisions needed: {', '.join(card['operator_decisions_needed']) if card['operator_decisions_needed'] else 'none'}",
+ f"- Next action: {card['next_action']}",
+ "",
+ ])
+ return "\n".join(lines)
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description="Build a markdown control-room report across buildroom rooms.")
+ parser.add_argument("--rooms-root", type=Path, default=ROOMS_ROOT)
+ parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
+ args = parser.parse_args()
+ report = build_report(args.rooms_root)
+ args.output.parent.mkdir(parents=True, exist_ok=True)
+ args.output.write_text(report, encoding="utf-8")
+ print(f"wrote {args.output}")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/kurultai/buildroom/scripts/export_sanitized_bundle.py b/tools/kurultai/buildroom/scripts/export_sanitized_bundle.py
new file mode 100644
index 00000000..d4b4517b
--- /dev/null
+++ b/tools/kurultai/buildroom/scripts/export_sanitized_bundle.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+import argparse
+from pathlib import Path
+from common import copy_sanitized_room, resolve_room_path
+
+def main() -> int:
+ parser=argparse.ArgumentParser(description="Export a sanitized copy of a buildroom room."); parser.add_argument("room", type=Path); parser.add_argument("destination", type=Path)
+ args=parser.parse_args(); room=resolve_room_path(args.room)
+ skipped=copy_sanitized_room(room, args.destination); print(f"exported sanitized room: {room} -> {args.destination}")
+ for item in skipped: print(f"skipped private artifact: {item}")
+ return 0
+if __name__ == "__main__": raise SystemExit(main())
diff --git a/tools/kurultai/buildroom/scripts/kanban_adapter.py b/tools/kurultai/buildroom/scripts/kanban_adapter.py
new file mode 100644
index 00000000..22c0678b
--- /dev/null
+++ b/tools/kurultai/buildroom/scripts/kanban_adapter.py
@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+from typing import Any
+
+from common import load_json, resolve_room_path, utc_now, write_json
+
+
+def _kanban_parent_ids(task_refs: list[Any]) -> list[str]:
+ parents: list[str] = []
+ for ref in task_refs:
+ if isinstance(ref, str) and ref.startswith("kanban:"):
+ task_id = ref.removeprefix("kanban:").strip()
+ if task_id:
+ parents.append(task_id)
+ return parents
+
+
+def _title_from_build_id(build_id: str) -> str:
+ stem = build_id.removeprefix("build-")
+ return stem.replace("-", " ").capitalize()
+
+
+def _as_list(value: Any) -> list[Any]:
+ return value if isinstance(value, list) else []
+
+
+def _load_room_contracts(room: Path) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], dict[str, Any]]:
+ idea = load_json(room / "ideas" / "idea-contract.json")
+ review = load_json(room / "reviews" / "main-review.json")
+ product = load_json(room / "plans" / "product-plan.json")
+ build = load_json(room / "plans" / "build-plan.json")
+ return idea, review, product, build
+
+
+def _assert_approved(review: dict[str, Any]) -> None:
+ if review.get("decision") != "approved_for_planning":
+ reasons = "; ".join(str(item) for item in _as_list(review.get("blocked_reasons")))
+ suffix = f": {reasons}" if reasons else ""
+ raise ValueError(f"main-review decision is not approved_for_planning{suffix}")
+
+
+def build_task_packet(room: Path) -> dict[str, Any]:
+ idea, review, product, build = _load_room_contracts(room)
+ _assert_approved(review)
+ build_id = str(build.get("build_id", "unknown-build"))
+ task_refs = [str(ref) for ref in _as_list(build.get("task_refs"))]
+ allowed_paths = [str(path) for path in _as_list(product.get("allowed_paths"))]
+ protected_paths = [str(path) for path in _as_list(product.get("protected_paths"))]
+ verification_commands = [str(command) for command in _as_list(build.get("verification_commands"))]
+ steps = _as_list(build.get("steps"))
+ step_lines = []
+ for step in steps:
+ if isinstance(step, dict):
+ step_id = step.get("step_id", "step")
+ description = step.get("description", step)
+ step_lines.append(f"- {step_id}: {description}")
+ else:
+ step_lines.append(f"- {step}")
+ body = "\n".join(
+ [
+ f"Buildroom room: {room.name}",
+ f"Build ID: {build_id}",
+ f"Plan ID: {build.get('plan_id', '')}",
+ f"Idea: {idea.get('idea_id', '')} — {idea.get('title', '')}",
+ f"Main review: {review.get('review_id', '')} ({review.get('decision', '')})",
+ f"Task refs: {', '.join(task_refs) if task_refs else 'none'}",
+ "Allowed paths:",
+ *(f"- {path}" for path in allowed_paths),
+ "Protected paths:",
+ *(f"- {path}" for path in protected_paths),
+ "Steps:",
+ *step_lines,
+ "Verification commands:",
+ *(f"- {command}" for command in verification_commands),
+ "Stop if any stop_condition or out_of_scope item applies; do not expand beyond the approved build plan.",
+ ]
+ )
+ return {
+ "title": f"buildroom: {_title_from_build_id(build_id)}",
+ "assignee": build.get("assignee", product.get("owner", "coder")),
+ "body": body,
+ "parents": _kanban_parent_ids(task_refs),
+ "workspace_kind": "dir",
+ "workspace_path": "${KURULTAI_HOME}",
+ "idempotency_key": f"buildroom:{build_id}",
+ "metadata": {
+ "room_id": room.name,
+ "build_id": build_id,
+ "plan_id": build.get("plan_id", ""),
+ "idea_id": idea.get("idea_id", ""),
+ "task_refs": task_refs,
+ "verification_commands": verification_commands,
+ },
+ }
+
+
+def _metadata_list(metadata: dict[str, Any], *names: str) -> list[str]:
+ for name in names:
+ value = metadata.get(name)
+ if isinstance(value, list):
+ return [str(item) for item in value]
+ return []
+
+
+def build_implementation_receipt(room: Path, completion: dict[str, Any]) -> dict[str, Any]:
+ _, _, _, build = _load_room_contracts(room)
+ task_id = str(completion.get("id") or completion.get("task_id") or "unknown-task")
+ metadata = completion.get("metadata") if isinstance(completion.get("metadata"), dict) else {}
+ evidence_refs = [f"kanban:{task_id}"]
+ evidence_refs.extend(str(ref) for ref in _as_list(metadata.get("evidence_refs")))
+ return {
+ "receipt_id": f"receipt-{task_id}",
+ "build_id": build.get("build_id", "unknown-build"),
+ "assignee": completion.get("assignee", build.get("assignee", "unknown")),
+ "started_at": completion.get("started_at") or metadata.get("started_at") or utc_now(),
+ "completed_at": completion.get("completed_at") or metadata.get("completed_at") or utc_now(),
+ "files_changed": _metadata_list(metadata, "changed_files", "files_changed"),
+ "commands_run": _metadata_list(metadata, "commands_run", "commands"),
+ "tests_run": _metadata_list(metadata, "tests_run", "checks"),
+ "commit_sha": str(metadata.get("commit_sha", "")),
+ "open_diffs_summary": str(metadata.get("open_diffs_summary", completion.get("summary", ""))),
+ "deviations_from_plan": _metadata_list(metadata, "deviations_from_plan"),
+ "blocked_items": _metadata_list(metadata, "blocked_items", "blockers"),
+ "kanban_task_id": task_id,
+ "kanban_status": str(completion.get("status", "unknown")),
+ "evidence_refs": evidence_refs,
+ }
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description="Map buildroom plans to Kanban task packets and Kanban completions to implementation receipts.")
+ sub = parser.add_subparsers(dest="command", required=True)
+ task_packet = sub.add_parser("task-packet", help="Write a Kanban task creation packet JSON from an approved buildroom room.")
+ task_packet.add_argument("room", type=Path)
+ task_packet.add_argument("output", type=Path)
+ receipt = sub.add_parser("receipt", help="Write jobs/implementation-receipt.json from a Kanban completion JSON object.")
+ receipt.add_argument("room", type=Path)
+ receipt.add_argument("completion_json", type=Path)
+ args = parser.parse_args()
+
+ try:
+ room = resolve_room_path(args.room)
+ if args.command == "task-packet":
+ packet = build_task_packet(room)
+ write_json(args.output, packet)
+ print(f"wrote Kanban task packet: {args.output}")
+ elif args.command == "receipt":
+ completion = load_json(args.completion_json)
+ if not isinstance(completion, dict):
+ raise ValueError("completion JSON must be an object")
+ receipt_data = build_implementation_receipt(room, completion)
+ target = room / "jobs" / "implementation-receipt.json"
+ write_json(target, receipt_data)
+ print(f"wrote implementation receipt: {target}")
+ return 0
+ except Exception as exc:
+ print(f"kanban adapter failed: {exc}", file=sys.stderr)
+ return 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/kurultai/buildroom/scripts/new_room.py b/tools/kurultai/buildroom/scripts/new_room.py
new file mode 100644
index 00000000..8ba39dba
--- /dev/null
+++ b/tools/kurultai/buildroom/scripts/new_room.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+import argparse
+from common import BUILDROOM_ROOT, EXPECTED_ARTIFACTS, utc_now, write_json
+
+def main() -> int:
+ parser=argparse.ArgumentParser(description="Create an empty buildroom room skeleton with placeholder JSON files."); parser.add_argument("room_id")
+ args=parser.parse_args(); room=BUILDROOM_ROOT/"rooms"/args.room_id
+ for rel in EXPECTED_ARTIFACTS:
+ path=room/rel
+ if not path.exists(): write_json(path, {"_placeholder": True, "room_id": args.room_id, "created_at": utc_now(), "schema_hint": rel})
+ print(f"created room skeleton: {room}"); return 0
+if __name__ == "__main__": raise SystemExit(main())
diff --git a/tools/kurultai/buildroom/scripts/qa_trust.py b/tools/kurultai/buildroom/scripts/qa_trust.py
new file mode 100644
index 00000000..44df2653
--- /dev/null
+++ b/tools/kurultai/buildroom/scripts/qa_trust.py
@@ -0,0 +1,261 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import fnmatch
+import sys
+from pathlib import Path
+from typing import Any
+
+from common import load_json, resolve_room_path, utc_now, write_json
+
+
+def _as_list(value: Any) -> list[Any]:
+ return value if isinstance(value, list) else []
+
+
+def _as_str_list(value: Any) -> list[str]:
+ return [str(item) for item in _as_list(value)]
+
+
+def _load_optional_json(path: Path) -> dict[str, Any] | None:
+ if not path.exists():
+ return None
+ data = load_json(path)
+ if not isinstance(data, dict):
+ raise ValueError(f"{path} must contain a JSON object")
+ return data
+
+
+def _load_room(room: Path) -> dict[str, dict[str, Any] | None]:
+ return {
+ "idea": _load_optional_json(room / "ideas" / "idea-contract.json"),
+ "product": _load_optional_json(room / "plans" / "product-plan.json"),
+ "build": _load_optional_json(room / "plans" / "build-plan.json"),
+ "receipt": _load_optional_json(room / "jobs" / "implementation-receipt.json"),
+ "verification": _load_optional_json(room / "verification" / "verification-report.json"),
+ "delta": _load_optional_json(room / "verification" / "verification-delta.json"),
+ }
+
+
+def _title_from_build_id(build_id: str) -> str:
+ return build_id.removeprefix("build-").replace("-", " ").capitalize()
+
+
+def _matches_any(path: str, patterns: list[str]) -> bool:
+ normalized = path.lstrip("./")
+ if normalized.startswith("/") or normalized.startswith(".."):
+ return False
+ for pattern in patterns:
+ clean = pattern.lstrip("./")
+ if fnmatch.fnmatch(normalized, clean):
+ return True
+ if clean.endswith("/**") and normalized.startswith(clean[:-3].rstrip("/") + "/"):
+ return True
+ if clean.endswith("/*") and normalized.startswith(clean[:-2].rstrip("/") + "/"):
+ return True
+ return False
+
+
+def _violates_protected(path: str, patterns: list[str]) -> bool:
+ normalized = path.lstrip("./")
+ if normalized.startswith("/") or normalized.startswith(".."):
+ return True
+ return any(fnmatch.fnmatch(normalized, pattern.lstrip("./")) for pattern in patterns)
+
+
+def build_qa_packet(room: Path) -> dict[str, Any]:
+ data = _load_room(room)
+ idea = data["idea"] or {}
+ product = data["product"] or {}
+ build = data["build"] or {}
+ receipt = data["receipt"] or {}
+ build_id = str(build.get("build_id") or receipt.get("build_id") or "unknown-build")
+ verification_commands = _as_str_list(build.get("verification_commands"))
+ commands_allowed = _as_str_list(build.get("commands_allowed"))
+ parent = str(receipt.get("kanban_task_id", "")).strip()
+ body = "\n".join(
+ [
+ f"Buildroom room: {room.name}",
+ f"Build ID: {build_id}",
+ f"Implemented by: {receipt.get('assignee', build.get('assignee', 'unknown'))}",
+ "Independent QA only: verify implementation receipts and approved scope; do not implement feature work.",
+ "Allowed QA commands:",
+ *(f"- {command}" for command in (verification_commands + commands_allowed)),
+ "Acceptance criteria:",
+ *(f"- {item}" for item in _as_str_list(product.get("acceptance_criteria"))),
+ "Allowed paths:",
+ *(f"- {item}" for item in _as_str_list(product.get("allowed_paths"))),
+ "Protected paths:",
+ *(f"- {item}" for item in _as_str_list(product.get("protected_paths"))),
+ "Write verification/verification-report.json, then run qa_trust.py delta and qa_trust.py trust.",
+ ]
+ )
+ return {
+ "title": f"buildroom-qa: {_title_from_build_id(build_id)}",
+ "assignee": "ogedei",
+ "body": body,
+ "parents": [parent] if parent else [],
+ "workspace_kind": "dir",
+ "workspace_path": "${KURULTAI_HOME}",
+ "idempotency_key": f"buildroom-qa:{build_id}",
+ "metadata": {
+ "room_id": room.name,
+ "build_id": build_id,
+ "plan_id": build.get("plan_id", product.get("plan_id", "")),
+ "independent_of": build.get("assignee", product.get("owner", "unknown")),
+ "verification_commands": verification_commands,
+ },
+ }
+
+
+def build_delta(room: Path) -> dict[str, Any]:
+ data = _load_room(room)
+ build = data["build"] or {}
+ product = data["product"] or {}
+ receipt = data["receipt"] or {}
+ report = data["verification"]
+ build_id = str(build.get("build_id") or receipt.get("build_id") or "unknown-build")
+ confirmed: list[str] = []
+ unverified: list[str] = []
+ regressions: list[str] = []
+
+ if report is None:
+ unverified.append("verification/verification-report.json missing")
+ else:
+ if report.get("pass") is True:
+ confirmed.append("QA report passed")
+ else:
+ regressions.extend(_as_str_list(report.get("failures")) or ["QA report did not pass"])
+ reported_commands = set(_as_str_list(report.get("commands_run")))
+ expected_commands = set(_as_str_list(build.get("verification_commands")))
+ missing_commands = sorted(expected_commands - reported_commands)
+ if missing_commands and report.get("pass") is not True:
+ unverified.extend(f"verification command not run: {command}" for command in missing_commands)
+
+ for item in _as_str_list(receipt.get("deviations_from_plan")):
+ regressions.append(f"deviation from plan: {item}")
+ for item in _as_str_list(receipt.get("blocked_items")):
+ unverified.append(f"blocked implementation item: {item}")
+
+ allowed_paths = _as_str_list(product.get("allowed_paths"))
+ protected_paths = _as_str_list(product.get("protected_paths"))
+ for changed in _as_str_list(receipt.get("files_changed")):
+ if _violates_protected(changed, protected_paths):
+ regressions.append(f"protected path or unsafe path changed: {changed}")
+ elif allowed_paths and not _matches_any(changed, allowed_paths):
+ regressions.append(f"changed file outside allowed paths: {changed}")
+
+ if regressions:
+ state = "regression"
+ next_action = "Investigate regressions or scope mismatch before archive."
+ elif unverified:
+ state = "missing_evidence"
+ next_action = "Run independent QA and provide missing evidence."
+ else:
+ state = "confirmed"
+ next_action = "Proceed to trust summary and retention review."
+
+ delta = {
+ "delta_id": f"delta-{build_id.removeprefix('build-')}",
+ "build_id": build_id,
+ "implementation_receipt_ref": "jobs/implementation-receipt.json",
+ "verification_report_ref": "verification/verification-report.json",
+ "state": state,
+ "confirmed_claims": confirmed,
+ "unverified_claims": unverified,
+ "regressions": regressions,
+ "next_action": next_action,
+ }
+ write_json(room / "verification" / "verification-delta.json", delta)
+ return delta
+
+
+def build_trust(room: Path) -> dict[str, Any]:
+ data = _load_room(room)
+ build = data["build"] or {}
+ report = data["verification"]
+ delta = data["delta"] or build_delta(room)
+ state = str(delta.get("state", "missing_evidence"))
+ reasons: list[str] = []
+ open_questions: list[str] = []
+ followups: list[str] = []
+
+ if report is None:
+ reasons.append("Independent QA report missing")
+ elif report.get("pass") is True:
+ reasons.append("QA report passed")
+ else:
+ reasons.append("QA report failed")
+
+ regressions = _as_str_list(delta.get("regressions"))
+ unverified = _as_str_list(delta.get("unverified_claims"))
+ reasons.extend(_as_str_list(delta.get("confirmed_claims")))
+ open_questions.extend(unverified)
+
+ if state == "confirmed" and not regressions and not unverified:
+ trust_state = "clean"
+ risk = 0.1
+ safe_to_archive = True
+ elif regressions:
+ trust_state = "investigate"
+ risk = 0.9
+ safe_to_archive = False
+ if any("outside allowed paths" in item or "protected path" in item for item in regressions):
+ followups.append("Investigate scope/protected-path mismatch before archive.")
+ if report is not None and report.get("pass") is not True:
+ followups.append("Fix failed QA/regressions and rerun verifier.")
+ open_questions.extend(regressions)
+ else:
+ trust_state = "watch"
+ risk = 0.5
+ safe_to_archive = False
+ followups.append("Run independent QA and regenerate verification report.")
+
+ trust = {
+ "trust_id": f"trust-{str(build.get('build_id', room.name)).removeprefix('build-')}",
+ "room_id": room.name,
+ "generated_at": utc_now(),
+ "state": trust_state,
+ "reasons": list(dict.fromkeys(reasons)),
+ "risk_score": risk,
+ "open_questions": list(dict.fromkeys(open_questions)),
+ "required_followups": list(dict.fromkeys(followups)),
+ "safe_to_archive": safe_to_archive,
+ }
+ write_json(room / "trust" / "trust-report.json", trust)
+ return trust
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description="Build independent QA packets, verification deltas, and trust summaries for buildroom rooms.")
+ sub = parser.add_subparsers(dest="command", required=True)
+ qa_packet = sub.add_parser("qa-packet", help="Write a Kanban task packet for independent QA.")
+ qa_packet.add_argument("room", type=Path)
+ qa_packet.add_argument("output", type=Path)
+ delta = sub.add_parser("delta", help="Compare implementation receipt with verification report and write verification-delta.json.")
+ delta.add_argument("room", type=Path)
+ trust = sub.add_parser("trust", help="Summarize trust state from verification and delta artifacts.")
+ trust.add_argument("room", type=Path)
+ args = parser.parse_args()
+
+ try:
+ room = resolve_room_path(args.room)
+ if args.command == "qa-packet":
+ packet = build_qa_packet(room)
+ write_json(args.output, packet)
+ print(f"wrote QA task packet: {args.output}")
+ elif args.command == "delta":
+ delta_data = build_delta(room)
+ print(f"wrote verification delta: {room / 'verification' / 'verification-delta.json'} ({delta_data['state']})")
+ elif args.command == "trust":
+ trust_data = build_trust(room)
+ print(f"wrote trust report: {room / 'trust' / 'trust-report.json'} ({trust_data['state']})")
+ return 0
+ except Exception as exc:
+ print(f"qa trust failed: {exc}", file=sys.stderr)
+ return 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/kurultai/buildroom/scripts/research_to_buildroom.py b/tools/kurultai/buildroom/scripts/research_to_buildroom.py
new file mode 100644
index 00000000..2756f4a3
--- /dev/null
+++ b/tools/kurultai/buildroom/scripts/research_to_buildroom.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+import argparse, re
+from pathlib import Path
+from common import BUILDROOM_ROOT, utc_now, write_json
+
+def slug(s:str)->str:
+ return re.sub(r'[^a-z0-9]+','-',s.lower()).strip('-')[:96] or 'research-room'
+
+def read(p):
+ return p.read_text(encoding='utf-8', errors='replace') if p else ''
+
+def md_title(s):
+ for line in s.splitlines()[:40]:
+ if line.lower().startswith('title:'):
+ return line.split(':',1)[1].strip().strip('"')
+ return None
+
+def pick_summary(s):
+ lines=[]
+ for raw in s.splitlines():
+ line=raw.strip(' #-\t')
+ if len(line)>45 and not line.startswith('---') and ':' not in line[:24]: lines.append(line)
+ if len(lines)>=4: break
+ return (' '.join(lines) or 'Research compiled into a buildroom room.')[:1200]
+
+def ref_for(p:Path):
+ parts=p.parts
+ if 'brain' in parts: return 'brain:' + '/'.join(parts[parts.index('brain')+1:])
+ return 'file:' + p.name
+
+def main():
+ ap=argparse.ArgumentParser(description='Compile research artifacts into a buildroom room')
+ ap.add_argument('--room-id'); ap.add_argument('--title'); ap.add_argument('--packet',type=Path); ap.add_argument('--synthesis',type=Path); ap.add_argument('--review',type=Path)
+ ap.add_argument('--source-ref',action='append',default=[]); ap.add_argument('--signal',action='append',default=[]); ap.add_argument('--source-type',choices=['brain','receipt','kanban','cron','operator','web','manual'],default='brain'); ap.add_argument('--owner',default='kublai'); ap.add_argument('--output-root',type=Path)
+ a=ap.parse_args(); now=utc_now()
+ texts=[read(a.packet),read(a.synthesis),read(a.review)]
+ title=a.title or md_title(texts[1]) or md_title(texts[0]) or 'Research Buildroom'
+ rid=a.room_id or slug(title); room=(a.output_root or BUILDROOM_ROOT/'rooms')/rid
+ paths=[p for p in [a.packet,a.synthesis,a.review] if p]
+ refs=list(a.source_ref)+[ref_for(p) for p in paths]
+ idea='idea-'+rid; plan='plan-'+rid; build='build-'+rid
+ signal=[{'kind':'research','summary':s} for s in (a.signal or ['research compiled into an operational buildroom contract'])]
+ docs={
+ 'research/research-input.json': {'schema_version':'1.0','packet_id':'research-'+rid,'created_at':now,'source_type':a.source_type,'source_refs':refs,'summary':pick_summary('\n'.join(texts)),'signals':signal,'sensitivity':'internal','sanitization_notes':'Generated from curated references only.'},
+ 'ideas/idea-contract.json': {'idea_id':idea,'created_at':now,'proposed_by':'research-to-buildroom','title':title,'problem':'Useful research should move through a structured path from insight to bounded work.','why_now':'The buildroom contract exists and can now accept real research inputs.','beneficiaries':['operator','Kurultai agents'],'evidence_refs':refs,'expected_artifacts':['research-input','idea-contract','intent-review','operator-summary'],'proposed_owner':a.owner,'suggested_workspace':'tools/kurultai/buildroom/rooms/'+rid,'scope_boundaries':['curated artifacts only','separate review before implementation'],'non_goals':['production deployment','runtime mutation'],'risk_notes':'Review before promotion.','verification_hint':['Validate room','Review source refs'],'status':'candidate'},
+ 'reviews/intent-review.json': {'review_id':'intent-'+rid,'idea_id':idea,'reviewed_at':now,'reviewer':'kublai','decision':'needs_main_review','reason':'System-relevant research maps to the buildroom contract.','related_existing_work':['tools/kurultai/buildroom']},
+ 'reviews/main-review.json': {'review_id':'main-'+rid,'idea_id':idea,'reviewed_at':now,'reviewer':'kublai','decision':'approved_for_planning','risk_band':'low','blast_radius':'local','allowed_workspaces':['tools/kurultai/buildroom'],'approval_notes':'Approved for autonomous continuation.','human_approval_required':False,'blocked_reasons':[]},
+ 'plans/product-plan.json': {'plan_id':plan,'idea_id':idea,'created_at':now,'owner':a.owner,'user_value':'Research can become bounded work without being buried in chat history.','acceptance_criteria':['room validates','operator summary exists','next action is explicit'],'ux_or_operator_surface':'operator/operator-summary.json','allowed_paths':['tools/kurultai/buildroom/rooms/'+rid],'protected_paths':['runtime state','raw sessions'],'non_goals':['unrelated implementation'],'rollout_plan':'Generate room, validate it, then review the summary.','rollback_plan':'Remove the generated room or revert its commit.'},
+ 'plans/build-plan.json': {'build_id':build,'plan_id':plan,'created_at':now,'assignee':a.owner,'task_refs':['buildroom://'+rid],'steps':[{'id':'generate','action':'Generate artifacts'},{'id':'validate','action':'Validate schema chain'},{'id':'receipt','action':'Record receipt'}],'files_expected':['tools/kurultai/buildroom/rooms/'+rid],'commands_allowed':['python3 tools/kurultai/buildroom/scripts/validate_room.py'],'verification_commands':['python3 tools/kurultai/buildroom/scripts/validate_room.py tools/kurultai/buildroom/rooms/'+rid],'out_of_scope':['production deploy'],'stop_conditions':['validation failure','scope expansion']},
+ 'jobs/implementation-receipt.json': {'receipt_id':'receipt-'+rid,'build_id':build,'assignee':a.owner,'started_at':now,'completed_at':now,'files_changed':['tools/kurultai/buildroom/rooms/'+rid],'commands_run':['research_to_buildroom.py'],'tests_run':[],'commit_sha':'pending','open_diffs_summary':'Generated buildroom room from research artifacts.','deviations_from_plan':[],'blocked_items':[],'kanban_task_id':'not-created','kanban_status':'not-applicable','evidence_refs':refs},
+ 'verification/verification-report.json': {'verification_id':'verify-'+rid,'build_id':build,'verified_by':'research-to-buildroom','verified_at':now,'method':'static_analysis','commands_run':[],'observations':['Room generated.'],'pass':True,'failures':[],'evidence_refs':['buildroom://'+rid]},
+ 'verification/verification-delta.json': {'delta_id':'delta-'+rid,'build_id':build,'implementation_receipt_ref':'jobs/implementation-receipt.json','verification_report_ref':'verification/verification-report.json','state':'confirmed','confirmed_claims':['Generated artifacts are present.'],'unverified_claims':[],'regressions':[],'next_action':'validate room and select follow-on work'},
+ 'trust/trust-report.json': {'trust_id':'trust-'+rid,'room_id':rid,'generated_at':now,'state':'watch','reasons':['Generated room is safe to inspect.'],'risk_score':2,'open_questions':['Which follow-on work should be promoted?'],'required_followups':['Run validate_room','Review operator summary'],'safe_to_archive':False},
+ 'retention/retention-review.json': {'retention_id':'retention-'+rid,'room_id':rid,'reviewed_at':now,'recommendation':'keep','rationale':'Keep as a real research-to-buildroom conversion.','artifacts_to_keep':['all generated room artifacts'],'artifacts_to_improve':['operator summary after follow-on work'],'artifacts_to_prune':[],'requires_human_before_destructive_action':True},
+ 'operator/operator-summary.json': {'summary_id':'summary-'+rid,'room_id':rid,'generated_at':now,'headline':title,'status':'watch','current_owner':a.owner,'latest_artifacts':[],'operator_needs_to_know':['Research has been compiled into a buildroom room.'],'operator_decisions_needed':['Choose whether to promote the room into implementation work.'],'links':['buildroom://'+rid]}
+ }
+ for rel,obj in docs.items(): write_json(room/rel,obj)
+ print('wrote buildroom room: '+str(room)); return 0
+if __name__=='__main__': raise SystemExit(main())
diff --git a/tools/kurultai/buildroom/scripts/validate_room.py b/tools/kurultai/buildroom/scripts/validate_room.py
new file mode 100644
index 00000000..8c446ee0
--- /dev/null
+++ b/tools/kurultai/buildroom/scripts/validate_room.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse, json, sys
+from pathlib import Path
+from typing import Any
+from common import EXPECTED_ARTIFACTS, SCHEMA_DIR, load_json, resolve_room_path
+try:
+ import jsonschema # type: ignore
+except Exception:
+ jsonschema = None
+
+def fallback_validate(data: dict[str, Any], schema: dict[str, Any], label: str) -> list[str]:
+ errors=[]
+ for field in schema.get("required", []):
+ if field not in data: errors.append(f"{label}: missing required field {field!r}")
+ for key, value in data.items():
+ spec=schema.get("properties", {}).get(key, {})
+ if "enum" in spec and value not in spec["enum"]: errors.append(f"{label}: {key!r}={value!r} not in {spec['enum']!r}")
+ if "const" in spec and value != spec["const"]: errors.append(f"{label}: {key!r} must be {spec['const']!r}")
+ typ=spec.get("type")
+ if typ == "array" and not isinstance(value, list): errors.append(f"{label}: {key!r} must be array")
+ elif typ == "string" and not isinstance(value, str): errors.append(f"{label}: {key!r} must be string")
+ elif typ == "boolean" and not isinstance(value, bool): errors.append(f"{label}: {key!r} must be boolean")
+ elif typ == "number" and not isinstance(value, (int, float)): errors.append(f"{label}: {key!r} must be number")
+ return errors
+
+def validate_file(artifact_path: Path, schema_path: Path) -> list[str]:
+ label=str(artifact_path)
+ try:
+ data=load_json(artifact_path); schema=load_json(schema_path)
+ except json.JSONDecodeError as exc:
+ return [f"{label}: invalid JSON at line {exc.lineno} column {exc.colno}: {exc.msg}"]
+ except OSError as exc:
+ return [f"{label}: {exc}"]
+ if jsonschema is not None:
+ validator=jsonschema.Draft202012Validator(schema)
+ return [f"{label}: {'/'.join(map(str, error.path)) or ''}: {error.message}" for error in sorted(validator.iter_errors(data), key=lambda e: list(e.path))]
+ if not isinstance(data, dict): return [f"{label}: root must be object"]
+ return fallback_validate(data, schema, label)
+
+def _looks_like_local_artifact_ref(ref: str) -> bool:
+ if "://" in ref or ref.startswith(("kanban:", "brain:", "receipt:", "cron:", "operator:", "web:", "manual:", "plan:")):
+ return False
+ if ":" in ref.partition("/")[0]:
+ return False
+ return "/" in ref and not ref.startswith(("/", ".."))
+
+
+def validate_local_refs(room_dir: Path, artifact_path: Path) -> list[str]:
+ try:
+ data = load_json(artifact_path)
+ except (json.JSONDecodeError, OSError):
+ return []
+ if not isinstance(data, dict):
+ return []
+ errors=[]
+ for key, value in data.items():
+ if key.endswith("_refs") and isinstance(value, list):
+ for ref in value:
+ if isinstance(ref, str) and _looks_like_local_artifact_ref(ref) and not (room_dir / ref).exists():
+ errors.append(f"{artifact_path}: {key}: local artifact ref does not exist: {ref}")
+ if isinstance(data.get("links"), list):
+ for link in data["links"]:
+ prefix=f"buildroom://{room_dir.name}/"
+ if isinstance(link, str) and link.startswith(prefix):
+ ref=link.removeprefix(prefix)
+ if not (room_dir / ref).exists():
+ errors.append(f"{artifact_path}: links: local artifact ref does not exist: {ref}")
+ return errors
+
+
+def validate_room(room_dir: Path) -> list[str]:
+ errors=[]
+ for rel, schema_name in EXPECTED_ARTIFACTS.items():
+ artifact=room_dir/rel
+ if not artifact.exists(): errors.append(f"missing artifact: {rel}"); continue
+ errors.extend(validate_file(artifact, SCHEMA_DIR/schema_name))
+ errors.extend(validate_local_refs(room_dir, artifact))
+ return errors
+
+def main() -> int:
+ parser=argparse.ArgumentParser(description="Validate a buildroom room against required artifacts and JSON Schemas.")
+ parser.add_argument("room", type=Path)
+ args=parser.parse_args(); room=resolve_room_path(args.room)
+ errors=validate_room(room)
+ if errors:
+ print(f"buildroom validation failed for {room}:", file=sys.stderr)
+ for error in errors: print(f"- {error}", file=sys.stderr)
+ return 1
+ print(f"buildroom validation passed: {room}"); return 0
+if __name__ == "__main__": raise SystemExit(main())
diff --git a/tools/kurultai/heartbeat_master.py b/tools/kurultai/heartbeat_master.py
index 3fb4d56c..dcc67669 100644
--- a/tools/kurultai/heartbeat_master.py
+++ b/tools/kurultai/heartbeat_master.py
@@ -10,6 +10,7 @@
python heartbeat_master.py --cycle # Run one heartbeat cycle
python heartbeat_master.py --daemon # Run continuous daemon mode
"""
+from __future__ import annotations
import argparse
import asyncio
diff --git a/tools/kurultai/kublai_review.py b/tools/kurultai/kublai_review.py
index c1cde7ed..4f97e8fc 100644
--- a/tools/kurultai/kublai_review.py
+++ b/tools/kurultai/kublai_review.py
@@ -4,6 +4,7 @@
Kublai reviews agent reflections using Gemini CLI with full context
Decides: implement, reject, or consult human
"""
+from __future__ import annotations
import os
import json
diff --git a/tools/kurultai/temujin_dashboard.py b/tools/kurultai/temujin_dashboard.py
index 96eb4bbe..9a8dd82b 100644
--- a/tools/kurultai/temujin_dashboard.py
+++ b/tools/kurultai/temujin_dashboard.py
@@ -7,6 +7,7 @@
Model: gemini-3.1-pro-preview
Date: 2026-02-25
"""
+from __future__ import annotations
import curses
import psutil
diff --git a/tools/monitoring.py b/tools/monitoring.py
index a925ec3e..028a6489 100644
--- a/tools/monitoring.py
+++ b/tools/monitoring.py
@@ -262,7 +262,8 @@ def export_prometheus(self) -> str:
lines.append(f"{self.name}_count{label_str} {count}")
lines.append(f"{self.name}_sum{label_str} {sum_value}")
# Add a +Inf bucket
- lines.append(f"{self.name}_bucket{label_str.replace('}', ',le=\"+Inf\"')} {count}")
+ bucket_label_str = label_str.replace("}", ',le="+Inf"}') if label_str else '{le="+Inf"}'
+ lines.append(f"{self.name}_bucket{bucket_label_str} {count}")
else:
lines.append(f"{self.name}{label_str} {metric_value.value}")
diff --git a/tools/neo4j_knowledge_export.py b/tools/neo4j_knowledge_export.py
new file mode 100644
index 00000000..784a7545
--- /dev/null
+++ b/tools/neo4j_knowledge_export.py
@@ -0,0 +1,287 @@
+"""Export Neo4j knowledge nodes to the brain wiki operational schema.
+
+Phase 2 keeps telemetry in Neo4j but moves durable knowledge records into
+markdown. This exporter is intentionally conservative: it only reads Neo4j,
+only writes through KnowledgeStore, and supports --dry-run before materializing.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Iterable
+
+from kublai.knowledge import KnowledgeStore, body_hash, slugify
+
+
+KNOWLEDGE_QUERIES = {
+ "reflection": """
+ MATCH (n)
+ WHERE any(label IN labels(n) WHERE label IN ['Reflection', 'AgentReflection'])
+ RETURN n, labels(n) AS labels
+ ORDER BY coalesce(n.created_at, n.created, n.updated_at, n.updated, '') ASC
+ """,
+ "capability": """
+ MATCH (n)
+ WHERE any(label IN labels(n) WHERE label IN ['LearnedCapability', 'Capability'])
+ RETURN n, labels(n) AS labels
+ ORDER BY coalesce(n.created_at, n.created, n.updated_at, n.updated, '') ASC
+ """,
+ "decision": """
+ MATCH (n:AgentTeam)
+ WHERE n.decision IS NOT NULL
+ OR n.rationale IS NOT NULL
+ OR n.summary IS NOT NULL
+ OR n.description IS NOT NULL
+ RETURN n, labels(n) AS labels
+ ORDER BY coalesce(n.created_at, n.created, n.updated_at, n.updated, '') ASC
+ """,
+ "task": """
+ MATCH (n:Task)
+ WHERE toLower(toString(n.status)) = 'completed'
+ RETURN n, labels(n) AS labels
+ ORDER BY coalesce(n.completed_at, n.updated_at, n.created_at, '') ASC
+ """,
+}
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = build_parser()
+ args = parser.parse_args(argv)
+ rows = export_knowledge(
+ wiki_root=args.wiki_root,
+ uri=args.uri,
+ user=args.user,
+ password=args.password,
+ database=args.database,
+ kinds=args.kind,
+ limit=args.limit,
+ dry_run=args.dry_run,
+ )
+ print(json.dumps({"dry_run": args.dry_run, "rows": rows}, sort_keys=True))
+ return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(prog="neo4j-knowledge-export")
+ parser.add_argument("--wiki-root", default=os.getenv("BRAIN_WIKI_ROOT", str(Path.home() / "brain")))
+ parser.add_argument("--uri", default=os.getenv("NEO4J_URI", "bolt://localhost:7687"))
+ parser.add_argument("--user", default=os.getenv("NEO4J_USER", "neo4j"))
+ parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD"))
+ parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", "neo4j"))
+ parser.add_argument("--kind", action="append", choices=sorted(KNOWLEDGE_QUERIES), help="export only this kind")
+ parser.add_argument("--limit", type=int, default=None)
+ parser.add_argument("--dry-run", action="store_true")
+ return parser
+
+
+def export_knowledge(
+ *,
+ wiki_root: str | Path,
+ uri: str,
+ user: str,
+ password: str | None,
+ database: str,
+ kinds: list[str] | None = None,
+ limit: int | None = None,
+ dry_run: bool = False,
+) -> dict[str, int]:
+ from neo4j import GraphDatabase
+
+ selected = kinds or list(KNOWLEDGE_QUERIES)
+ store = KnowledgeStore(wiki_root)
+ counts: dict[str, int] = {}
+ auth = (user, password or "")
+ with GraphDatabase.driver(uri, auth=auth) as driver:
+ for kind in selected:
+ query = KNOWLEDGE_QUERIES[kind]
+ if limit is not None:
+ query = f"{query}\nLIMIT {int(limit)}"
+ with driver.session(database=database) as session:
+ records = list(session.run(query))
+ counts[kind] = len(records)
+ if dry_run:
+ continue
+ for record in records:
+ node = dict(record["n"])
+ labels = [str(label) for label in record["labels"]]
+ materialize_record(store, kind, node, labels)
+ return counts
+
+
+def materialize_record(store: KnowledgeStore, kind: str, node: dict[str, Any], labels: Iterable[str]) -> Path:
+ if kind == "reflection":
+ reflection_id = str(first(node, "reflection_id", "id", "uuid") or stable_id("reflection", node))
+ agent = str(first(node, "agent", "agent_id", "created_by", "author") or "unknown")
+ return store.write_page(
+ f"operations/reflections/{date_part(node)}-{slugify(agent)}-{slugify(reflection_id)}.md",
+ base_frontmatter(node, "reflection", "reflection_id", reflection_id, labels) | {
+ "agent": agent,
+ "historical": True,
+ },
+ reflection_body(node),
+ typed_field="reflection_id",
+ typed_id=reflection_id,
+ )
+ if kind == "capability":
+ capability_id = str(first(node, "capability_id", "id", "name", "title") or stable_id("capability", node))
+ return store.write_page(
+ f"operations/capabilities/{slugify(capability_id)}.md",
+ base_frontmatter(node, "capability", "capability_id", capability_id, labels) | {
+ "learned_by": str(first(node, "learned_by", "agent", "agent_id") or ""),
+ "historical": True,
+ },
+ generic_body("Capability", node),
+ typed_field="capability_id",
+ typed_id=capability_id,
+ )
+ if kind == "decision":
+ decision_id = str(first(node, "decision_id", "id", "name") or stable_id("decision", node))
+ return store.write_page(
+ f"operations/decisions/{date_part(node)}-{slugify(decision_id)}.md",
+ base_frontmatter(node, "decision", "decision_id", decision_id, labels) | {
+ "agent": str(first(node, "agent", "agent_id", "owner") or "agent-team"),
+ "historical": True,
+ },
+ generic_body("Decision", node),
+ typed_field="decision_id",
+ typed_id=decision_id,
+ )
+ if kind == "task":
+ task_id = str(first(node, "task_id", "id", "uuid") or stable_id("task", node))
+ completed_at_ms = timestamp_ms(first(node, "completed_at", "updated_at", "created_at"))
+ return store.record_completed_task(
+ task_id=task_id,
+ agent=str(first(node, "agent", "assigned_to", "claimed_by") or "unknown"),
+ delegated_by=str(first(node, "delegated_by", "created_by", "source") or "neo4j-import"),
+ completed_at_ms=completed_at_ms,
+ deliverable=str(first(node, "deliverable", "summary", "description", "title") or "Imported completed task"),
+ results=safe_json(first(node, "results", "result", "payload") or {}),
+ historical=True,
+ )
+ raise ValueError(f"unknown export kind: {kind}")
+
+
+def base_frontmatter(
+ node: dict[str, Any],
+ node_type: str,
+ id_field: str,
+ typed_id: str,
+ labels: Iterable[str],
+) -> dict[str, Any]:
+ date = date_part(node)
+ body = generic_body(node_type.title(), node)
+ return {
+ "type": node_type,
+ id_field: typed_id,
+ "title": str(first(node, "title", "name", "summary", "description") or typed_id)[:120],
+ "status": str(first(node, "status") or "active").lower(),
+ "created": date,
+ "updated": date,
+ "sources": 1,
+ "neo4j_labels": sorted(labels),
+ "completion_body_hash": body_hash(body),
+ "tags": ["kublai", node_type, "historical"],
+ }
+
+
+def reflection_body(node: dict[str, Any]) -> str:
+ fields = [
+ ("Context", first(node, "context", "description", "summary")),
+ ("Expected Behavior", first(node, "expected_behavior")),
+ ("Actual Behavior", first(node, "actual_behavior")),
+ ("Root Cause", first(node, "root_cause")),
+ ("Lesson", first(node, "lesson", "content", "body")),
+ ]
+ return sections("Historical Reflection", fields, node)
+
+
+def generic_body(title: str, node: dict[str, Any]) -> str:
+ preferred = [
+ ("Summary", first(node, "summary", "description", "content", "body")),
+ ("Decision", first(node, "decision")),
+ ("Rationale", first(node, "rationale", "reason")),
+ ("Result", first(node, "result", "results", "outcome")),
+ ]
+ return sections(title, preferred, node)
+
+
+def sections(title: str, fields: list[tuple[str, Any]], node: dict[str, Any]) -> str:
+ chunks = [f"# {title}\n"]
+ for heading, value in fields:
+ if value in (None, "", [], {}):
+ continue
+ chunks.append(f"## {heading}\n\n{stringify(value)}\n")
+ chunks.append("## Neo4j Properties\n")
+ chunks.append("```json\n")
+ chunks.append(json.dumps({k: stringify(v) for k, v in sorted(node.items())}, indent=2, sort_keys=True))
+ chunks.append("\n```\n")
+ return "\n".join(chunks)
+
+
+def first(node: dict[str, Any], *keys: str) -> Any:
+ for key in keys:
+ value = node.get(key)
+ if value not in (None, "", [], {}):
+ return value
+ return None
+
+
+def stable_id(prefix: str, node: dict[str, Any]) -> str:
+ payload = json.dumps({k: stringify(v) for k, v in sorted(node.items())}, sort_keys=True)
+ import hashlib
+
+ return f"{prefix}-{hashlib.sha256(payload.encode('utf-8')).hexdigest()[:16]}"
+
+
+def date_part(node: dict[str, Any]) -> str:
+ value = first(node, "created_at", "created", "completed_at", "updated_at", "updated")
+ dt = parse_datetime(value)
+ return dt.date().isoformat()
+
+
+def timestamp_ms(value: Any) -> int:
+ return int(parse_datetime(value).timestamp() * 1000)
+
+
+def parse_datetime(value: Any) -> datetime:
+ if value is None:
+ return datetime.now(timezone.utc)
+ if isinstance(value, datetime):
+ return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
+ text = str(value)
+ if text.endswith("Z"):
+ text = text[:-1] + "+00:00"
+ try:
+ return datetime.fromisoformat(text)
+ except ValueError:
+ return datetime.now(timezone.utc)
+
+
+def safe_json(value: Any) -> dict[str, Any]:
+ if isinstance(value, dict):
+ return value
+ if isinstance(value, str):
+ try:
+ loaded = json.loads(value)
+ return loaded if isinstance(loaded, dict) else {"value": loaded}
+ except json.JSONDecodeError:
+ return {"value": value}
+ return {"value": stringify(value)}
+
+
+def stringify(value: Any) -> str:
+ if hasattr(value, "iso_format"):
+ return value.iso_format()
+ if hasattr(value, "isoformat"):
+ return value.isoformat()
+ if isinstance(value, (dict, list)):
+ return json.dumps(value, sort_keys=True)
+ return str(value)
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/reflection_memory.py b/tools/reflection_memory.py
index 216e1aab..8b0bd9f3 100644
--- a/tools/reflection_memory.py
+++ b/tools/reflection_memory.py
@@ -9,18 +9,36 @@
over time through self-analysis and pattern recognition.
"""
+from __future__ import annotations
+
import hashlib
+import json
import logging
+import os
+import sys
import uuid
from datetime import datetime, timezone
+from pathlib import Path
from typing import Dict, List, Optional, Any
from neo4j.exceptions import Neo4jError
+from kublai.brain_service_client import call as brain_service_call
+from kublai.knowledge import KnowledgeStore
+
# Configure logging
logger = logging.getLogger(__name__)
+def _body_hash_from_page(path: Path) -> str:
+ text = path.read_text(encoding="utf-8")
+ if text.startswith("---\n"):
+ end = text.find("\n---\n", 4)
+ if end >= 0:
+ text = text[end + 5 :]
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
class ReflectionNotFoundError(Exception):
"""Raised when a reflection ID is not found."""
pass
@@ -67,6 +85,8 @@ def __init__(
# Try to load sentence-transformers for embeddings
self._embedding_model = None
try:
+ if "PYTEST_CURRENT_TEST" in os.environ and "sentence_transformers" not in sys.modules:
+ raise ImportError("skip real sentence-transformers model during tests")
from sentence_transformers import SentenceTransformer
self._embedding_model = SentenceTransformer('all-MiniLM-L6-v2')
logger.info("Loaded sentence-transformers model for embeddings")
@@ -75,6 +95,80 @@ def __init__(
logger.info(f"AgentReflectionMemory initialized with embedding_dimension={embedding_dimension}")
+ def _brain_reads_enabled(self) -> bool:
+ return os.getenv("KUBLAI_KNOWLEDGE_READS", "").lower() in {"1", "true", "brain", "brain-service"}
+
+ def _dual_write_enabled(self) -> bool:
+ return os.getenv("KUBLAI_KNOWLEDGE_DUAL_WRITE", "").lower() in {"1", "true", "yes"}
+
+ def _brain_call(self, method: str, params: Dict[str, Any]) -> Any:
+ socket_path = os.getenv("BRAIN_SERVICE_SOCKET", "/tmp/brain-service.sock")
+ response = brain_service_call(socket_path, method, params)
+ if not response.get("ok"):
+ raise ReflectionError(response.get("message") or response.get("error") or "brain-service call failed")
+ return response.get("result")
+
+ def _node_to_reflection(self, node: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ if not node:
+ return None
+ frontmatter = node.get("frontmatter") or {}
+ reflection = dict(frontmatter)
+ reflection.setdefault("id", frontmatter.get("reflection_id") or node.get("typed_id"))
+ reflection.setdefault("agent", node.get("agent"))
+ reflection["body"] = node.get("body_text", "")
+ return reflection
+
+ def _dual_write_reflection(
+ self,
+ *,
+ reflection_id: str,
+ agent: str,
+ mistake_type: str,
+ context: str,
+ expected_behavior: str,
+ actual_behavior: str,
+ root_cause: str,
+ lesson: str,
+ ) -> None:
+ if not self._dual_write_enabled():
+ return
+ wiki_root = Path(os.getenv("BRAIN_WIKI_ROOT", str(Path.home() / "brain")))
+ body = (
+ f"# Reflection {reflection_id}\n\n"
+ f"## Context\n\n{context}\n\n"
+ f"## Expected Behavior\n\n{expected_behavior}\n\n"
+ f"## Actual Behavior\n\n{actual_behavior}\n\n"
+ f"## Root Cause\n\n{root_cause}\n\n"
+ f"## Lesson\n\n{lesson}\n"
+ )
+ store = KnowledgeStore(wiki_root)
+ path = store.record_reflection(
+ agent=agent,
+ reflection_id=reflection_id,
+ body=body,
+ tags=["kublai", "reflection", mistake_type],
+ )
+ self._append_dual_write_log(
+ {
+ "operation_id": str(uuid.uuid4()),
+ "kind": "reflection",
+ "idempotency_key": f"reflection:{reflection_id}",
+ "wiki_path": path.relative_to(wiki_root.resolve()).as_posix(),
+ "body_hash": _body_hash_from_page(path),
+ "neo4j_label": "Reflection",
+ "recorded_at": self._now().isoformat(),
+ }
+ )
+
+ def _append_dual_write_log(self, record: Dict[str, Any]) -> None:
+ log_path = os.getenv("BRAIN_DUAL_WRITE_LOG")
+ if not log_path:
+ return
+ target = Path(log_path).expanduser()
+ target.parent.mkdir(parents=True, exist_ok=True)
+ with target.open("a", encoding="utf-8") as handle:
+ handle.write(json.dumps(record, sort_keys=True) + "\n")
+
def _generate_id(self) -> str:
"""Generate a unique ID using the memory's method or fallback to uuid."""
if hasattr(self.memory, '_generate_id'):
@@ -213,8 +307,19 @@ def record_mistake(
)
record = result.single()
if record:
+ reflection_id = record["reflection_id"]
+ self._dual_write_reflection(
+ reflection_id=reflection_id,
+ agent=agent,
+ mistake_type=mistake_type,
+ context=context,
+ expected_behavior=expected_behavior,
+ actual_behavior=actual_behavior,
+ root_cause=root_cause,
+ lesson=lesson,
+ )
logger.info(f"Reflection recorded: {reflection_id} for agent {agent}")
- return record["reflection_id"]
+ return reflection_id
else:
raise ReflectionError("Reflection recording failed: no record returned")
except Neo4jError as e:
@@ -231,6 +336,14 @@ def get_reflection(self, reflection_id: str) -> Optional[Dict]:
Returns:
Reflection dict if found, None otherwise
"""
+ if self._brain_reads_enabled():
+ try:
+ return self._node_to_reflection(
+ self._brain_call("knowledge.get", {"node_type": "reflection", "typed_id": reflection_id})
+ )
+ except Exception as exc:
+ logger.warning(f"brain-service reflection read failed, falling back to Neo4j: {exc}")
+
cypher = """
MATCH (r:Reflection {id: $reflection_id})
RETURN r
@@ -270,6 +383,21 @@ def list_reflections(
Returns:
List of reflection dicts
"""
+ if self._brain_reads_enabled():
+ try:
+ nodes = self._brain_call(
+ "knowledge.list",
+ {
+ "node_type": "reflection",
+ "agent": agent,
+ "status": "active" if consolidated is False else None,
+ "limit": limit,
+ },
+ )
+ return [r for r in (self._node_to_reflection(node) for node in nodes) if r is not None]
+ except Exception as exc:
+ logger.warning(f"brain-service reflection list failed, falling back to Neo4j: {exc}")
+
conditions = []
params = {"limit": limit}
@@ -283,6 +411,13 @@ def list_reflections(
where_clause = "WHERE " + " AND ".join(conditions) if conditions else ""
+ # Skip reflections evaluated within the last 24 hours (prevents accumulation loop)
+ evaluated_filter = "(r.evaluated_at IS NULL OR r.evaluated_at < datetime() - duration('P1D'))"
+ if where_clause:
+ where_clause += f" AND {evaluated_filter}"
+ else:
+ where_clause = f"WHERE {evaluated_filter}"
+
cypher = f"""
MATCH (r:Reflection)
{where_clause}
@@ -327,6 +462,25 @@ def search_similar_reflections(
Returns:
List of reflection dicts with similarity scores
"""
+ if self._brain_reads_enabled():
+ try:
+ nodes = self._brain_call(
+ "knowledge.search",
+ {"query": query_text, "node_type": "reflection", "limit": limit},
+ )
+ reflections = []
+ for node in nodes:
+ reflection = self._node_to_reflection(node)
+ if reflection is None:
+ continue
+ if agent and reflection.get("agent") != agent:
+ continue
+ reflection["similarity"] = node.get("score")
+ reflections.append(reflection)
+ return reflections
+ except Exception as exc:
+ logger.warning(f"brain-service reflection search failed, falling back to Neo4j: {exc}")
+
query_embedding = self._generate_embedding(query_text)
# Build query with optional agent filter
@@ -377,8 +531,7 @@ def search_similar_reflections(
reflection.pop("embedding", None)
reflection["similarity"] = record["similarity"]
reflections.append(reflection)
- if reflections:
- return reflections
+ return reflections # Return even if empty — empty is a valid result
except Neo4jError:
pass # Vector index not available, fall back to manual