feat(doctor): add --deep runtime diagnostics and middleware slot matrix

clients/launcher/cmd/doctor_deep.go

@@ -0,0 +1,128 @@
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/PurpleAILAB/Decepticon/clients/launcher/internal/compose"
+)
+
+// deepRuntimeReport mirrors packages/decepticon/decepticon/diagnostics.py.
+type deepRuntimeReport struct {
+	OK     bool `json:"ok"`
+	Components map[string]struct {
+		OK     bool `json:"ok"`
+		Detail string `json:"detail"`
+	} `json:"components"`
+	SlotMatrix map[string]struct {
+		Resolved []string          `json:"resolved"`
+		Absent   map[string]string `json:"absent"`
+	} `json:"slot_matrix"`
+}
+
+// addDeepRuntimeChecks runs the read-only diagnostics module inside the
+// langgraph container (python -m decepticon.diagnostics) and folds the
+// result into the doctor report. Missing stack is not an error here: the
+// base checks already report service status.
+func addDeepRuntimeChecks(ctx context.Context, report *doctorReport) {
+	c := compose.New()
+	if _, err := os.Stat(c.ComposeFile); err != nil {
+		report.Checks = append(report.Checks, doctorCheck{
+			Name: "runtime_deep", Required: false, OK: false,
+			Detail: "Compose file missing; runtime diagnostics skipped",
+		})
+		return
+	}
+	probeCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
+	defer cancel()
+	out, err := c.ExecCapture(probeCtx, "langgraph", "python", "-m", "decepticon.diagnostics")
+	if err != nil {
+		report.Checks = append(report.Checks, doctorCheck{
+			Name: "runtime_deep", Required: false, OK: false,
+			Detail: fmt.Sprintf("runtime diagnostics failed: %v", err),
+			Recovery: "Start services with `decepticon start --no-update` and retry `decepticon doctor --deep`",
+		})
+		return
+	}
+
+	var parsed deepRuntimeReport
+	if err := json.Unmarshal(trimJSON(out), &parsed); err != nil {
+		report.Checks = append(report.Checks, doctorCheck{
+			Name: "runtime_deep", Required: false, OK: false,
+			Detail: fmt.Sprintf("runtime diagnostics returned invalid JSON: %v", err),
+		})
+		return
+	}
+
+	for _, name := range []string{"kg", "skillogy", "hitl", "model_fallback", "mcp_auth"} {
+		component, exists := parsed.Components[name]
+		if !exists {
+			continue
+		}
+		report.Checks = append(report.Checks, doctorCheck{
+			Name:     fmt.Sprintf("runtime_%s", name),
+			Required: false,
+			OK:       component.OK,
+			Detail:   component.Detail,
+			Recovery: recoveryHintFor(name),
+		})
+	}
+
+	absentSummary := summarizeAbsentRoles(parsed.SlotMatrix)
+	report.Checks = append(report.Checks, doctorCheck{
+		Name: "runtime_slot_matrix", Required: false, OK: true,
+		Detail: fmt.Sprintf("%d roles evaluated; conditional slots absent in: %s",
+			len(parsed.SlotMatrix), absentSummary),
+	})
+}
+
+func trimJSON(out []byte) []byte {
+	start := 0
+	for start < len(out) && out[start] != '{' && out[start] != '[' {
+		start++
+	}
+	return out[start:]
+}
+
+func recoveryHintFor(name string) string {
+	switch name {
+	case "kg":
+		return "Set DECEPTICON_NEO4J_* in .env and verify the stack with `decepticon start --no-update`; the KG slot stays absent until reachable"
+	case "skillogy":
+		return "Check the skillogy service (`decepticon logs skillogy`); with DECEPTICON_USE_SKILLOGY off the filesystem fallback is active"
+	case "hitl":
+		return "Set DECEPTICON_HITL__ENABLED=1 to enable human-in-the-loop approvals"
+	case "model_fallback":
+		return "Set DECEPTICON_MODEL_<ROLE>_FALLBACK chains to enable model fallback"
+	case "mcp_auth":
+		return "Set DECEPTICON_MCP_TOKEN to require shared-secret auth on the MCP bridge"
+	default:
+		return ""
+	}
+}
+
+func summarizeAbsentRoles(matrix map[string]struct {
+	Resolved []string          `json:"resolved"`
+	Absent   map[string]string `json:"absent"`
+}) string {
+	counts := map[string]int{}
+	for _, role := range matrix {
+		for slot := range role.Absent {
+			counts[slot]++
+		}
+	}
+	summary := ""
+	for slot, count := range counts {
+		if summary != "" {
+			summary += ", "
+		}
+		summary += fmt.Sprintf("%s (%d roles)", slot, count)
+	}
+	if summary == "" {
+		return "none"
+	}
+	return summary
+}

clients/launcher/internal/compose/compose.go

@@ -1,6 +1,7 @@
 package compose
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"os"
@@ -379,3 +380,18 @@ func (c *Compose) RemoveOrphanedCLI() {
 		_ = rm.Run()
 	}
 }
+
+// ExecCapture runs a command inside a running service and returns its
+// stdout, for read-only diagnostics (`doctor --deep`). Unlike Exec it
+// captures instead of streaming.
+func (c *Compose) ExecCapture(ctx context.Context, service string, command ...string) ([]byte, error) {
+	cmdArgs := append(c.baseArgs(), "exec", "-T", service)
+	cmdArgs = append(cmdArgs, command...)
+	cmd := exec.CommandContext(ctx, c.Runtime.Bin, cmdArgs...)
+	cmd.Env = c.composeEnv()
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("%s compose exec %s: %w", c.Runtime.Name, service, err)
+	}
+	return out, nil
+}

packages/decepticon/decepticon/diagnostics.py

@@ -0,0 +1,191 @@
+"""Deep runtime diagnostics for `decepticon doctor --deep`.
+
+Reports, read-only, the state that the host-side doctor cannot see:
+which middleware slots resolve for the configured roles (the conditional
+KG / HITL / model-fallback slots fail *silently* at assembly time —
+``agents/middleware_slots.py`` returns ``None`` plus one log line), the
+KG store revision, Skillogy health, and the MCP auth mode.
+
+Run inside the langgraph container:
+
+    python -m decepticon.diagnostics
+
+Exit code 0 when every required component reports healthy, 1 otherwise.
+The output is a single JSON document consumed by the launcher's
+``decepticon doctor --deep`` (compose exec).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from typing import Any
+
+from decepticon_core.contracts.slots import SLOTS_PER_ROLE, MiddlewareSlot
+from decepticon_core.utils.logging import get_logger
+
+log = get_logger("diagnostics")
+
+
+def _truthy(value: str | None) -> bool:
+    return str(value).strip().lower() not in ("", "0", "false", "no", "off")
+
+
+def _hitl_status() -> dict[str, Any]:
+    enabled = _truthy(os.environ.get("DECEPTICON_HITL__ENABLED"))
+    return {
+        "ok": enabled,
+        "detail": "HITL approval middleware enabled"
+        if enabled
+        else "HITL disabled (DECEPTICON_HITL__ENABLED not set) — slot will be absent",
+    }
+
+
+def _model_fallback_status() -> dict[str, Any]:
+    roles: dict[str, Any] = {}
+    any_chain = False
+    for role in SLOTS_PER_ROLE:
+        chain = os.environ.get(f"DECEPTICON_MODEL_{role.upper()}_FALLBACK", "").strip()
+        roles[role] = chain
+        if chain:
+            any_chain = True
+    return {
+        "ok": any_chain,
+        "detail": "fallback chains configured for at least one role"
+        if any_chain
+        else "no DECEPTICON_MODEL_<ROLE>_FALLBACK chains — slot will be absent",
+        "roles": roles,
+    }
+
+
+def _kg_status() -> dict[str, Any]:
+    """KG store resolution + reachability. Lazy import: neo4j is an optional extra."""
+    from decepticon.middleware.kg_internal.store import (
+        KGStoreConfig,
+        KGStoreUnavailableError,
+        KGStoreConfigError,
+    )
+
+    try:
+        config = KGStoreConfig.from_env()
+    except (KGStoreConfigError, KGStoreUnavailableError) as exc:
+        return {
+            "ok": False,
+            "detail": f"KG store not configured — slot will be absent: {exc}",
+        }
+    except Exception as exc:  # noqa: BLE001 — diagnostics must not crash
+        return {"ok": False, "detail": f"KG config error: {exc}"}
+
+    try:
+        from decepticon.middleware.kg_internal.store import KGStore
+
+        store = KGStore(config)
+        # Read-only reachability probe scoped to the diagnostics label: a
+        # per-engagement revision query would create state for a fake
+        # label, so we verify driver connectivity with a scoped count.
+        rows = store.execute_read(
+            "MATCH (n {engagement: $engagement}) RETURN count(n) AS nodes",
+            engagement="diagnostics",
+        )
+        return {
+            "ok": True,
+            "detail": f"KG store reachable at {config.uri} (database {config.database})",
+            "diagnostic_scope_nodes": rows[0]["nodes"] if rows else 0,
+        }
+    except Exception as exc:  # noqa: BLE001
+        return {
+            "ok": False,
+            "detail": f"KG store configured at {config.uri} but unreachable: {exc}",
+        }
+
+
+def _skillogy_status() -> dict[str, Any]:
+    if not _truthy(os.environ.get("DECEPTICON_USE_SKILLOGY")):
+        return {
+            "ok": True,
+            "detail": "Skillogy disabled — skills middleware filesystem fallback active",
+        }
+    import urllib.request
+
+    url = os.environ.get("DECEPTICON_SKILLOGY_URL", "http://skillogy:9100").rstrip("/")
+    try:
+        with urllib.request.urlopen(f"{url}/health", timeout=5) as response:
+            ok = 200 <= response.status < 300
+            return {
+                "ok": ok,
+                "detail": f"Skillogy at {url}: HTTP {response.status}",
+            }
+    except Exception as exc:  # noqa: BLE001
+        return {"ok": False, "detail": f"Skillogy at {url} unreachable: {exc}"}
+
+
+def _mcp_status() -> dict[str, Any]:
+    token = os.environ.get("DECEPTICON_MCP_TOKEN")
+    mode = "shared-secret" if token else "unauthenticated (loopback default)"
+    return {"ok": True, "detail": f"MCP bridge auth mode: {mode}", "mode": mode}
+
+
+def slot_matrix() -> dict[str, Any]:
+    """For every configured role, report which of its slots resolve.
+
+    Conditional slots (KG / HITL / model_fallback) are evaluated against the
+    same guard conditions the factories use at assembly time; the remaining
+    slots are unconditional and always resolve.
+    """
+    hitl = _hitl_status()
+    fallback = _model_fallback_status()
+    kg = _kg_status()
+    conditional = {
+        MiddlewareSlot.HITL_APPROVAL: hitl,
+        MiddlewareSlot.MODEL_FALLBACK: fallback,
+        MiddlewareSlot.KG: kg,
+    }
+    matrix: dict[str, Any] = {}
+    for role, slots in SLOTS_PER_ROLE.items():
+        resolved: list[str] = []
+        absent: dict[str, str] = {}
+        for slot in slots:
+            report = conditional.get(slot)
+            if report is None:
+                resolved.append(str(slot))
+                continue
+            if report["ok"]:
+                resolved.append(str(slot))
+            else:
+                absent[str(slot)] = report["detail"]
+        matrix[role] = {"resolved": resolved, "absent": absent}
+    return matrix
+
+
+def deep_report() -> dict[str, Any]:
+    hitl = _hitl_status()
+    fallback = _model_fallback_status()
+    kg = _kg_status()
+    skillogy = _skillogy_status()
+    mcp = _mcp_status()
+    components = {
+        "hitl": hitl,
+        "model_fallback": fallback,
+        "kg": kg,
+        "skillogy": skillogy,
+        "mcp_auth": mcp,
+    }
+    required_ok = all(
+        components[name]["ok"] for name in ("kg", "skillogy") if name in components
+    )
+    return {
+        "ok": required_ok,
+        "components": components,
+        "slot_matrix": slot_matrix(),
+    }
+
+
+def main() -> None:
+    report = deep_report()
+    print(json.dumps(report, indent=2, default=str, ensure_ascii=False))
+    raise SystemExit(0 if report["ok"] else 1)
+
+
+if __name__ == "__main__":
+    main()