diff --git a/CHAGELOG.md b/CHAGELOG.md index b91feeed7..8ed59925b 100644 --- a/CHAGELOG.md +++ b/CHAGELOG.md @@ -15,6 +15,11 @@ Keep proposed and unmerged changes out of released sections. ## [Unreleased] +- Proposed [PR #1046](https://github.com/BitterSecurity/Decepticon/pull/1046): + add read-only runtime diagnostics — `decepticon doctor --deep` reports the + middleware slot matrix for every role (KG/HITL/model_fallback previously + failed silently), KG reachability, Skillogy health and MCP auth mode. + Addresses issue #1040. - Proposed (launcher scope recommendations): engagement creation now runs passive discovery — certificate-transparency logs (crt.sh) and passive DNS (HackerTarget, both keyless) — against the first hostname target and diff --git a/clients/launcher/cmd/doctor.go b/clients/launcher/cmd/doctor.go index b079727dd..f22d8ab10 100644 --- a/clients/launcher/cmd/doctor.go +++ b/clients/launcher/cmd/doctor.go @@ -44,15 +44,27 @@ Compose V2, configuration (.env and docker-compose.yml) and authentication, LangGraph readiness, the opscontrol Unix socket and daemon health, and optional blue-team monitoring endpoints. +With --deep, read-only runtime diagnostics also run inside the +langgraph container (python -m decepticon.diagnostics): KG store +reachability, Skillogy health, HITL/model-fallback slot resolution and +the MCP auth mode. + Nothing is modified. The command exits non-zero when a required check fails; optional checks are reported but do not affect the exit code.`, Example: ` decepticon doctor Run all diagnostics and print a report - decepticon doctor --json Print a machine-readable report for CI`, + decepticon doctor --json Print a machine-readable report for CI + decepticon doctor --deep Also run the in-container runtime diagnostics`, Args: cobra.NoArgs, } command.Flags().Bool("json", false, "Print a machine-readable report") + command.Flags().Bool("deep", false, "Also run read-only runtime diagnostics inside the langgraph container (requires the stack)") command.RunE = func(cmd *cobra.Command, _ []string) error { report := diagnose(cmd.Context()) + if deep, err := cmd.Flags().GetBool("deep"); err != nil { + return err + } else if deep { + addDeepRuntimeChecks(cmd.Context(), &report) + } jsonOutput, err := cmd.Flags().GetBool("json") if err != nil { return err diff --git a/clients/launcher/cmd/doctor_deep.go b/clients/launcher/cmd/doctor_deep.go new file mode 100644 index 000000000..67aef5762 --- /dev/null +++ b/clients/launcher/cmd/doctor_deep.go @@ -0,0 +1,128 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + "github.com/PurpleAILAB/Decepticon/clients/launcher/internal/compose" +) + +// deepRuntimeReport mirrors packages/decepticon/decepticon/diagnostics.py. +type deepRuntimeReport struct { + OK bool `json:"ok"` + Components map[string]struct { + OK bool `json:"ok"` + Detail string `json:"detail"` + } `json:"components"` + SlotMatrix map[string]struct { + Resolved []string `json:"resolved"` + Absent map[string]string `json:"absent"` + } `json:"slot_matrix"` +} + +// addDeepRuntimeChecks runs the read-only diagnostics module inside the +// langgraph container (python -m decepticon.diagnostics) and folds the +// result into the doctor report. Missing stack is not an error here: the +// base checks already report service status. +func addDeepRuntimeChecks(ctx context.Context, report *doctorReport) { + c := compose.New() + if _, err := os.Stat(c.ComposeFile); err != nil { + report.Checks = append(report.Checks, doctorCheck{ + Name: "runtime_deep", Required: false, OK: false, + Detail: "Compose file missing; runtime diagnostics skipped", + }) + return + } + probeCtx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + out, err := c.ExecCapture(probeCtx, "langgraph", "python", "-m", "decepticon.diagnostics") + if err != nil { + report.Checks = append(report.Checks, doctorCheck{ + Name: "runtime_deep", Required: false, OK: false, + Detail: fmt.Sprintf("runtime diagnostics failed: %v", err), + Recovery: "Start services with `decepticon start --no-update` and retry `decepticon doctor --deep`", + }) + return + } + + var parsed deepRuntimeReport + if err := json.Unmarshal(trimJSON(out), &parsed); err != nil { + report.Checks = append(report.Checks, doctorCheck{ + Name: "runtime_deep", Required: false, OK: false, + Detail: fmt.Sprintf("runtime diagnostics returned invalid JSON: %v", err), + }) + return + } + + for _, name := range []string{"kg", "skillogy", "hitl", "model_fallback", "mcp_auth"} { + component, exists := parsed.Components[name] + if !exists { + continue + } + report.Checks = append(report.Checks, doctorCheck{ + Name: fmt.Sprintf("runtime_%s", name), + Required: false, + OK: component.OK, + Detail: component.Detail, + Recovery: recoveryHintFor(name), + }) + } + + absentSummary := summarizeAbsentRoles(parsed.SlotMatrix) + report.Checks = append(report.Checks, doctorCheck{ + Name: "runtime_slot_matrix", Required: false, OK: true, + Detail: fmt.Sprintf("%d roles evaluated; conditional slots absent in: %s", + len(parsed.SlotMatrix), absentSummary), + }) +} + +func trimJSON(out []byte) []byte { + start := 0 + for start < len(out) && out[start] != '{' && out[start] != '[' { + start++ + } + return out[start:] +} + +func recoveryHintFor(name string) string { + switch name { + case "kg": + return "Set DECEPTICON_NEO4J_* in .env and verify the stack with `decepticon start --no-update`; the KG slot stays absent until reachable" + case "skillogy": + return "Check the skillogy service (`decepticon logs skillogy`); with DECEPTICON_USE_SKILLOGY off the filesystem fallback is active" + case "hitl": + return "Set DECEPTICON_HITL__ENABLED=1 to enable human-in-the-loop approvals" + case "model_fallback": + return "Set DECEPTICON_MODEL__FALLBACK chains to enable model fallback" + case "mcp_auth": + return "Set DECEPTICON_MCP_TOKEN to require shared-secret auth on the MCP bridge" + default: + return "" + } +} + +func summarizeAbsentRoles(matrix map[string]struct { + Resolved []string `json:"resolved"` + Absent map[string]string `json:"absent"` +}) string { + counts := map[string]int{} + for _, role := range matrix { + for slot := range role.Absent { + counts[slot]++ + } + } + summary := "" + for slot, count := range counts { + if summary != "" { + summary += ", " + } + summary += fmt.Sprintf("%s (%d roles)", slot, count) + } + if summary == "" { + return "none" + } + return summary +} diff --git a/clients/launcher/internal/compose/compose.go b/clients/launcher/internal/compose/compose.go index 2c08e303d..7d198bb69 100644 --- a/clients/launcher/internal/compose/compose.go +++ b/clients/launcher/internal/compose/compose.go @@ -1,6 +1,7 @@ package compose import ( + "context" "fmt" "io" "os" @@ -379,3 +380,18 @@ func (c *Compose) RemoveOrphanedCLI() { _ = rm.Run() } } + +// ExecCapture runs a command inside a running service and returns its +// stdout, for read-only diagnostics (`doctor --deep`). Unlike Exec it +// captures instead of streaming. +func (c *Compose) ExecCapture(ctx context.Context, service string, command ...string) ([]byte, error) { + cmdArgs := append(c.baseArgs(), "exec", "-T", service) + cmdArgs = append(cmdArgs, command...) + cmd := exec.CommandContext(ctx, c.Runtime.Bin, cmdArgs...) + cmd.Env = c.composeEnv() + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("%s compose exec %s: %w", c.Runtime.Name, service, err) + } + return out, nil +} diff --git a/packages/decepticon/decepticon/diagnostics.py b/packages/decepticon/decepticon/diagnostics.py new file mode 100644 index 000000000..7d4ba84f7 --- /dev/null +++ b/packages/decepticon/decepticon/diagnostics.py @@ -0,0 +1,212 @@ +"""Deep runtime diagnostics for `decepticon doctor --deep`. + +Reports, read-only, the state that the host-side doctor cannot see: +which middleware slots resolve for the configured roles (the conditional +KG / HITL / model-fallback slots fail *silently* at assembly time — +``agents/middleware_slots.py`` returns ``None`` plus one log line), the +KG store revision, Skillogy health, and the MCP auth mode. + +Run inside the langgraph container: + + python -m decepticon.diagnostics + +Exit code 0 when every required component reports healthy, 1 otherwise. +The output is a single JSON document consumed by the launcher's +``decepticon doctor --deep`` (compose exec). +""" + +from __future__ import annotations + +import json +import os +from typing import Any + +from decepticon_core.contracts.slots import SLOTS_PER_ROLE, MiddlewareSlot +from decepticon_core.utils.logging import get_logger + +log = get_logger("diagnostics") + + +def _truthy(value: str | None) -> bool: + # An unset variable is not "enabled": str(None) would read "none" and + # silently pass the falsy set below, reporting absent slots as OK. + if value is None: + return False + return value.strip().lower() not in ("", "0", "false", "no", "off") + + +def _hitl_status() -> dict[str, Any]: + enabled = _truthy(os.environ.get("DECEPTICON_HITL__ENABLED")) + return { + "ok": enabled, + "detail": "HITL approval middleware enabled" + if enabled + else "HITL disabled (DECEPTICON_HITL__ENABLED not set) — slot will be absent", + } + + +def _model_fallback_status() -> dict[str, Any]: + """Resolve fallback chains through the real config source. + + ``ModelFallbackMiddleware`` is fed from ``LLMFactory.get_fallback_models`` + — the credential-chain mapping resolved at assembly time. No + ``DECEPTICON_MODEL__FALLBACK`` env var exists; checking one always + reported "no chains" regardless of the operator's actual credentials. + """ + try: + from decepticon.llm.factory import LLMFactory + + factory = LLMFactory() + except Exception as exc: # noqa: BLE001 — diagnostics must not crash + return {"ok": False, "detail": f"model router unavailable: {exc}"} + + roles: dict[str, Any] = {} + any_chain = False + for role in SLOTS_PER_ROLE: + try: + # Same guard condition the assembly path uses: an empty chain + # means the conditional slot returns None and stays absent. + chain = factory.get_fallback_models(role) + except Exception as exc: # noqa: BLE001 + roles[role] = f"unresolved: {exc}" + continue + roles[role] = len(chain) + if chain: + any_chain = True + return { + "ok": any_chain, + "detail": "fallback chains configured for at least one role" + if any_chain + else "no fallback chains in the credential mapping — slot will be absent", + "roles": roles, + } + + +def _kg_status() -> dict[str, Any]: + """KG store resolution + reachability. Lazy import: neo4j is an optional extra.""" + from decepticon.middleware.kg_internal.store import ( + KGStoreConfig, + KGStoreConfigError, + KGStoreUnavailableError, + ) + + try: + config = KGStoreConfig.from_env() + except (KGStoreConfigError, KGStoreUnavailableError) as exc: + return { + "ok": False, + "detail": f"KG store not configured — slot will be absent: {exc}", + } + except Exception as exc: # noqa: BLE001 — diagnostics must not crash + return {"ok": False, "detail": f"KG config error: {exc}"} + + try: + from decepticon.middleware.kg_internal.store import KGStore + + store = KGStore(config) + # Read-only reachability probe scoped to the diagnostics label: a + # per-engagement revision query would create state for a fake + # label, so we verify driver connectivity with a scoped count. + rows = store.execute_read( + "MATCH (n {engagement: $engagement}) RETURN count(n) AS nodes", + engagement="diagnostics", + ) + return { + "ok": True, + "detail": f"KG store reachable at {config.uri} (database {config.database})", + "diagnostic_scope_nodes": rows[0]["nodes"] if rows else 0, + } + except Exception as exc: # noqa: BLE001 + return { + "ok": False, + "detail": f"KG store configured at {config.uri} but unreachable: {exc}", + } + + +def _skillogy_status() -> dict[str, Any]: + if not _truthy(os.environ.get("DECEPTICON_USE_SKILLOGY")): + return { + "ok": True, + "detail": "Skillogy disabled — skills middleware filesystem fallback active", + } + import urllib.request + + url = os.environ.get("DECEPTICON_SKILLOGY_URL", "http://skillogy:9100").rstrip("/") + try: + with urllib.request.urlopen(f"{url}/health", timeout=5) as response: + ok = 200 <= response.status < 300 + return { + "ok": ok, + "detail": f"Skillogy at {url}: HTTP {response.status}", + } + except Exception as exc: # noqa: BLE001 + return {"ok": False, "detail": f"Skillogy at {url} unreachable: {exc}"} + + +def _mcp_status() -> dict[str, Any]: + token = os.environ.get("DECEPTICON_MCP_TOKEN") + mode = "shared-secret" if token else "unauthenticated (loopback default)" + return {"ok": True, "detail": f"MCP bridge auth mode: {mode}", "mode": mode} + + +def slot_matrix() -> dict[str, Any]: + """For every configured role, report which of its slots resolve. + + Conditional slots (KG / HITL / model_fallback) are evaluated against the + same guard conditions the factories use at assembly time; the remaining + slots are unconditional and always resolve. + """ + hitl = _hitl_status() + fallback = _model_fallback_status() + kg = _kg_status() + conditional = { + MiddlewareSlot.HITL_APPROVAL: hitl, + MiddlewareSlot.MODEL_FALLBACK: fallback, + MiddlewareSlot.KG: kg, + } + matrix: dict[str, Any] = {} + for role, slots in SLOTS_PER_ROLE.items(): + resolved: list[str] = [] + absent: dict[str, str] = {} + for slot in slots: + report = conditional.get(slot) + if report is None: + resolved.append(str(slot)) + continue + if report["ok"]: + resolved.append(str(slot)) + else: + absent[str(slot)] = report["detail"] + matrix[role] = {"resolved": resolved, "absent": absent} + return matrix + + +def deep_report() -> dict[str, Any]: + hitl = _hitl_status() + fallback = _model_fallback_status() + kg = _kg_status() + skillogy = _skillogy_status() + mcp = _mcp_status() + components = { + "hitl": hitl, + "model_fallback": fallback, + "kg": kg, + "skillogy": skillogy, + "mcp_auth": mcp, + } + required_ok = all(components[name]["ok"] for name in ("kg", "skillogy") if name in components) + return { + "ok": required_ok, + "components": components, + "slot_matrix": slot_matrix(), + } + + +def main() -> None: + report = deep_report() + print(json.dumps(report, indent=2, default=str, ensure_ascii=False)) + raise SystemExit(0 if report["ok"] else 1) + + +if __name__ == "__main__": + main()