From 0f82484cae2c852c04393fe16a5cfb2b8faac162 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 15 Sep 2026 15:50:41 -0700 Subject: [PATCH 1/3] feat: add bounded runtime health checks Signed-off-by: Yuchen Zhang --- README.md | 4 +- adapter-contract/python/README.md | 4 + adapter-contract/python/pypi.md | 4 + .../nemo_fabric_adapter_contract/models.py | 73 ++ adapter-contract/typescript/README.md | 6 + .../schemas/adapter-descriptor.schema.json | 6 + .../adapter-health-request.schema.json | 25 + .../schemas/adapter-health-result.schema.json | 151 +++ .../typescript/scripts/check-package.mjs | 6 + .../typescript/scripts/generate.mjs | 2 + .../src/generated/adapter-descriptor.ts | 4 + .../src/generated/adapter-health-request.ts | 19 + .../src/generated/adapter-health-result.ts | 67 ++ adapter-contract/typescript/src/index.ts | 2 + .../typescript/test/execution.test.ts | 21 + adapters/python/common/README.md | 15 + .../nemo_fabric_adapters/common/lifecycle.py | 262 +++++- .../mini-swe-agent.fabric-adapter.json | 1 + .../nooa/nooa-bench.fabric-adapter.json | 1 + adapters/python/nooa/nooa.fabric-adapter.json | 1 + .../remote-agent.fabric-adapter.json | 1 + adapters/typescript/common/README.md | 9 + adapters/typescript/common/src/lifecycle.ts | 289 +++++- .../typescript/common/test/lifecycle.test.mjs | 101 ++ adapters/typescript/pi/pi.fabric-adapter.json | 1 + crates/fabric-core/src/config.rs | 4 + crates/fabric-core/src/lib.rs | 17 +- crates/fabric-core/src/runtime.rs | 874 +++++++++++++++++- crates/fabric-core/src/schema.rs | 28 +- crates/fabric-python/src/lib.rs | 23 + .../tutorials/adapter-descriptor.md | 3 + .../adapter-contract/tutorials/conformance.md | 1 + docs/adapter-contract/tutorials/execution.md | 40 + .../api/python-library-reference/index.md | 2 + .../nemo_fabric.runtime.md | 33 + .../nemo_fabric.types.md | 180 ++++ .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 2 +- .../config/struct-runtimecapabilities.mdx | 6 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../nemo-fabric-core/error/index.mdx | 2 +- .../nemo-fabric-core/fn-version.mdx | 2 +- .../nemo-fabric-core/index.mdx | 10 + .../runtime/enum-errorstage.mdx | 2 +- .../runtime/enum-healthcheckstatus.mdx | 129 +++ .../enum-openaichatcompletionchunkobject.mdx | 2 +- .../runtime/enum-openaistreamhost.mdx | 2 +- .../runtime/enum-openaistreamprofile.mdx | 2 +- .../enum-openaistreamprotocolversion.mdx | 2 +- .../runtime/enum-openaistreamrecord.mdx | 2 +- .../runtime/enum-runstatus.mdx | 2 +- .../runtime/enum-runtimeactivity.mdx | 129 +++ .../runtime/enum-runtimeliveness.mdx | 129 +++ .../runtime/enum-runtimereadiness.mdx | 122 +++ .../runtime/fn-check-runtime-health.mdx | 14 + .../runtime/fn-invoke-openai-stream.mdx | 2 +- .../runtime/fn-invoke-runtime.mdx | 2 +- .../runtime/fn-prepare-environment.mdx | 2 +- .../nemo-fabric-core/runtime/fn-run-plan.mdx | 2 +- .../runtime/fn-start-runtime.mdx | 2 +- .../runtime/fn-stop-runtime.mdx | 2 +- .../nemo-fabric-core/runtime/index.mdx | 12 +- .../runtime/struct-adapterhealthrequest.mdx | 98 ++ .../runtime/struct-adapterhealthresult.mdx | 106 +++ .../runtime/struct-adapterinvocation.mdx | 2 +- .../runtime/struct-adapterreadiness.mdx | 98 ++ .../runtime/struct-artifactmanifest.mdx | 2 +- .../runtime/struct-artifactref.mdx | 2 +- .../runtime/struct-environmenthandle.mdx | 2 +- .../runtime/struct-errorinfo.mdx | 2 +- .../runtime/struct-fabricevent.mdx | 2 +- .../runtime/struct-healthcheck.mdx | 118 +++ .../runtime/struct-invocationhandle.mdx | 2 +- .../struct-openaichatcompletionchunk.mdx | 2 +- ...struct-openaichatcompletionchunkchoice.mdx | 2 +- .../struct-openaichatcompletionchunkdelta.mdx | 2 +- .../runtime/struct-openaistreaminvocation.mdx | 2 +- .../runtime/struct-openaistreamsink.mdx | 2 +- .../runtime/struct-openaistreamtransport.mdx | 2 +- .../runtime/struct-runrequest.mdx | 2 +- .../runtime/struct-runresult.mdx | 2 +- .../runtime/struct-runtimecontext.mdx | 2 +- .../runtime/struct-runtimehandle.mdx | 2 +- .../runtime/struct-runtimehealth.mdx | 122 +++ .../struct-runtimetelemetrycontext.mdx | 2 +- .../runtime/struct-runusage.mdx | 2 +- .../runtime/struct-telemetryref.mdx | 2 +- .../schema/enum-schemaname.mdx | 23 +- .../nemo-fabric-core/schema/index.mdx | 2 +- docs/sdk/python.mdx | 31 + schemas/SCHEMA.md | 6 + .../adapter-descriptor.schema.json | 6 + .../adapter-health-request.schema.json | 25 + .../adapter-health-result.schema.json | 151 +++ schemas/sdk/run-plan.schema.json | 6 + schemas/sdk/runtime-health.schema.json | 215 +++++ .../src/nemo_fabric/__init__.py | 4 + .../src/nemo_fabric/_native.pyi | 5 + .../src/nemo_fabric/runtime.py | 60 +- .../src/nemo_fabric/types.py | 123 +++ skills/nemo-fabric-build-adapter/SKILL.md | 9 + skills/nemo-fabric-integrate/SKILL.md | 4 + .../references/sdk-api-inventory.md | 1 + tests/adapter_contract/test_health.py | 43 + tests/adapters/test_mini_swe_agent.py | 1 + tests/adapters/test_nooa_adapter.py | 1 + tests/adapters/test_pi_adapter.py | 1 + tests/python/test_adapter_lifecycle_health.py | 90 ++ tests/python/test_runtime.py | 85 ++ tests/python/test_sdk_contract.py | 37 + 112 files changed, 4292 insertions(+), 90 deletions(-) create mode 100644 adapter-contract/typescript/schemas/adapter-health-request.schema.json create mode 100644 adapter-contract/typescript/schemas/adapter-health-result.schema.json create mode 100644 adapter-contract/typescript/src/generated/adapter-health-request.ts create mode 100644 adapter-contract/typescript/src/generated/adapter-health-result.ts create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-healthcheckstatus.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimeactivity.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimeliveness.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimereadiness.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-check-runtime-health.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterhealthrequest.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterhealthresult.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterreadiness.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-healthcheck.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehealth.mdx create mode 100644 schemas/adapter-contract/adapter-health-request.schema.json create mode 100644 schemas/adapter-contract/adapter-health-result.schema.json create mode 100644 schemas/sdk/runtime-health.schema.json create mode 100644 tests/adapter_contract/test_health.py create mode 100644 tests/python/test_adapter_lifecycle_health.py diff --git a/README.md b/README.md index 8b764ff4c..9863fa4b9 100644 --- a/README.md +++ b/README.md @@ -244,8 +244,8 @@ Use the following resources to learn about NeMo Fabric: - [Example Notebooks](examples/notebooks/README.md) provide a guided tour of the Python SDK. - The [Python SDK guide](docs/sdk/python.mdx) covers typed configuration, - planning, diagnostics, requests, multi-turn runtimes, streaming, parallelism, - results, and errors. + planning, diagnostics, requests, multi-turn runtimes, bounded runtime health + checks, streaming, parallelism, results, and errors. - The [Experimentation CLI guide](docs/experimentation/cli.mdx) covers presets, maintained examples, and editable application scaffolds. - The [getting started overview](docs/about-nemo-fabric/overview.mdx) explains diff --git a/adapter-contract/python/README.md b/adapter-contract/python/README.md index 9bb29c7f1..21f2f974f 100644 --- a/adapter-contract/python/README.md +++ b/adapter-contract/python/README.md @@ -28,6 +28,10 @@ An adapter descriptor opts into the southbound configuration with `config.input=agent_config`. Python adapters using the optional common lifecycle host pass `AgentConfig.from_mapping` as the `config_loader`. +The models module also exports `AdapterHealthRequest`, `AdapterHealthResult`, +`AdapterReadiness`, and `HealthCheck` for optional bounded health hooks used by +the maintained local lifecycle hosts. + ## Install Install the package directly when developing a Python adapter: diff --git a/adapter-contract/python/pypi.md b/adapter-contract/python/pypi.md index c69f1b28d..6d6626ca0 100644 --- a/adapter-contract/python/pypi.md +++ b/adapter-contract/python/pypi.md @@ -34,6 +34,10 @@ NeMo Fabric delivers `AgentConfig` as the southbound configuration. Python adapters using the optional common lifecycle host pass `AgentConfig.from_mapping` as the `config_loader`. +The models module also exports `AdapterHealthRequest`, `AdapterHealthResult`, +`AdapterReadiness`, and `HealthCheck` for optional bounded health hooks used by +the maintained local lifecycle hosts. + ## Install Install the package directly when developing a Python adapter: diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py index 0885ee216..24b647e2b 100644 --- a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py @@ -534,6 +534,79 @@ def _validate(self) -> None: raise ContractValidationError("succeeded result must not include an error") +class RuntimeReadiness(StrEnum): + """Whether an adapter knows its runtime can currently accept work.""" + + READY = "ready" + NOT_READY = "not_ready" + UNKNOWN = "unknown" + + +class HealthCheckStatus(StrEnum): + """Outcome of one adapter health check.""" + + OK = "ok" + FAILED = "failed" + UNKNOWN = "unknown" + UNSUPPORTED = "unsupported" + + +@dataclass(slots=True, kw_only=True) +class HealthCheck(ContractModel): + """One timestamped adapter or dependency health observation.""" + + name: str + status: HealthCheckStatus + reason_code: str + observed_at_millis: int + age_millis: int + message: str | None = _optional() + metadata: dict[str, JsonValue] = _json_dict() + + def _validate(self) -> None: + _nonblank(self.name, "name") + _nonblank(self.reason_code, "reason_code") + _bounded_int(self.observed_at_millis, "observed_at_millis", (1 << 128) - 1) + _bounded_int(self.age_millis, "age_millis", (1 << 64) - 1) + if self.message is not None: + _nonblank(self.message, "message") + + +@dataclass(slots=True, kw_only=True) +class AdapterHealthRequest(ContractModel): + """Budget passed to an optional adapter health hook.""" + + runtime_id: str + timeout_millis: int + + def _validate(self) -> None: + _nonblank(self.runtime_id, "runtime_id") + _bounded_int(self.timeout_millis, "timeout_millis", (1 << 64) - 1) + if self.timeout_millis == 0: + raise ContractValidationError( + "must be greater than zero", path=("timeout_millis",) + ) + + +@dataclass(slots=True, kw_only=True) +class AdapterReadiness(ContractModel): + """Adapter-owned readiness observation.""" + + state: RuntimeReadiness + reason_code: str + + def _validate(self) -> None: + _nonblank(self.reason_code, "reason_code") + + +@dataclass(slots=True, kw_only=True) +class AdapterHealthResult(ContractModel): + """Optional adapter-specific contribution to runtime health.""" + + readiness: AdapterReadiness | None = _optional() + checks: list[HealthCheck] = _empty_list() + + class ControlLocation(StrEnum): """Where Fabric control code runs relative to the task environment.""" diff --git a/adapter-contract/typescript/README.md b/adapter-contract/typescript/README.md index d80652604..b19beb4d2 100644 --- a/adapter-contract/typescript/README.md +++ b/adapter-contract/typescript/README.md @@ -63,6 +63,8 @@ adapter-contract package: ```typescript import type { + AdapterHealthRequest, + AdapterHealthResult, AgentRunRequest, AgentRunResult, } from "nemo-fabric-adapter-contract"; @@ -74,6 +76,10 @@ Fabric normalizes the result. Token counts originate from JSON Schema `uint64` values but are represented as JavaScript `number`; values greater than `Number.MAX_SAFE_INTEGER` cannot be represented exactly. +Use `AdapterHealthRequest` and `AdapterHealthResult` only for an optional, +bounded health method served by a compatible local lifecycle host. Health +checks must not invoke the agent or mutate runtime state. + ## JSON Schemas The package bundles byte-identical copies of the canonical schemas. Consumers diff --git a/adapter-contract/typescript/schemas/adapter-descriptor.schema.json b/adapter-contract/typescript/schemas/adapter-descriptor.schema.json index 637dfbf2a..4dda2e2e7 100644 --- a/adapter-contract/typescript/schemas/adapter-descriptor.schema.json +++ b/adapter-contract/typescript/schemas/adapter-descriptor.schema.json @@ -254,6 +254,11 @@ "description": "Whether an in-flight invocation can be cancelled.", "type": "boolean" }, + "health": { + "default": false, + "description": "Whether the selected runtime exposes bounded health observations.", + "type": "boolean" + }, "metadata": { "additionalProperties": true, "description": "Additional adapter-specific capability metadata.", @@ -296,6 +301,7 @@ "$ref": "#/$defs/RuntimeCapabilities", "default": { "cancellation": false, + "health": false, "service": false, "streaming": false, "updates": false diff --git a/adapter-contract/typescript/schemas/adapter-health-request.schema.json b/adapter-contract/typescript/schemas/adapter-health-request.schema.json new file mode 100644 index 000000000..db721f294 --- /dev/null +++ b/adapter-contract/typescript/schemas/adapter-health-request.schema.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "Request passed to an optional adapter health hook.", + "properties": { + "runtime_id": { + "description": "Runtime being inspected.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "timeout_millis": { + "description": "Remaining health budget in milliseconds.", + "format": "uint64", + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "runtime_id", + "timeout_millis" + ], + "title": "AdapterHealthRequest", + "type": "object" +} \ No newline at end of file diff --git a/adapter-contract/typescript/schemas/adapter-health-result.schema.json b/adapter-contract/typescript/schemas/adapter-health-result.schema.json new file mode 100644 index 000000000..7a998afe0 --- /dev/null +++ b/adapter-contract/typescript/schemas/adapter-health-result.schema.json @@ -0,0 +1,151 @@ +{ + "$defs": { + "AdapterReadiness": { + "additionalProperties": false, + "description": "Adapter-owned readiness observation.", + "properties": { + "reason_code": { + "description": "Stable machine-readable reason.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "state": { + "$ref": "#/$defs/RuntimeReadiness", + "description": "Adapter readiness state." + } + }, + "required": [ + "state", + "reason_code" + ], + "type": "object" + }, + "HealthCheck": { + "additionalProperties": false, + "description": "One timestamped runtime or adapter health observation.", + "properties": { + "age_millis": { + "description": "Age of the evidence when this report was assembled.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "message": { + "description": "Optional human-readable diagnostic detail.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "metadata": { + "additionalProperties": true, + "description": "Additional non-sensitive check metadata.", + "type": "object" + }, + "name": { + "description": "Stable, namespaced check name.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed_at_millis": { + "description": "Unix timestamp in milliseconds when the evidence was observed.", + "format": "uint128", + "minimum": 0, + "type": "integer" + }, + "reason_code": { + "description": "Stable machine-readable reason.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "status": { + "$ref": "#/$defs/HealthCheckStatus", + "description": "Structured check outcome." + } + }, + "required": [ + "name", + "status", + "reason_code", + "observed_at_millis", + "age_millis" + ], + "type": "object" + }, + "HealthCheckStatus": { + "description": "Outcome of one runtime health check.", + "oneOf": [ + { + "const": "ok", + "description": "The check passed.", + "type": "string" + }, + { + "const": "failed", + "description": "The check observed a failure.", + "type": "string" + }, + { + "const": "unknown", + "description": "The check could not establish a result.", + "type": "string" + }, + { + "const": "unsupported", + "description": "The selected adapter does not implement the check.", + "type": "string" + } + ] + }, + "RuntimeReadiness": { + "description": "Whether Fabric knows the runtime can currently accept work.", + "oneOf": [ + { + "const": "ready", + "description": "The runtime can accept work under its current invocation policy.", + "type": "string" + }, + { + "const": "not_ready", + "description": "The runtime is known not to accept work.", + "type": "string" + }, + { + "const": "unknown", + "description": "Fabric lacks enough fresh evidence to decide.", + "type": "string" + } + ] + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "Optional adapter-specific contribution to a runtime health report.", + "properties": { + "checks": { + "description": "Adapter or dependency checks.", + "items": { + "$ref": "#/$defs/HealthCheck" + }, + "type": "array" + }, + "readiness": { + "anyOf": [ + { + "$ref": "#/$defs/AdapterReadiness" + }, + { + "type": "null" + } + ], + "description": "Adapter-owned readiness override when one is known." + } + }, + "title": "AdapterHealthResult", + "type": "object" +} \ No newline at end of file diff --git a/adapter-contract/typescript/scripts/check-package.mjs b/adapter-contract/typescript/scripts/check-package.mjs index e0b4c67e9..9c6c7ed27 100644 --- a/adapter-contract/typescript/scripts/check-package.mjs +++ b/adapter-contract/typescript/scripts/check-package.mjs @@ -42,6 +42,10 @@ try { "dist/generated/agent-run-request.js", "dist/generated/agent-run-result.d.ts", "dist/generated/agent-run-result.js", + "dist/generated/adapter-health-request.d.ts", + "dist/generated/adapter-health-request.js", + "dist/generated/adapter-health-result.d.ts", + "dist/generated/adapter-health-result.js", "dist/generated/runtime-context.d.ts", "dist/generated/runtime-context.js", "dist/index.d.ts", @@ -56,6 +60,8 @@ try { "schemas/agent-config.schema.json", "schemas/agent-run-request.schema.json", "schemas/agent-run-result.schema.json", + "schemas/adapter-health-request.schema.json", + "schemas/adapter-health-result.schema.json", "schemas/runtime-context.schema.json", ]); const missingFiles = [...expectedFiles].filter( diff --git a/adapter-contract/typescript/scripts/generate.mjs b/adapter-contract/typescript/scripts/generate.mjs index 1410b3c2e..9a26bb42e 100644 --- a/adapter-contract/typescript/scripts/generate.mjs +++ b/adapter-contract/typescript/scripts/generate.mjs @@ -53,6 +53,8 @@ const schemaSpecs = [ output: "agent-run-result.ts", generate: generateRunResult, }, + { name: "adapter-health-request", output: "adapter-health-request.ts" }, + { name: "adapter-health-result", output: "adapter-health-result.ts" }, { name: "runtime-context", output: "runtime-context.ts" }, ]; diff --git a/adapter-contract/typescript/src/generated/adapter-descriptor.ts b/adapter-contract/typescript/src/generated/adapter-descriptor.ts index 16d39d3cb..957cdffeb 100644 --- a/adapter-contract/typescript/src/generated/adapter-descriptor.ts +++ b/adapter-contract/typescript/src/generated/adapter-descriptor.ts @@ -113,6 +113,10 @@ export type RuntimeCapabilities = { * Whether an in-flight invocation can be cancelled. */ cancellation?: boolean; + /** + * Whether the selected runtime exposes bounded health observations. + */ + health?: boolean; /** * Additional adapter-specific capability metadata. */ diff --git a/adapter-contract/typescript/src/generated/adapter-health-request.ts b/adapter-contract/typescript/src/generated/adapter-health-request.ts new file mode 100644 index 000000000..c9e60910a --- /dev/null +++ b/adapter-contract/typescript/src/generated/adapter-health-request.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// This file is generated from the canonical adapter-contract JSON Schemas. +// Do not edit it directly; run `npm run generate` instead. + +/** + * Request passed to an optional adapter health hook. + */ +export interface AdapterHealthRequest { + /** + * Runtime being inspected. + */ + runtime_id: string; + /** + * Remaining health budget in milliseconds. + */ + timeout_millis: number; +} diff --git a/adapter-contract/typescript/src/generated/adapter-health-result.ts b/adapter-contract/typescript/src/generated/adapter-health-result.ts new file mode 100644 index 000000000..7a542a2ce --- /dev/null +++ b/adapter-contract/typescript/src/generated/adapter-health-result.ts @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// This file is generated from the canonical adapter-contract JSON Schemas. +// Do not edit it directly; run `npm run generate` instead. + +import type { JsonObject } from "../json.js"; + +/** + * Optional adapter-specific contribution to a runtime health report. + */ +export interface AdapterHealthResult { + /** + * Adapter or dependency checks. + */ + checks?: HealthCheck[]; + /** + * Adapter-owned readiness override when one is known. + */ + readiness?: AdapterReadiness | null; +} +/** + * One timestamped runtime or adapter health observation. + */ +export interface HealthCheck { + /** + * Age of the evidence when this report was assembled. + */ + age_millis: number; + /** + * Optional human-readable diagnostic detail. + */ + message?: string | null; + /** + * Additional non-sensitive check metadata. + */ + metadata?: JsonObject; + /** + * Stable, namespaced check name. + */ + name: string; + /** + * Unix timestamp in milliseconds when the evidence was observed. + */ + observed_at_millis: number; + /** + * Stable machine-readable reason. + */ + reason_code: string; + /** + * Structured check outcome. + */ + status: "ok" | "failed" | "unknown" | "unsupported"; +} +/** + * Adapter-owned readiness observation. + */ +export interface AdapterReadiness { + /** + * Stable machine-readable reason. + */ + reason_code: string; + /** + * Adapter readiness state. + */ + state: "ready" | "not_ready" | "unknown"; +} diff --git a/adapter-contract/typescript/src/index.ts b/adapter-contract/typescript/src/index.ts index cc93cb7cb..c3343e175 100644 --- a/adapter-contract/typescript/src/index.ts +++ b/adapter-contract/typescript/src/index.ts @@ -12,6 +12,8 @@ export type * from "./generated/adapter-target-descriptor.js"; export type * from "./generated/agent-config.js"; export type * from "./generated/agent-run-request.js"; export type * from "./generated/agent-run-result.js"; +export type * from "./generated/adapter-health-request.js"; +export type * from "./generated/adapter-health-result.js"; export type * from "./generated/runtime-context.js"; export type { JsonArray, diff --git a/adapter-contract/typescript/test/execution.test.ts b/adapter-contract/typescript/test/execution.test.ts index e658dd874..752cd94ac 100644 --- a/adapter-contract/typescript/test/execution.test.ts +++ b/adapter-contract/typescript/test/execution.test.ts @@ -2,12 +2,31 @@ // SPDX-License-Identifier: Apache-2.0 import type { + AdapterHealthRequest, + AdapterHealthResult, AgentRunError, AgentRunRequest, AgentRunResult, AgentRunStatus, } from "../src/index.js"; +const healthRequest: AdapterHealthRequest = { + runtime_id: "runtime-1", + timeout_millis: 1000, +}; +const healthResult: AdapterHealthResult = { + readiness: { state: "ready", reason_code: "ready" }, + checks: [ + { + name: "adapter.health", + status: "ok", + reason_code: "adapter_ready", + observed_at_millis: 1, + age_millis: 0, + }, + ], +}; + const requests: AgentRunRequest[] = [ { input: null }, { input: true }, @@ -34,6 +53,8 @@ void requests; void results; void status; void runError; +void healthRequest; +void healthResult; // @ts-expect-error successful results cannot include an error const invalidSuccess: AgentRunResult = { diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index e01410ee7..a7a33d9c6 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -116,6 +116,21 @@ state during `start`. Adapter stdout is reserved for the protocol; diagnostics are redirected to stderr. A host crash or protocol timeout terminates that runtime. +## Runtime Health + +The common host starts an authenticated loopback health endpoint and returns +its connection metadata to NeMo Fabric during lifecycle startup. The endpoint +is independent of the ordered lifecycle channel, so it responds while +`invoke` is running. Do not log or persist the endpoint token. + +The host always reports lifecycle activity and marks inference probing as +unsupported. An adapter can optionally implement +`async health(request) -> AdapterHealthResult` to add fast, adapter-owned +checks. Respect `request.timeout_millis`; do not invoke the agent, contact a +model merely to test availability, consume quota, or mutate runtime state. A +missing, failed, or timed-out hook becomes structured unsupported or unknown +health data and does not fail the runtime. + ## Relay Request Correlation In-process Relay SDK adapters that own their Agent scope can use diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/lifecycle.py b/adapters/python/common/src/nemo_fabric_adapters/common/lifecycle.py index c449b1efa..5d06da5b4 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/lifecycle.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/lifecycle.py @@ -6,9 +6,12 @@ from __future__ import annotations import asyncio +import hmac import json import os +import secrets import sys +import time import traceback from collections.abc import Awaitable from collections.abc import Callable @@ -18,13 +21,20 @@ from contextlib import redirect_stdout from contextlib import suppress from dataclasses import dataclass +from dataclasses import field from typing import Any from typing import Protocol from typing import TextIO +from nemo_fabric_adapter_contract.models import AdapterHealthRequest +from nemo_fabric_adapter_contract.models import AdapterHealthResult +from nemo_fabric_adapter_contract.models import AdapterReadiness from nemo_fabric_adapter_contract.models import AgentRunRequest from nemo_fabric_adapter_contract.models import AgentRunResult +from nemo_fabric_adapter_contract.models import HealthCheck +from nemo_fabric_adapter_contract.models import HealthCheckStatus from nemo_fabric_adapter_contract.models import RuntimeContext +from nemo_fabric_adapter_contract.models import RuntimeReadiness class AdapterRuntime(Protocol): @@ -56,6 +66,11 @@ async def stop(self) -> None: _OPENAI_STREAM_RECORD_LIMIT = 1024 * 1024 _UINT32_MAX = (1 << 32) - 1 _UINT64_MAX = (1 << 64) - 1 +_HEALTH_CONTROL_HOST = "127.0.0.1" +_HEALTH_CONTROL_PROTOCOL = "fabric.health/v1alpha1" +_HEALTH_REQUEST_LIMIT = 1024 * 1024 +_HEALTH_REQUEST_TIMEOUT = 10.0 +_HEALTH_RESPONSE_RESERVE_MILLIS = 50 class LifecycleError(Exception): @@ -391,11 +406,21 @@ class _HostState: runtime: AdapterRuntime | None = None runtime_id: str | None = None failed: bool = False + invoking: bool = False + stopping: bool = False + health_server: asyncio.AbstractServer | None = None + health_token: str | None = None + health_tasks: set[asyncio.Task[None]] = field(default_factory=set) def clear(self) -> None: self.runtime = None self.runtime_id = None self.failed = False + self.invoking = False + self.stopping = False + self.health_server = None + self.health_token = None + self.health_tasks.clear() def _error( @@ -504,6 +529,217 @@ async def _stop_after_eof(runtime: AdapterRuntime) -> None: traceback.print_exc(file=sys.stderr) +def _now_millis() -> int: + return time.time_ns() // 1_000_000 + + +def _health_check( + name: str, + status: HealthCheckStatus, + reason_code: str, + *, + message: str | None = None, +) -> HealthCheck: + return HealthCheck( + name=name, + status=status, + reason_code=reason_code, + observed_at_millis=_now_millis(), + age_millis=0, + message=message, + ) + + +async def _close_health_server(state: _HostState) -> None: + server = state.health_server + state.health_server = None + state.health_token = None + if server is not None: + server.close() + await server.wait_closed() + tasks = tuple(state.health_tasks) + state.health_tasks.clear() + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + +def _schedule_health_connection( + state: _HostState, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, +) -> None: + task = asyncio.create_task(_handle_health_connection(state, reader, writer)) + state.health_tasks.add(task) + task.add_done_callback(state.health_tasks.discard) + + +async def _start_health_server(state: _HostState) -> dict[str, Any]: + token = secrets.token_urlsafe(32) + server = await asyncio.start_server( + lambda reader, writer: _schedule_health_connection(state, reader, writer), + _HEALTH_CONTROL_HOST, + 0, + limit=_HEALTH_REQUEST_LIMIT, + ) + socket = server.sockets[0] + port = socket.getsockname()[1] + state.health_server = server + state.health_token = token + return { + "health_control": { + "protocol_version": _HEALTH_CONTROL_PROTOCOL, + "host": _HEALTH_CONTROL_HOST, + "port": port, + "token": token, + } + } + + +async def _handle_health_connection( + state: _HostState, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, +) -> None: + try: + encoded = await asyncio.wait_for( + reader.readline(), timeout=_HEALTH_REQUEST_TIMEOUT + ) + if not encoded or len(encoded) > _HEALTH_REQUEST_LIMIT: + return + message = json.loads(encoded) + if not isinstance(message, dict): + return + token = message.get("token") + if ( + not isinstance(token, str) + or state.health_token is None + or not hmac.compare_digest(token, state.health_token) + ): + return + if message.get("protocol_version") != _HEALTH_CONTROL_PROTOCOL: + return + request = AdapterHealthRequest.from_mapping( + { + "runtime_id": message.get("runtime_id"), + "timeout_millis": message.get("timeout_millis"), + } + ) + if request.runtime_id != state.runtime_id or state.runtime is None: + return + result = await _adapter_health(state, request) + response = { + "protocol_version": _HEALTH_CONTROL_PROTOCOL, + "runtime_id": request.runtime_id, + "result": result.to_mapping(), + } + writer.write( + json.dumps(response, separators=(",", ":"), ensure_ascii=False).encode() + + b"\n" + ) + await writer.drain() + except Exception: + traceback.print_exc(file=sys.stderr) + finally: + writer.close() + with suppress(OSError): + await writer.wait_closed() + + +async def _adapter_health( + state: _HostState, + request: AdapterHealthRequest, +) -> AdapterHealthResult: + checks = [ + _health_check( + "dependency.inference", + HealthCheckStatus.UNSUPPORTED, + "inference_probe_prohibited", + ) + ] + if state.failed: + return AdapterHealthResult( + readiness=AdapterReadiness( + state=RuntimeReadiness.NOT_READY, + reason_code="runtime_failed", + ), + checks=checks, + ) + if state.stopping: + return AdapterHealthResult( + readiness=AdapterReadiness( + state=RuntimeReadiness.NOT_READY, + reason_code="stop_in_progress", + ), + checks=checks, + ) + + readiness = AdapterReadiness( + state=( + RuntimeReadiness.NOT_READY + if state.invoking + else RuntimeReadiness.READY + ), + reason_code=("invocation_in_progress" if state.invoking else "ready"), + ) + hook = getattr(state.runtime, "health", None) + if not callable(hook): + checks.append( + _health_check( + "adapter.health", + HealthCheckStatus.UNSUPPORTED, + "check_unsupported", + ) + ) + return AdapterHealthResult(readiness=readiness, checks=checks) + + hook_budget_millis = max( + 0, + request.timeout_millis - _HEALTH_RESPONSE_RESERVE_MILLIS, + ) + if hook_budget_millis == 0: + checks.append( + _health_check( + "adapter.health", + HealthCheckStatus.UNKNOWN, + "adapter_health_timed_out", + ) + ) + return AdapterHealthResult(readiness=readiness, checks=checks) + try: + result = await asyncio.wait_for( + _adapter_call("health", lambda: hook(request)), + timeout=hook_budget_millis / 1000, + ) + if not isinstance(result, AdapterHealthResult): + raise LifecycleError( + "lifecycle_invalid_health_response", + "Adapter health hook must return AdapterHealthResult", + ) + except TimeoutError: + checks.append( + _health_check( + "adapter.health", + HealthCheckStatus.UNKNOWN, + "adapter_health_timed_out", + ) + ) + except LifecycleError: + checks.append( + _health_check( + "adapter.health", + HealthCheckStatus.UNKNOWN, + "adapter_health_failed", + ) + ) + else: + checks.extend(result.checks) + if not state.invoking and result.readiness is not None: + readiness = result.readiness + return AdapterHealthResult(readiness=readiness, checks=checks) + + def _validated_request( message: dict[str, Any], operation: str ) -> tuple[dict[str, Any], str]: @@ -572,7 +808,16 @@ async def _handle_start( state.runtime = candidate state.runtime_id = message_runtime_id state.failed = False - return _response("start") + try: + output = await _start_health_server(state) + except Exception: + state.clear() + await _stop_after_eof(candidate) + raise LifecycleError( + "lifecycle_health_control_failed", + "Lifecycle host could not start its health control endpoint", + ) + return _response("start", output=output) async def _handle_invoke( @@ -667,7 +912,9 @@ async def _handle_stop( state: _HostState, runtime: AdapterRuntime, ) -> dict[str, Any]: + state.stopping = True try: + await _close_health_server(state) await _adapter_call("stop", runtime.stop) finally: state.clear() @@ -692,9 +939,17 @@ async def _dispatch( ) runtime = _active_runtime(state, message_runtime_id) if operation == "invoke": - return await _handle_invoke(state, runtime, payload) + state.invoking = True + try: + return await _handle_invoke(state, runtime, payload) + finally: + state.invoking = False if operation == "invoke_openai_stream": - return await _handle_invoke_openai_stream(state, runtime, payload) + state.invoking = True + try: + return await _handle_invoke_openai_stream(state, runtime, payload) + finally: + state.invoking = False return await _handle_stop(state, runtime) @@ -790,6 +1045,7 @@ async def _serve( break finally: if state.runtime is not None: + await _close_health_server(state) await _stop_after_eof(state.runtime) diff --git a/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json b/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json index 70a6e0929..1063e6b88 100644 --- a/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json +++ b/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json @@ -54,6 +54,7 @@ "capabilities": { "service": false, "streaming": false, + "health": true, "updates": false, "cancellation": false }, diff --git a/adapters/python/nooa/nooa-bench.fabric-adapter.json b/adapters/python/nooa/nooa-bench.fabric-adapter.json index 8ef2b3f71..04e151a22 100644 --- a/adapters/python/nooa/nooa-bench.fabric-adapter.json +++ b/adapters/python/nooa/nooa-bench.fabric-adapter.json @@ -54,6 +54,7 @@ "cancellation": false, "service": false, "streaming": false, + "health": true, "updates": false } } diff --git a/adapters/python/nooa/nooa.fabric-adapter.json b/adapters/python/nooa/nooa.fabric-adapter.json index 475c144c2..bf150b076 100644 --- a/adapters/python/nooa/nooa.fabric-adapter.json +++ b/adapters/python/nooa/nooa.fabric-adapter.json @@ -57,6 +57,7 @@ "cancellation": false, "service": false, "streaming": false, + "health": true, "updates": false } } diff --git a/adapters/python/remote-agent/remote-agent.fabric-adapter.json b/adapters/python/remote-agent/remote-agent.fabric-adapter.json index a6563914a..4e1c7e414 100644 --- a/adapters/python/remote-agent/remote-agent.fabric-adapter.json +++ b/adapters/python/remote-agent/remote-agent.fabric-adapter.json @@ -82,6 +82,7 @@ "capabilities": { "service": false, "streaming": true, + "health": true, "updates": false, "cancellation": false }, diff --git a/adapters/typescript/common/README.md b/adapters/typescript/common/README.md index 1e6746738..4d3310377 100644 --- a/adapters/typescript/common/README.md +++ b/adapters/typescript/common/README.md @@ -26,6 +26,15 @@ await serve(() => new MyAdapterRuntime()); The factory may return a runtime directly or resolve one asynchronously. The host begins reading lifecycle input before it awaits asynchronous adapter setup. +The host also exposes an authenticated loopback health endpoint that is +independent of ordered lifecycle traffic. It reports idle or busy activity even +when the runtime omits the optional +`health(request): Promise` method. Implement that method +only for fast adapter-owned checks, respect `request.timeout_millis`, and do not +invoke the agent or probe an inference model. Missing, failed, and timed-out +hooks become structured health data without failing the runtime. Do not log or +persist health-control credentials. + This package is intended to be published as the shared runtime dependency for TypeScript adapters. Its public API will be versioned independently from the adapters that use it. diff --git a/adapters/typescript/common/src/lifecycle.ts b/adapters/typescript/common/src/lifecycle.ts index dd0b73c3d..1964bb758 100644 --- a/adapters/typescript/common/src/lifecycle.ts +++ b/adapters/typescript/common/src/lifecycle.ts @@ -6,13 +6,17 @@ // stop operations, and returns normalized responses while keeping diagnostics // off the protocol output stream. +import { randomBytes, timingSafeEqual } from "node:crypto"; import { createRequire } from "node:module"; +import { createServer, type Server, type Socket } from "node:net"; import { createInterface } from "node:readline"; import type { Readable, Writable } from "node:stream"; import type { ValidateFunction } from "ajv"; import type { AgentConfig, + AdapterHealthRequest, + AdapterHealthResult, AgentRunRequest, AgentRunResult, JsonObject, @@ -31,6 +35,7 @@ export interface AdapterStartInput { export interface AdapterRuntime { start(input: AdapterStartInput): Promise; invoke(request: AgentRunRequest, context: RuntimeContext): Promise; + health?(request: AdapterHealthRequest): Promise; stop(): Promise; } @@ -46,6 +51,11 @@ interface HostState { runtime?: AdapterRuntime; runtimeId?: string; failed: boolean; + invoking: boolean; + stopping: boolean; + healthServer?: Server; + healthSockets?: Set; + healthToken?: string; } interface LifecycleRequest { @@ -97,6 +107,10 @@ ajv.addFormat("uint64", { type: "number", validate: (value: number) => Number.isSafeInteger(value) && value >= 0, }); +ajv.addFormat("uint128", { + type: "number", + validate: (value: number) => Number.isSafeInteger(value) && value >= 0, +}); ajv.addFormat("double", { type: "number", validate: (value: number) => Number.isFinite(value), @@ -105,6 +119,14 @@ const validateAgentConfig = compileSchema("agent-config"); const validateAgentRunRequest = compileSchema("agent-run-request"); const validateAgentRunResult = compileSchema("agent-run-result"); const validateRuntimeContext = compileSchema("runtime-context"); +const validateAdapterHealthRequest = compileSchema("adapter-health-request"); +const validateAdapterHealthResult = compileSchema("adapter-health-result"); + +const HEALTH_CONTROL_HOST = "127.0.0.1"; +const HEALTH_CONTROL_PROTOCOL = "fabric.health/v1alpha1"; +const HEALTH_REQUEST_LIMIT = 1024 * 1024; +const HEALTH_REQUEST_TIMEOUT_MILLIS = 10_000; +const HEALTH_RESPONSE_RESERVE_MILLIS = 50; function compileSchema(name: string): ValidateFunction { const schema = require(`nemo-fabric-adapter-contract/schemas/${name}`) as object; @@ -233,6 +255,236 @@ async function stopQuietly(runtime: AdapterRuntime, diagnostics: Writable): Prom } } +function nowMillis(): number { + return Date.now(); +} + +function healthCheck( + name: string, + status: "ok" | "failed" | "unknown" | "unsupported", + reasonCode: string, +): NonNullable[number] { + return { + name, + status, + reason_code: reasonCode, + observed_at_millis: nowMillis(), + age_millis: 0, + }; +} + +async function closeHealthServer(state: HostState): Promise { + const server = state.healthServer; + const sockets = state.healthSockets; + state.healthServer = undefined; + state.healthSockets = undefined; + state.healthToken = undefined; + if (server === undefined) { + return; + } + for (const socket of sockets ?? []) { + socket.destroy(); + } + await new Promise((resolve) => server.close(() => resolve())); +} + +async function startHealthServer(state: HostState): Promise { + const token = randomBytes(32).toString("base64url"); + const sockets = new Set(); + const server = createServer((socket) => { + sockets.add(socket); + socket.once("close", () => sockets.delete(socket)); + void handleHealthConnection(state, socket); + }); + await new Promise((resolve, reject) => { + const onError = (error: Error): void => reject(error); + server.once("error", onError); + server.listen(0, HEALTH_CONTROL_HOST, () => { + server.off("error", onError); + resolve(); + }); + }); + const address = server.address(); + if (address === null || typeof address === "string") { + server.close(); + throw new Error("health control server did not expose a TCP address"); + } + state.healthServer = server; + state.healthSockets = sockets; + state.healthToken = token; + return { + health_control: { + protocol_version: HEALTH_CONTROL_PROTOCOL, + host: HEALTH_CONTROL_HOST, + port: address.port, + token, + }, + }; +} + +function readHealthLine(socket: Socket): Promise { + return new Promise((resolve, reject) => { + let encoded = ""; + const cleanup = (): void => { + socket.off("data", onData); + socket.off("error", onError); + socket.off("end", onEnd); + socket.off("timeout", onTimeout); + }; + const onData = (chunk: Buffer): void => { + encoded += chunk.toString("utf8"); + if (Buffer.byteLength(encoded) > HEALTH_REQUEST_LIMIT) { + cleanup(); + reject(new Error("health request exceeds the size limit")); + return; + } + const newline = encoded.indexOf("\n"); + if (newline >= 0) { + cleanup(); + resolve(encoded.slice(0, newline)); + } + }; + const onError = (error: Error): void => { + cleanup(); + reject(error); + }; + const onEnd = (): void => { + cleanup(); + reject(new Error("health request ended before a complete record")); + }; + const onTimeout = (): void => { + cleanup(); + reject(new Error("health request timed out")); + }; + socket.on("data", onData); + socket.once("error", onError); + socket.once("end", onEnd); + socket.once("timeout", onTimeout); + socket.setTimeout(HEALTH_REQUEST_TIMEOUT_MILLIS); + }); +} + +function tokensEqual(actual: unknown, expected: string | undefined): boolean { + if (typeof actual !== "string" || expected === undefined) { + return false; + } + const actualBytes = Buffer.from(actual); + const expectedBytes = Buffer.from(expected); + return actualBytes.length === expectedBytes.length && timingSafeEqual(actualBytes, expectedBytes); +} + +async function handleHealthConnection(state: HostState, socket: Socket): Promise { + let responseSent = false; + try { + const message = requireRecord( + JSON.parse(await readHealthLine(socket)) as unknown, + "lifecycle_invalid_health_request", + "Health request must be an object", + ); + if (!tokensEqual(message.token, state.healthToken)) { + return; + } + if (message.protocol_version !== HEALTH_CONTROL_PROTOCOL) { + return; + } + const request = validate( + validateAdapterHealthRequest, + { + runtime_id: message.runtime_id, + timeout_millis: message.timeout_millis, + }, + "lifecycle_invalid_health_request", + "Health request does not match its typed contract", + ); + if (request.runtime_id !== state.runtimeId || state.runtime === undefined) { + return; + } + const result = await adapterHealth(state, request); + socket.end( + `${JSON.stringify({ + protocol_version: HEALTH_CONTROL_PROTOCOL, + runtime_id: request.runtime_id, + result, + })}\n`, + ); + responseSent = true; + } catch { + // Invalid and unauthenticated health requests fail closed without details. + } finally { + if (!responseSent) { + socket.destroy(); + } + } +} + +class HealthHookTimeout extends Error {} + +async function adapterHealth( + state: HostState, + request: AdapterHealthRequest, +): Promise { + const checks: NonNullable = [ + healthCheck("dependency.inference", "unsupported", "inference_probe_prohibited"), + ]; + if (state.failed) { + return { readiness: { state: "not_ready", reason_code: "runtime_failed" }, checks }; + } + if (state.stopping) { + return { readiness: { state: "not_ready", reason_code: "stop_in_progress" }, checks }; + } + + let readiness: NonNullable = state.invoking + ? { state: "not_ready", reason_code: "invocation_in_progress" } + : { state: "ready", reason_code: "ready" }; + const runtime = state.runtime; + const hook = runtime?.health; + if (hook === undefined) { + checks.push(healthCheck("adapter.health", "unsupported", "check_unsupported")); + return { readiness, checks }; + } + + const hookBudgetMillis = Math.max( + 0, + request.timeout_millis - HEALTH_RESPONSE_RESERVE_MILLIS, + ); + if (hookBudgetMillis === 0) { + checks.push(healthCheck("adapter.health", "unknown", "adapter_health_timed_out")); + return { readiness, checks }; + } + let timeout: ReturnType | undefined; + try { + const result = await Promise.race([ + callAdapter("health", () => hook.call(runtime, request)), + new Promise((_resolve, reject) => { + timeout = setTimeout(() => reject(new HealthHookTimeout()), hookBudgetMillis); + }), + ]); + validate( + validateAdapterHealthResult, + result, + "lifecycle_invalid_health_response", + "Adapter health hook returned an invalid result", + ); + checks.push(...(result.checks ?? [])); + if (!state.invoking && result.readiness !== undefined && result.readiness !== null) { + readiness = result.readiness; + } + } catch (error) { + checks.push( + healthCheck( + "adapter.health", + "unknown", + error instanceof HealthHookTimeout ? "adapter_health_timed_out" : "adapter_health_failed", + ), + ); + } finally { + if (timeout !== undefined) { + clearTimeout(timeout); + } + } + return { readiness, checks }; +} + function success(operation: string, output: unknown = null): LifecycleResponse { return { operation, outcome: { status: "succeeded", output } }; } @@ -271,16 +523,22 @@ async function dispatch( candidate = await callAdapter("start", factory); const active = candidate; await callAdapter("start", () => active.start(decodeStart(request.payload))); + state.runtime = active; + state.runtimeId = messageRuntimeId; + state.failed = false; + state.invoking = false; + state.stopping = false; + const output = await startHealthServer(state); + return success("start", output); } catch (error) { + await closeHealthServer(state); + state.runtime = undefined; + state.runtimeId = undefined; if (candidate !== undefined) { await stopQuietly(candidate, diagnostics); } throw error; } - state.runtime = candidate; - state.runtimeId = messageRuntimeId; - state.failed = false; - return success("start"); } if (state.runtime === undefined || state.runtimeId === undefined) { @@ -292,17 +550,24 @@ async function dispatch( if (request.operation === "stop") { const active = state.runtime; - await callAdapter("stop", () => active.stop()); - state.runtime = undefined; - state.runtimeId = undefined; - state.failed = false; - return success("stop"); + state.stopping = true; + await closeHealthServer(state); + try { + await callAdapter("stop", () => active.stop()); + state.runtime = undefined; + state.runtimeId = undefined; + state.failed = false; + return success("stop"); + } finally { + state.stopping = false; + } } if (state.failed) { throw new LifecycleError("lifecycle_runtime_failed", "Lifecycle runtime cannot accept another invocation"); } const { request: invocation, context } = decodeInvocation(request.payload); + state.invoking = true; try { const result = await callAdapter("invoke", () => state.runtime!.invoke(invocation, context)); validate( @@ -320,6 +585,8 @@ async function dispatch( state.failed = true; } throw error; + } finally { + state.invoking = false; } } @@ -360,7 +627,7 @@ export async function serve(factory: AdapterRuntimeFactory, options: LifecycleHo process.stdout.write = process.stderr.write.bind(process.stderr) as typeof process.stdout.write; } const lines = createInterface({ input, crlfDelay: Infinity }); - const state: HostState = { failed: false }; + const state: HostState = { failed: false, invoking: false, stopping: false }; try { for await (const line of lines) { @@ -408,6 +675,8 @@ export async function serve(factory: AdapterRuntimeFactory, options: LifecycleHo } } finally { lines.close(); + state.stopping = true; + await closeHealthServer(state); if (state.runtime !== undefined) { await stopQuietly(state.runtime, diagnostics); } diff --git a/adapters/typescript/common/test/lifecycle.test.mjs b/adapters/typescript/common/test/lifecycle.test.mjs index 3950c08ef..c549446b1 100644 --- a/adapters/typescript/common/test/lifecycle.test.mjs +++ b/adapters/typescript/common/test/lifecycle.test.mjs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import assert from "node:assert/strict"; +import { connect } from "node:net"; import { PassThrough } from "node:stream"; import test from "node:test"; @@ -66,6 +67,106 @@ async function exchange(factory, messages) { return encoded.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)); } +function responseReader(output) { + let encoded = ""; + let wake; + output.setEncoding("utf8"); + output.on("data", (chunk) => { + encoded += chunk; + wake?.(); + wake = undefined; + }); + return async () => { + while (!encoded.includes("\n")) { + await new Promise((resolve) => { + wake = resolve; + }); + } + const newline = encoded.indexOf("\n"); + const line = encoded.slice(0, newline); + encoded = encoded.slice(newline + 1); + return JSON.parse(line); + }; +} + +async function checkHealth(control, runtimeId, timeoutMillis = 1000) { + const socket = connect(control.port, control.host); + socket.setEncoding("utf8"); + let encoded = ""; + const response = new Promise((resolve, reject) => { + socket.on("data", (chunk) => { + encoded += chunk; + const newline = encoded.indexOf("\n"); + if (newline >= 0) { + resolve(JSON.parse(encoded.slice(0, newline))); + } + }); + socket.once("error", reject); + socket.once("end", () => { + if (!encoded.includes("\n")) { + reject(new Error("health connection ended without a response")); + } + }); + }); + await new Promise((resolve) => socket.once("connect", resolve)); + socket.write(`${JSON.stringify({ + protocol_version: "fabric.health/v1alpha1", + token: control.token, + runtime_id: runtimeId, + timeout_millis: timeoutMillis, + })}\n`); + return response; +} + +test("serves health independently while an invocation is busy", async () => { + const input = new PassThrough(); + const output = new PassThrough(); + const diagnostics = new PassThrough(); + const nextResponse = responseReader(output); + let releaseInvocation; + let invocationStarted; + const started = new Promise((resolve) => { + invocationStarted = resolve; + }); + const blocked = new Promise((resolve) => { + releaseInvocation = resolve; + }); + const runtime = { + async start() {}, + async invoke() { + invocationStarted(); + await blocked; + return { status: "succeeded", output: null }; + }, + async stop() {}, + }; + const serving = serve(() => runtime, { input, output, diagnostics }); + input.write(`${JSON.stringify(start("runtime-1"))}\n`); + const startResponse = await nextResponse(); + const control = startResponse.outcome.output.health_control; + + input.write(`${JSON.stringify(invoke("runtime-1", "one", "one"))}\n`); + await started; + const healthResponse = await checkHealth(control, "runtime-1"); + + assert.equal(healthResponse.result.readiness.state, "not_ready"); + assert.equal(healthResponse.result.readiness.reason_code, "invocation_in_progress"); + assert.deepEqual( + healthResponse.result.checks.map((check) => [check.name, check.status]), + [ + ["dependency.inference", "unsupported"], + ["adapter.health", "unsupported"], + ], + ); + + releaseInvocation(); + await nextResponse(); + input.write(`${JSON.stringify(stop("runtime-1"))}\n`); + await nextResponse(); + input.end(); + await serving; +}); + test("serves two ordered invocations and stops one runtime", async () => { const calls = []; const runtime = { diff --git a/adapters/typescript/pi/pi.fabric-adapter.json b/adapters/typescript/pi/pi.fabric-adapter.json index 00add517a..f9944c8d0 100644 --- a/adapters/typescript/pi/pi.fabric-adapter.json +++ b/adapters/typescript/pi/pi.fabric-adapter.json @@ -63,6 +63,7 @@ }, "capabilities": { "streaming": false, + "health": true, "cancellation": false, "updates": false, "service": false diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 35a7f6e4c..2df0535a9 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -3486,6 +3486,7 @@ fn resolve_runtime_capabilities( RuntimeCapabilities { service: implemented_runtime && descriptor_capabilities.service, streaming: implemented_runtime && descriptor_capabilities.streaming, + health: implemented_runtime && descriptor_capabilities.health, updates: implemented_runtime && descriptor_capabilities.updates, cancellation: implemented_runtime && descriptor_capabilities.cancellation, metadata: descriptor_capabilities.metadata, @@ -3872,6 +3873,9 @@ pub struct RuntimeCapabilities { /// Whether invocations can emit progressive output. #[serde(default)] pub streaming: bool, + /// Whether the selected runtime exposes bounded health observations. + #[serde(default)] + pub health: bool, /// Whether a running runtime can accept config updates. #[serde(default)] pub updates: bool, diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index f86f551a1..10937fb0c 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -39,13 +39,16 @@ pub use config::{ pub use doctor::{DoctorCheck, DoctorReport, DoctorStatus, doctor_plan}; pub use error::{FabricError, Result}; pub use runtime::{ - AdapterInvocation, ArtifactManifest, ArtifactRef, EnvironmentHandle, ErrorInfo, ErrorStage, - FabricEvent, InvocationHandle, OpenAiChatCompletionChunk, OpenAiChatCompletionChunkChoice, - OpenAiChatCompletionChunkDelta, OpenAiChatCompletionChunkObject, OpenAiStreamHost, - OpenAiStreamInvocation, OpenAiStreamProfile, OpenAiStreamProtocolVersion, OpenAiStreamRecord, - OpenAiStreamSink, OpenAiStreamTransport, RunRequest, RunResult, RunStatus, RunUsage, - RuntimeContext, RuntimeHandle, RuntimeTelemetryContext, TelemetryRef, invoke_openai_stream, - invoke_runtime, prepare_environment, run_plan, start_runtime, stop_runtime, + AdapterHealthRequest, AdapterHealthResult, AdapterInvocation, AdapterReadiness, + ArtifactManifest, ArtifactRef, EnvironmentHandle, ErrorInfo, ErrorStage, FabricEvent, + HealthCheck, HealthCheckStatus, InvocationHandle, OpenAiChatCompletionChunk, + OpenAiChatCompletionChunkChoice, OpenAiChatCompletionChunkDelta, + OpenAiChatCompletionChunkObject, OpenAiStreamHost, OpenAiStreamInvocation, OpenAiStreamProfile, + OpenAiStreamProtocolVersion, OpenAiStreamRecord, OpenAiStreamSink, OpenAiStreamTransport, + RunRequest, RunResult, RunStatus, RunUsage, RuntimeActivity, RuntimeContext, RuntimeHandle, + RuntimeHealth, RuntimeLiveness, RuntimeReadiness, RuntimeTelemetryContext, TelemetryRef, + check_runtime_health, invoke_openai_stream, invoke_runtime, prepare_environment, run_plan, + start_runtime, stop_runtime, }; pub use schema::{ SchemaName, generate_all_schemas, generate_schema, generate_schema_json, write_schema_snapshots, diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 28b610174..6f49d1847 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -7,9 +7,10 @@ use std::collections::BTreeMap; use std::ffi::OsString; use std::fs::File; use std::io::{BufRead, BufReader, ErrorKind, Write}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpStream}; use std::path::{Path, PathBuf}; use std::process::{Child, ChildStdin, Command, Stdio}; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; use std::sync::{Arc, LazyLock, Mutex}; use std::thread; @@ -40,6 +41,8 @@ const LOCAL_HOST_START_TIMEOUT: Duration = Duration::from_secs(90); // timeouts and should return a normalized response before this bound. const LOCAL_HOST_INVOKE_TIMEOUT: Duration = Duration::from_secs(60 * 60); const LOCAL_HOST_STOP_TIMEOUT: Duration = Duration::from_secs(10); +const HEALTH_CONTROL_PROTOCOL: &str = "fabric.health/v1alpha1"; +const HEALTH_CONTROL_RESPONSE_LIMIT: u64 = 1024 * 1024; const LOCAL_HOST_EXIT_GRACE: Duration = Duration::from_secs(2); const LOCAL_HOST_DIAGNOSTIC_LIMIT: usize = 16 * 1024; #[cfg(test)] @@ -67,7 +70,7 @@ const DEFAULT_PYTHON: &str = "python3"; const DEFAULT_PYTHON: &str = "python.exe"; #[cfg(test)] static TEST_STOPPED_AGENTS: Mutex> = Mutex::new(Vec::new()); -static LOCAL_HOSTS: LazyLock>>>> = +static LOCAL_HOSTS: LazyLock>>> = LazyLock::new(|| Mutex::new(BTreeMap::new())); /// A request passed to a NeMo Fabric-managed harness runtime. @@ -319,6 +322,144 @@ pub struct RuntimeHandle { pub environment: EnvironmentHandle, } +/// Whether the adapter host can be reached independently of invocation traffic. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum RuntimeLiveness { + /// The adapter process and health control path responded within the deadline. + Responsive, + /// The process was observed running but its health control path did not respond. + Unresponsive, + /// Fabric directly observed that the adapter process exited. + Exited, + /// Fabric could not establish liveness. + Unknown, +} + +/// Current runtime activity observed by Fabric. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum RuntimeActivity { + /// No invocation or stop is in progress. + Idle, + /// At least one invocation is in progress or waiting on the runtime. + Busy, + /// Runtime shutdown is in progress. + Stopping, + /// Fabric could not establish activity. + Unknown, +} + +/// Whether Fabric knows the runtime can currently accept work. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum RuntimeReadiness { + /// The runtime can accept work under its current invocation policy. + Ready, + /// The runtime is known not to accept work. + NotReady, + /// Fabric lacks enough fresh evidence to decide. + Unknown, +} + +/// Outcome of one runtime health check. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum HealthCheckStatus { + /// The check passed. + Ok, + /// The check observed a failure. + Failed, + /// The check could not establish a result. + Unknown, + /// The selected adapter does not implement the check. + Unsupported, +} + +/// One timestamped runtime or adapter health observation. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct HealthCheck { + /// Stable, namespaced check name. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub name: String, + /// Structured check outcome. + pub status: HealthCheckStatus, + /// Stable machine-readable reason. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub reason_code: String, + /// Unix timestamp in milliseconds when the evidence was observed. + pub observed_at_millis: u128, + /// Age of the evidence when this report was assembled. + pub age_millis: u64, + /// Optional human-readable diagnostic detail. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub message: Option, + /// Additional non-sensitive check metadata. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub metadata: BTreeMap, +} + +/// Bounded health report for one started runtime. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct RuntimeHealth { + /// Runtime represented by this report. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub runtime_id: String, + /// Unix timestamp in milliseconds when the report completed. + pub checked_at_millis: u128, + /// Total probe duration in milliseconds. + pub duration_millis: u128, + /// Adapter-host liveness. + pub liveness: RuntimeLiveness, + /// Current invocation and shutdown activity. + pub activity: RuntimeActivity, + /// Current admission readiness. + pub readiness: RuntimeReadiness, + /// Stable reason for the top-level readiness decision. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub reason_code: String, + /// Ordered common and adapter-specific observations. + pub checks: Vec, +} + +/// Request passed to an optional adapter health hook. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AdapterHealthRequest { + /// Runtime being inspected. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub runtime_id: String, + /// Remaining health budget in milliseconds. + #[schemars(range(min = 1))] + pub timeout_millis: u64, +} + +/// Adapter-owned readiness observation. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AdapterReadiness { + /// Adapter readiness state. + pub state: RuntimeReadiness, + /// Stable machine-readable reason. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub reason_code: String, +} + +/// Optional adapter-specific contribution to a runtime health report. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AdapterHealthResult { + /// Adapter-owned readiness override when one is known. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub readiness: Option, + /// Adapter or dependency checks. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub checks: Vec, +} + /// One request sent to a runtime. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct InvocationHandle { @@ -724,6 +865,76 @@ struct LocalAdapterHost { relay_config: Option, } +struct LocalAdapterHostHandle { + host: Mutex, + health_control: Option, + active_invocations: AtomicUsize, + stopping: AtomicBool, +} + +impl LocalAdapterHostHandle { + fn activity(&self) -> RuntimeActivity { + if self.stopping.load(Ordering::Acquire) { + RuntimeActivity::Stopping + } else if self.active_invocations.load(Ordering::Acquire) > 0 { + RuntimeActivity::Busy + } else { + RuntimeActivity::Idle + } + } +} + +struct InvocationActivityGuard<'a>(&'a LocalAdapterHostHandle); + +impl Drop for InvocationActivityGuard<'_> { + fn drop(&mut self) { + self.0.active_invocations.fetch_sub(1, Ordering::AcqRel); + } +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +struct LocalHealthControl { + protocol_version: String, + host: String, + port: u16, + token: String, +} + +impl std::fmt::Debug for LocalHealthControl { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("LocalHealthControl") + .field("protocol_version", &self.protocol_version) + .field("host", &self.host) + .field("port", &self.port) + .field("token", &"[REDACTED]") + .finish() + } +} + +#[derive(Default, Deserialize)] +struct AdapterLifecycleStartOutput { + #[serde(default)] + health_control: Option, +} + +#[derive(Serialize)] +struct HealthControlRequest<'a> { + protocol_version: &'static str, + token: &'a str, + runtime_id: &'a str, + timeout_millis: u64, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct HealthControlResponse { + protocol_version: String, + runtime_id: String, + result: AdapterHealthResult, +} + /// Invoke a NeMo Fabric run plan. pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { let runtime = start_runtime(plan)?; @@ -889,6 +1100,436 @@ pub fn invoke_openai_stream( }) } +/// Inspect a started runtime without invoking the agent or changing its state. +pub fn check_runtime_health( + plan: &RunPlan, + runtime: &RuntimeHandle, + timeout: Duration, +) -> Result { + validate_runtime_handle(plan, runtime)?; + if timeout.is_zero() { + return Err(FabricError::InvalidConfig { + field: "health.timeout_seconds".to_string(), + reason: "must be greater than zero".to_string(), + }); + } + if !uses_local_host(plan) { + return Err(FabricError::UnsupportedRuntimeAdapter { + harness: runtime.harness.clone(), + adapter_kind: runtime.adapter_kind, + }); + } + + let started = Instant::now(); + let observed_at = now_millis(); + let Some(host) = local_hosts().get(&runtime.runtime_id).cloned() else { + return Ok(finish_health_report( + runtime, + started, + RuntimeLiveness::Unknown, + RuntimeActivity::Unknown, + RuntimeReadiness::Unknown, + "host_unavailable", + vec![ + health_check( + "adapter.process", + HealthCheckStatus::Unknown, + "host_unavailable", + observed_at, + ), + health_check( + "adapter.control", + HealthCheckStatus::Unsupported, + "control_unsupported", + observed_at, + ), + ], + )); + }; + + let activity = host.activity(); + let process = inspect_local_host_process(&host); + let mut checks = vec![match process { + ProcessObservation::Running => health_check( + "adapter.process", + HealthCheckStatus::Ok, + "process_running", + observed_at, + ), + ProcessObservation::Exited => health_check( + "adapter.process", + HealthCheckStatus::Failed, + "process_exited", + observed_at, + ), + ProcessObservation::Unknown => health_check( + "adapter.process", + HealthCheckStatus::Unknown, + "process_status_unknown", + observed_at, + ), + }]; + + if process == ProcessObservation::Exited { + checks.push(health_check( + "adapter.control", + HealthCheckStatus::Failed, + "process_exited", + observed_at, + )); + return Ok(finish_health_report( + runtime, + started, + RuntimeLiveness::Exited, + activity, + RuntimeReadiness::NotReady, + "process_exited", + checks, + )); + } + + if activity == RuntimeActivity::Stopping { + checks.push(health_check( + "adapter.control", + HealthCheckStatus::Unknown, + "stop_in_progress", + observed_at, + )); + return Ok(finish_health_report( + runtime, + started, + RuntimeLiveness::Unknown, + activity, + RuntimeReadiness::NotReady, + "stop_in_progress", + checks, + )); + } + + if !runtime_health_capability(plan) { + checks.push(health_check( + "adapter.control", + HealthCheckStatus::Unsupported, + "control_unsupported", + observed_at, + )); + let (readiness, reason) = if activity == RuntimeActivity::Busy { + (RuntimeReadiness::NotReady, "invocation_in_progress") + } else { + (RuntimeReadiness::Unknown, "control_unsupported") + }; + return Ok(finish_health_report( + runtime, + started, + RuntimeLiveness::Unknown, + activity, + readiness, + reason, + checks, + )); + } + + let Some(control) = host.health_control.as_ref() else { + checks.push(health_check( + "adapter.control", + HealthCheckStatus::Unsupported, + "control_unsupported", + observed_at, + )); + return Ok(finish_health_report( + runtime, + started, + RuntimeLiveness::Unknown, + activity, + if activity == RuntimeActivity::Busy { + RuntimeReadiness::NotReady + } else { + RuntimeReadiness::Unknown + }, + if activity == RuntimeActivity::Busy { + "invocation_in_progress" + } else { + "control_unsupported" + }, + checks, + )); + }; + + match probe_health_control(control, &runtime.runtime_id, started, timeout) { + HealthControlProbe::Succeeded(result) => { + checks.push(health_check( + "adapter.control", + HealthCheckStatus::Ok, + "probe_succeeded", + now_millis(), + )); + checks.extend(result.checks); + let (readiness, reason_code) = match activity { + RuntimeActivity::Busy => ( + RuntimeReadiness::NotReady, + "invocation_in_progress".to_string(), + ), + RuntimeActivity::Stopping => { + (RuntimeReadiness::NotReady, "stop_in_progress".to_string()) + } + RuntimeActivity::Idle | RuntimeActivity::Unknown => result + .readiness + .map(|readiness| (readiness.state, readiness.reason_code)) + .unwrap_or((RuntimeReadiness::Unknown, "readiness_unknown".to_string())), + }; + Ok(finish_health_report( + runtime, + started, + RuntimeLiveness::Responsive, + activity, + readiness, + &reason_code, + checks, + )) + } + HealthControlProbe::TimedOut => { + checks.push(health_check( + "adapter.control", + HealthCheckStatus::Unknown, + "probe_timed_out", + now_millis(), + )); + Ok(finish_health_report( + runtime, + started, + if process == ProcessObservation::Running { + RuntimeLiveness::Unresponsive + } else { + RuntimeLiveness::Unknown + }, + activity, + if activity == RuntimeActivity::Busy { + RuntimeReadiness::NotReady + } else { + RuntimeReadiness::Unknown + }, + if activity == RuntimeActivity::Busy { + "invocation_in_progress" + } else { + "probe_timed_out" + }, + checks, + )) + } + HealthControlProbe::Unavailable(reason_code) => { + checks.push(health_check( + "adapter.control", + HealthCheckStatus::Unknown, + reason_code, + now_millis(), + )); + Ok(finish_health_report( + runtime, + started, + if process == ProcessObservation::Running { + RuntimeLiveness::Unresponsive + } else { + RuntimeLiveness::Unknown + }, + activity, + if activity == RuntimeActivity::Busy { + RuntimeReadiness::NotReady + } else { + RuntimeReadiness::Unknown + }, + if activity == RuntimeActivity::Busy { + "invocation_in_progress" + } else { + reason_code + }, + checks, + )) + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ProcessObservation { + Running, + Exited, + Unknown, +} + +fn inspect_local_host_process(host: &LocalAdapterHostHandle) -> ProcessObservation { + match host.host.try_lock() { + Ok(mut host) => match host.child.try_wait() { + Ok(Some(_)) => ProcessObservation::Exited, + Ok(None) => ProcessObservation::Running, + Err(_) => ProcessObservation::Unknown, + }, + Err(std::sync::TryLockError::Poisoned(error)) => { + let mut host = error.into_inner(); + match host.child.try_wait() { + Ok(Some(_)) => ProcessObservation::Exited, + Ok(None) => ProcessObservation::Running, + Err(_) => ProcessObservation::Unknown, + } + } + Err(std::sync::TryLockError::WouldBlock) => ProcessObservation::Unknown, + } +} + +fn runtime_health_capability(plan: &RunPlan) -> bool { + plan.capabilities.health + && plan + .adapter_descriptor + .as_ref() + .is_some_and(|adapter| adapter.descriptor.capabilities.health) +} + +enum HealthControlProbe { + Succeeded(AdapterHealthResult), + TimedOut, + Unavailable(&'static str), +} + +fn probe_health_control( + control: &LocalHealthControl, + runtime_id: &str, + started: Instant, + timeout: Duration, +) -> HealthControlProbe { + if control.protocol_version != HEALTH_CONTROL_PROTOCOL || control.host != "127.0.0.1" { + return HealthControlProbe::Unavailable("control_protocol_error"); + } + let Some(remaining) = timeout.checked_sub(started.elapsed()) else { + return HealthControlProbe::TimedOut; + }; + if remaining.is_zero() { + return HealthControlProbe::TimedOut; + } + let address = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), control.port); + let mut stream = match TcpStream::connect_timeout(&address, remaining) { + Ok(stream) => stream, + Err(error) if is_timeout_error(&error) => return HealthControlProbe::TimedOut, + Err(_) => return HealthControlProbe::Unavailable("control_unavailable"), + }; + let Some(remaining) = timeout.checked_sub(started.elapsed()) else { + return HealthControlProbe::TimedOut; + }; + if remaining.is_zero() + || stream.set_read_timeout(Some(remaining)).is_err() + || stream.set_write_timeout(Some(remaining)).is_err() + { + return HealthControlProbe::TimedOut; + } + let request = HealthControlRequest { + protocol_version: HEALTH_CONTROL_PROTOCOL, + token: &control.token, + runtime_id, + timeout_millis: remaining.as_millis().clamp(1, u128::from(u64::MAX)) as u64, + }; + let mut encoded = match serde_json::to_vec(&request) { + Ok(encoded) => encoded, + Err(_) => return HealthControlProbe::Unavailable("control_protocol_error"), + }; + encoded.push(b'\n'); + if let Err(error) = stream.write_all(&encoded).and_then(|()| stream.flush()) { + return if is_timeout_error(&error) { + HealthControlProbe::TimedOut + } else { + HealthControlProbe::Unavailable("control_unavailable") + }; + } + let mut line = String::new(); + let mut limited = + std::io::Read::take(BufReader::new(stream), HEALTH_CONTROL_RESPONSE_LIMIT + 1); + let read_result = limited.read_line(&mut line); + if let Err(error) = read_result { + return if is_timeout_error(&error) { + HealthControlProbe::TimedOut + } else { + HealthControlProbe::Unavailable("control_unavailable") + }; + } + if line.len() as u64 > HEALTH_CONTROL_RESPONSE_LIMIT || !line.ends_with('\n') { + return HealthControlProbe::Unavailable("control_protocol_error"); + } + if line.len() > 1024 * 1024 { + return HealthControlProbe::Unavailable("control_protocol_error"); + } + let response: HealthControlResponse = match serde_json::from_str(&line) { + Ok(response) => response, + Err(_) => return HealthControlProbe::Unavailable("control_protocol_error"), + }; + if response.protocol_version != HEALTH_CONTROL_PROTOCOL + || response.runtime_id != runtime_id + || !valid_adapter_health_result(&response.result) + { + return HealthControlProbe::Unavailable("control_protocol_error"); + } + HealthControlProbe::Succeeded(response.result) +} + +fn valid_adapter_health_result(result: &AdapterHealthResult) -> bool { + let readiness_valid = result + .readiness + .as_ref() + .is_none_or(|readiness| !readiness.reason_code.trim().is_empty()); + readiness_valid + && result.checks.iter().all(|check| { + !check.name.trim().is_empty() + && !check.reason_code.trim().is_empty() + && check + .message + .as_ref() + .is_none_or(|message| !message.trim().is_empty()) + }) +} + +fn is_timeout_error(error: &std::io::Error) -> bool { + matches!(error.kind(), ErrorKind::TimedOut | ErrorKind::WouldBlock) +} + +fn health_check( + name: impl Into, + status: HealthCheckStatus, + reason_code: impl Into, + observed_at_millis: u128, +) -> HealthCheck { + HealthCheck { + name: name.into(), + status, + reason_code: reason_code.into(), + observed_at_millis, + age_millis: 0, + message: None, + metadata: BTreeMap::new(), + } +} + +fn finish_health_report( + runtime: &RuntimeHandle, + started: Instant, + liveness: RuntimeLiveness, + activity: RuntimeActivity, + readiness: RuntimeReadiness, + reason_code: impl Into, + mut checks: Vec, +) -> RuntimeHealth { + let checked_at_millis = now_millis(); + for check in &mut checks { + check.age_millis = checked_at_millis + .saturating_sub(check.observed_at_millis) + .min(u128::from(u64::MAX)) as u64; + } + RuntimeHealth { + runtime_id: runtime.runtime_id.clone(), + checked_at_millis, + duration_millis: started.elapsed().as_millis(), + liveness, + activity, + readiness, + reason_code: reason_code.into(), + checks, + } +} + fn validate_openai_stream_transport(transport: &OpenAiStreamTransport) -> Result<()> { if transport.port == 0 { return Err(FabricError::InvalidOpenAiStreamTransport { @@ -1171,18 +1812,45 @@ impl RuntimeAdapter for LocalHostAdapter { let request = AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Start(Box::new(start))); let mut host = spawn_local_host(plan, &runtime, artifacts, relay_config)?; - if let Err(error) = exchange_lifecycle_message( + let start_output = match exchange_lifecycle_message( &mut host, &runtime.runtime_id, &request, LOCAL_HOST_START_TIMEOUT, ) { - let _ = terminate_local_host(&mut host); - let _ = remove_local_host_files(&host); - return Err(error); - } - - local_hosts().insert(runtime.runtime_id.clone(), Arc::new(Mutex::new(host))); + Ok(output) => output, + Err(error) => { + let _ = terminate_local_host(&mut host); + let _ = remove_local_host_files(&host); + return Err(error); + } + }; + let start_output: AdapterLifecycleStartOutput = if start_output.is_null() { + AdapterLifecycleStartOutput::default() + } else { + match serde_json::from_value(start_output) { + Ok(output) => output, + Err(source) => { + let error = lifecycle_error( + AdapterLifecycleOperation::Start, + &runtime.runtime_id, + "protocol_error", + format!("invalid lifecycle start output: {source}"), + local_host_diagnostics(&host), + ); + let _ = terminate_local_host(&mut host); + let _ = remove_local_host_files(&host); + return Err(error); + } + } + }; + let handle = LocalAdapterHostHandle { + host: Mutex::new(host), + health_control: start_output.health_control, + active_invocations: AtomicUsize::new(0), + stopping: AtomicBool::new(false), + }; + local_hosts().insert(runtime.runtime_id.clone(), Arc::new(handle)); Ok(runtime) } @@ -1209,7 +1877,8 @@ impl RuntimeAdapter for LocalHostAdapter { let Some(host) = local_hosts().remove(&runtime.runtime_id) else { return Ok(vec![local_host_stop_event(runtime, true, false)]); }; - let mut host = host.lock().unwrap_or_else(|error| error.into_inner()); + host.stopping.store(true, Ordering::Release); + let mut host = host.host.lock().unwrap_or_else(|error| error.into_inner()); let request = AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Stop(AdapterLifecycleStop { runtime_id: runtime.runtime_id.clone(), @@ -1355,9 +2024,11 @@ fn run_local_host_invocation_with_timeout( "", ) })?; + host.active_invocations.fetch_add(1, Ordering::AcqRel); + let _activity = InvocationActivityGuard(&host); let exchange_result = { - let mut host_guard = host.lock().unwrap_or_else(|error| error.into_inner()); + let mut host_guard = host.host.lock().unwrap_or_else(|error| error.into_inner()); let artifacts = host_guard.artifacts.clone(); let relay_config = host_guard.relay_config.clone(); let fabric_home = prepare_fabric_home(&artifacts, runtime, &invocation)?; @@ -1593,7 +2264,7 @@ fn run_local_host_invocation_with_timeout( fn invalidate_timed_out_local_host( runtime_id: &str, - expected_host: &Arc>, + expected_host: &Arc, host: &mut LocalAdapterHost, ) { { @@ -1682,7 +2353,7 @@ fn local_host_stop_event( ) } -fn local_hosts() -> std::sync::MutexGuard<'static, BTreeMap>>> { +fn local_hosts() -> std::sync::MutexGuard<'static, BTreeMap>> { LOCAL_HOSTS .lock() .unwrap_or_else(|error| error.into_inner()) @@ -2961,6 +3632,7 @@ mod tests { "adapter_id": "acme.fabric.local-host", "adapter_kind": "python", "runner": {"module": "fake_host"}, + "capabilities": {"health": true}, "settings_schema": { "type": "object", "properties": { @@ -3009,6 +3681,7 @@ mod tests { import os import socket import sys +import threading import time MODE = os.environ.get("FABRIC_FAKE_HOST_MODE", "success") @@ -3079,6 +3752,50 @@ def write_openai_stream(sink, chunks): if read_http_response(stream) != 200: raise RuntimeError("stream listener rejected records") +def start_health_control(runtime_id): + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + token = "fake-health-token" + + def serve_health(): + client, _address = listener.accept() + with client: + stream = client.makefile("rb") + request = json.loads(stream.readline()) + if request.get("token") != token: + return + if MODE == "health_timeout": + time.sleep(1) + return + result = { + "readiness": {"state": "ready", "reason_code": "ready"}, + "checks": [{ + "name": "adapter.health", + "status": "ok", + "reason_code": "adapter_ready", + "observed_at_millis": int(time.time() * 1000), + "age_millis": 0, + }], + } + response = { + "protocol_version": "fabric.health/v1alpha1", + "runtime_id": runtime_id, + "result": result, + } + client.sendall(json.dumps(response).encode() + b"\n") + listener.close() + + threading.Thread(target=serve_health, daemon=True).start() + return { + "health_control": { + "protocol_version": "fabric.health/v1alpha1", + "host": "127.0.0.1", + "port": listener.getsockname()[1], + "token": token, + } + } + for line in sys.stdin: message = json.loads(line) operation = message["operation"] @@ -3087,7 +3804,12 @@ for line in sys.stdin: print("start diagnostic", file=sys.stderr, flush=True) response("start", error=failure("start", "fake_start", "start rejected")) sys.exit(16) - response("start") + output = ( + start_health_control(message["payload"]["runtime_context"]["runtime_id"]) + if MODE in {"health_success", "health_timeout", "health_busy"} + else None + ) + response("start", output=output) if MODE == "crash_after_start": os.close(0) print("host crashed intentionally", file=sys.stderr, flush=True) @@ -3095,6 +3817,9 @@ for line in sys.stdin: sys.exit(17) elif operation in {"invoke", "invoke_openai_stream"}: invocations += 1 + if MODE == "health_busy": + print("busy invocation accepted", file=sys.stderr, flush=True) + time.sleep(1) if MODE == "invoke_stderr": print(f"diagnostic-{invocations}", file=sys.stderr, flush=True) if MODE == "invoke_timeout": @@ -3316,6 +4041,114 @@ for line in sys.stdin: ) } + #[test] + fn local_host_health_reports_ready_without_mutating_runtime() { + let (root, plan) = local_host_plan("health_success"); + let runtime = start_runtime(&plan).expect("start local host"); + + let health = check_runtime_health(&plan, &runtime, Duration::from_secs(1)) + .expect("check runtime health"); + + assert_eq!(health.runtime_id, runtime.runtime_id); + assert_eq!(health.liveness, RuntimeLiveness::Responsive); + assert_eq!(health.activity, RuntimeActivity::Idle); + assert_eq!(health.readiness, RuntimeReadiness::Ready); + assert_eq!(health.reason_code, "ready"); + assert!(health.checks.iter().any(|check| { + check.name == "adapter.health" && check.status == HealthCheckStatus::Ok + })); + + let result = invoke_runtime(&plan, &runtime, RunRequest::text("after health")) + .expect("invoke after health check"); + assert_eq!(result.status, RunStatus::Succeeded); + stop_runtime(&plan, &runtime).expect("stop local host"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn local_host_health_marks_legacy_control_unsupported() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).expect("start local host"); + + let health = check_runtime_health(&plan, &runtime, Duration::from_secs(1)) + .expect("check runtime health"); + + assert_eq!(health.liveness, RuntimeLiveness::Unknown); + assert_eq!(health.activity, RuntimeActivity::Idle); + assert_eq!(health.readiness, RuntimeReadiness::Unknown); + assert_eq!(health.reason_code, "control_unsupported"); + assert!(health.checks.iter().any(|check| { + check.name == "adapter.control" && check.status == HealthCheckStatus::Unsupported + })); + + stop_runtime(&plan, &runtime).expect("stop local host"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn local_host_health_timeout_is_bounded_data() { + let (root, plan) = local_host_plan("health_timeout"); + let runtime = start_runtime(&plan).expect("start local host"); + let started = Instant::now(); + + let health = check_runtime_health(&plan, &runtime, Duration::from_millis(75)) + .expect("health timeout is a report"); + + assert!(started.elapsed() < Duration::from_millis(500)); + assert_eq!(health.liveness, RuntimeLiveness::Unresponsive); + assert_eq!(health.readiness, RuntimeReadiness::Unknown); + assert_eq!(health.reason_code, "probe_timed_out"); + assert!(health.checks.iter().any(|check| { + check.name == "adapter.control" && check.reason_code == "probe_timed_out" + })); + + stop_runtime(&plan, &runtime).expect("stop local host"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn local_host_health_uses_control_path_during_invocation() { + let (root, plan) = local_host_plan("health_busy"); + let runtime = start_runtime(&plan).expect("start local host"); + let stderr_path = local_hosts()[&runtime.runtime_id] + .host + .lock() + .expect("local host") + .stderr_path + .clone(); + let invoke_plan = plan.clone(); + let invoke_runtime_handle = runtime.clone(); + let invocation = thread::spawn(move || { + invoke_runtime( + &invoke_plan, + &invoke_runtime_handle, + RunRequest::text("busy"), + ) + }); + let accepted_deadline = Instant::now() + Duration::from_secs(2); + while !fs::read_to_string(&stderr_path) + .expect("read host stderr") + .contains("busy invocation accepted") + { + assert!( + Instant::now() < accepted_deadline, + "busy invocation was not accepted" + ); + thread::sleep(Duration::from_millis(10)); + } + + let health = check_runtime_health(&plan, &runtime, Duration::from_millis(500)) + .expect("check busy runtime health"); + + assert_eq!(health.liveness, RuntimeLiveness::Responsive); + assert_eq!(health.activity, RuntimeActivity::Busy); + assert_eq!(health.readiness, RuntimeReadiness::NotReady); + assert_eq!(health.reason_code, "invocation_in_progress"); + invocation.join().expect("join invocation").expect("invoke"); + stop_runtime(&plan, &runtime).expect("stop local host"); + let _ = fs::remove_dir_all(root); + } + #[test] fn adapter_lifecycle_always_receives_southbound_agent_config() { let (root, plan) = local_host_plan("success"); @@ -3611,6 +4444,7 @@ for line in sys.stdin: .cloned() .expect("active local host"); let relay_config_path = host + .host .lock() .expect("local host") .relay_config @@ -3661,7 +4495,7 @@ for line in sys.stdin: .get(&runtime.runtime_id) .cloned() .expect("active local host"); - let runtime_dir = host.lock().expect("local host").runtime_dir.clone(); + let runtime_dir = host.host.lock().expect("local host").runtime_dir.clone(); let error = run_local_host_adapter_with_timeout( &plan, @@ -3681,7 +4515,8 @@ for line in sys.stdin: )); assert!(!local_hosts().contains_key(&runtime.runtime_id)); assert!( - host.lock() + host.host + .lock() .expect("local host") .child .try_wait() @@ -3883,7 +4718,7 @@ for line in sys.stdin: .get(&runtime.runtime_id) .cloned() .expect("active local host"); - let stderr_path = host.lock().expect("local host").stderr_path.clone(); + let stderr_path = host.host.lock().expect("local host").stderr_path.clone(); let first_plan = plan.clone(); let first_runtime = runtime.clone(); @@ -4149,7 +4984,7 @@ for line in sys.stdin: .get(&runtime.runtime_id) .cloned() .expect("active local host"); - let stderr_path = host.lock().expect("local host").stderr_path.clone(); + let stderr_path = host.host.lock().expect("local host").stderr_path.clone(); let closed_deadline = Instant::now() + Duration::from_secs(2); while !fs::read_to_string(&stderr_path) .expect("read host stderr") @@ -4162,7 +4997,8 @@ for line in sys.stdin: thread::sleep(Duration::from_millis(10)); } assert!( - host.lock() + host.host + .lock() .expect("local host") .child .try_wait() diff --git a/crates/fabric-core/src/schema.rs b/crates/fabric-core/src/schema.rs index bff9cc78c..21c9692c3 100644 --- a/crates/fabric-core/src/schema.rs +++ b/crates/fabric-core/src/schema.rs @@ -14,9 +14,9 @@ use crate::config::{ }; use crate::error::{FabricError, Result}; use crate::runtime::{ - AdapterInvocation, ArtifactManifest, EnvironmentHandle, ErrorInfo, FabricEvent, - InvocationHandle, OpenAiStreamInvocation, OpenAiStreamRecord, RunRequest, RunResult, - RuntimeContext, RuntimeHandle, + AdapterHealthRequest, AdapterHealthResult, AdapterInvocation, ArtifactManifest, + EnvironmentHandle, ErrorInfo, FabricEvent, InvocationHandle, OpenAiStreamInvocation, + OpenAiStreamRecord, RunRequest, RunResult, RuntimeContext, RuntimeHandle, RuntimeHealth, }; use crate::{AgentRunRequest, AgentRunResult}; @@ -39,6 +39,10 @@ pub enum SchemaName { RunPlan, /// Initialized-runtime invocation payload schema. AdapterInvocation, + /// Adapter-facing bounded health-hook request. + AdapterHealthRequest, + /// Adapter-facing bounded health-hook result. + AdapterHealthResult, /// Adapter-facing native OpenAI streaming invocation schema. OpenAiStreamInvocation, /// Adapter-native OpenAI streaming NDJSON record schema. @@ -49,6 +53,8 @@ pub enum SchemaName { EnvironmentHandle, /// Runtime handle schema. RuntimeHandle, + /// Bounded runtime health report. + RuntimeHealth, /// Invocation handle schema. InvocationHandle, /// Runtime request schema. @@ -65,7 +71,7 @@ pub enum SchemaName { impl SchemaName { /// All public schemas in stable output order. - pub const ALL: [Self; 19] = [ + pub const ALL: [Self; 22] = [ Self::Agent, Self::AgentConfig, Self::AgentRunRequest, @@ -74,11 +80,14 @@ impl SchemaName { Self::AdapterTargetDescriptor, Self::RunPlan, Self::AdapterInvocation, + Self::AdapterHealthRequest, + Self::AdapterHealthResult, Self::OpenAiStreamInvocation, Self::OpenAiStreamRecord, Self::RuntimeContext, Self::EnvironmentHandle, Self::RuntimeHandle, + Self::RuntimeHealth, Self::InvocationHandle, Self::RunRequest, Self::RunResult, @@ -98,11 +107,14 @@ impl SchemaName { Self::AdapterTargetDescriptor => "adapter-target-descriptor", Self::RunPlan => "run-plan", Self::AdapterInvocation => "adapter-invocation", + Self::AdapterHealthRequest => "adapter-health-request", + Self::AdapterHealthResult => "adapter-health-result", Self::OpenAiStreamInvocation => "openai-stream-invocation", Self::OpenAiStreamRecord => "openai-stream-record", Self::RuntimeContext => "runtime-context", Self::EnvironmentHandle => "environment-handle", Self::RuntimeHandle => "runtime-handle", + Self::RuntimeHealth => "runtime-health", Self::InvocationHandle => "invocation-handle", Self::RunRequest => "run-request", Self::RunResult => "run-result", @@ -127,6 +139,8 @@ impl SchemaName { | Self::AdapterDescriptor | Self::AdapterTargetDescriptor | Self::AdapterInvocation + | Self::AdapterHealthRequest + | Self::AdapterHealthResult | Self::OpenAiStreamInvocation | Self::OpenAiStreamRecord | Self::RuntimeContext => PathBuf::from("adapter-contract").join(filename), @@ -147,6 +161,8 @@ impl SchemaName { } "run-plan" | "run_plan" => Ok(Self::RunPlan), "adapter-invocation" | "adapter_invocation" => Ok(Self::AdapterInvocation), + "adapter-health-request" | "adapter_health_request" => Ok(Self::AdapterHealthRequest), + "adapter-health-result" | "adapter_health_result" => Ok(Self::AdapterHealthResult), "openai-stream-invocation" | "openai_stream_invocation" => { Ok(Self::OpenAiStreamInvocation) } @@ -154,6 +170,7 @@ impl SchemaName { "runtime-context" | "runtime_context" => Ok(Self::RuntimeContext), "environment-handle" | "environment_handle" => Ok(Self::EnvironmentHandle), "runtime-handle" | "runtime_handle" => Ok(Self::RuntimeHandle), + "runtime-health" | "runtime_health" => Ok(Self::RuntimeHealth), "invocation-handle" | "invocation_handle" => Ok(Self::InvocationHandle), "run-request" | "run_request" => Ok(Self::RunRequest), "run-result" | "run_result" => Ok(Self::RunResult), @@ -182,11 +199,14 @@ pub fn generate_schema(schema: SchemaName) -> Result { SchemaName::AdapterTargetDescriptor => to_value(schema_for!(AdapterTargetDescriptor)), SchemaName::RunPlan => to_value(schema_for!(RunPlan)), SchemaName::AdapterInvocation => to_value(schema_for!(AdapterInvocation)), + SchemaName::AdapterHealthRequest => to_value(schema_for!(AdapterHealthRequest)), + SchemaName::AdapterHealthResult => to_value(schema_for!(AdapterHealthResult)), SchemaName::OpenAiStreamInvocation => to_value(schema_for!(OpenAiStreamInvocation)), SchemaName::OpenAiStreamRecord => to_value(schema_for!(OpenAiStreamRecord)), SchemaName::RuntimeContext => to_value(schema_for!(RuntimeContext)), SchemaName::EnvironmentHandle => to_value(schema_for!(EnvironmentHandle)), SchemaName::RuntimeHandle => to_value(schema_for!(RuntimeHandle)), + SchemaName::RuntimeHealth => to_value(schema_for!(RuntimeHealth)), SchemaName::InvocationHandle => to_value(schema_for!(InvocationHandle)), SchemaName::RunRequest => to_value(schema_for!(RunRequest)), SchemaName::RunResult => to_value(schema_for!(RunResult)), diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 723512d72..a3257fce1 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -163,6 +163,28 @@ fn invoke_openai_stream( to_json(&result) } +/// Inspect a previously started runtime without changing its lifecycle state. +#[pyfunction] +fn check_runtime_health( + py: Python<'_>, + plan_json: String, + runtime_json: String, + timeout_millis: u64, +) -> PyResult { + let plan = parse_run_plan(plan_json)?; + let runtime = parse_runtime_handle(runtime_json)?; + let health = py + .detach(|| { + nemo_fabric_core::check_runtime_health( + &plan, + &runtime, + Duration::from_millis(timeout_millis), + ) + }) + .map_err(to_py_error)?; + to_json(&health) +} + /// Stop a previously started runtime and return FabricEvent list JSON. #[pyfunction] fn stop_runtime(py: Python<'_>, plan_json: String, runtime_json: String) -> PyResult { @@ -183,6 +205,7 @@ fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(start_runtime, m)?)?; m.add_function(wrap_pyfunction!(invoke_runtime, m)?)?; m.add_function(wrap_pyfunction!(invoke_openai_stream, m)?)?; + m.add_function(wrap_pyfunction!(check_runtime_health, m)?)?; m.add_function(wrap_pyfunction!(stop_runtime, m)?)?; Ok(()) } diff --git a/docs/adapter-contract/tutorials/adapter-descriptor.md b/docs/adapter-contract/tutorials/adapter-descriptor.md index b32a3d154..e4a05c618 100644 --- a/docs/adapter-contract/tutorials/adapter-descriptor.md +++ b/docs/adapter-contract/tutorials/adapter-descriptor.md @@ -47,6 +47,9 @@ native cancellation or streaming feature does not become a NeMo Fabric capability until the adapter binding implements the corresponding contract. Relay-backed ATOF streaming does not require `capabilities.streaming`; that flag is reserved for the optional native OpenAI streaming operation. +Set `capabilities.health` only when the selected local host implements the +authenticated health-control protocol. The maintained Python and TypeScript +common hosts provide that protocol. ## To Implement the Adapter Descriptor diff --git a/docs/adapter-contract/tutorials/conformance.md b/docs/adapter-contract/tutorials/conformance.md index 11634b11e..e0aeb149c 100644 --- a/docs/adapter-contract/tutorials/conformance.md +++ b/docs/adapter-contract/tutorials/conformance.md @@ -72,6 +72,7 @@ Test each descriptor claim independently: | Telemetry output | The output is produced and correlated to the intended invocation. | | Relay-backed stream | Ordinary `invoke` completes while correlated Agent Trajectory Observability Format (ATOF) records reach `Runtime.invoke_stream()`. | | Native OpenAI stream | Empty and multi-chunk streams, invalid records, early close, a separate terminal value, and exactly one target invocation. | +| Runtime health | Idle and busy reports, authenticated-control rejection, deadline enforcement, unsupported checks, and a subsequent successful invocation after a negative health result. | Do not claim reserved cancellation, update, or service capabilities until the installed NeMo Fabric runtime binding exposes and tests the corresponding diff --git a/docs/adapter-contract/tutorials/execution.md b/docs/adapter-contract/tutorials/execution.md index 49de858b0..9c8fae07b 100644 --- a/docs/adapter-contract/tutorials/execution.md +++ b/docs/adapter-contract/tutorials/execution.md @@ -29,6 +29,7 @@ The minimum adapter implements these operations: | --- | --- | | `start` | Validate startup-only requirements, translate `AgentConfig`, and retain one isolated target runtime. | | `invoke` | Translate one `AgentRunRequest`, execute the retained target, and return one `AgentRunResult`. | +| `health` | Optionally contribute bounded adapter-specific readiness and dependency observations through the common host. | | `stop` | Attempt to release every adapter-owned resource, including after partial startup or failed invocation. | The required order is one successful `start`, zero or more ordered `invoke` @@ -179,6 +180,44 @@ binding exposes and tests the corresponding adapter operation. native streaming is added only through the declared `invoke_openai_stream` capability. +### 5. Add Adapter-Specific Health Only When Useful + +The maintained common hosts expose an authenticated loopback health endpoint +that remains independent of ordered lifecycle traffic. Set +`capabilities.health: true` when the adapter uses that host. Fabric reports +process and control-path observations even when the runtime class does not +implement an adapter-specific hook. + +Implement the optional Python hook only for fast checks that do not invoke the +agent: + +```python +from nemo_fabric_adapter_contract.models import AdapterHealthResult +from nemo_fabric_adapter_contract.models import AdapterReadiness +from nemo_fabric_adapter_contract.models import RuntimeReadiness + + +class TargetRuntime: + async def health(self, request): + return AdapterHealthResult( + readiness=AdapterReadiness( + state=RuntimeReadiness.READY, + reason_code="ready", + ) + ) +``` + +Respect `request.timeout_millis`. Return stable reason codes and timestamped +`HealthCheck` values for adapter-owned dependencies. Do not send inference +requests, consume model quota, mutate conversation state, or include secrets in +messages or metadata. The common host reports an omitted hook as unsupported +and converts a hook timeout or failure into unknown health data without failing +the runtime. + +**Success Check**: Health responds within the caller's deadline during both an +idle runtime and an active invocation, and a failed health hook does not change +the next invocation outcome. + ## Summary In this tutorial, you have: @@ -189,6 +228,7 @@ In this tutorial, you have: - Separated lifecycle failures from terminal target failures without exposing secrets. - Added native streaming only where the declared capability requires it. +- Added bounded adapter health only where the descriptor declares it. ## Next Steps diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index db0b654f8..a4457654e 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -64,12 +64,14 @@ SPDX-License-Identifier: Apache-2.0 --> - [`types.DoctorReport`](./nemo_fabric.types.md#class-doctorreport): Aggregate preflight diagnostics for a resolved run plan. - [`types.ErrorInfo`](./nemo_fabric.types.md#class-errorinfo): Structured failure returned inside a normalized ``RunResult``. - [`types.FabricEvent`](./nemo_fabric.types.md#class-fabricevent): One normalized lifecycle or invocation event. +- [`types.HealthCheck`](./nemo_fabric.types.md#class-healthcheck): One timestamped runtime or adapter health observation. - [`types.RunOutput`](./nemo_fabric.types.md#class-runoutput): Normalized adapter output. - [`types.RunPlan`](./nemo_fabric.types.md#class-runplan): Immutable execution plan produced before a runtime is started. - [`types.RunResult`](./nemo_fabric.types.md#class-runresult): Normalized terminal result from one NeMo Fabric invocation. - [`types.RunUsage`](./nemo_fabric.types.md#class-runusage): Normalized invocation usage reported by an adapter target. - [`types.RuntimeCapabilities`](./nemo_fabric.types.md#class-runtimecapabilities): Operations declared by the resolved runtime and adapter. - [`types.RuntimeHandle`](./nemo_fabric.types.md#class-runtimehandle): Opaque identity and binding for one started runtime. +- [`types.RuntimeHealth`](./nemo_fabric.types.md#class-runtimehealth): Bounded health report for one started runtime. - [`types.TelemetryRef`](./nemo_fabric.types.md#class-telemetryref): Reference to external or persisted telemetry for a run. - [`errors.FabricCapabilityError`](./nemo_fabric.errors.md#class-fabriccapabilityerror): Operation rejected by resolved runtime capabilities or implementation status. - [`errors.FabricConfigError`](./nemo_fabric.errors.md#class-fabricconfigerror): Invalid SDK input, request shape, factory, or resolved config. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md index a9bde67df..58cc969ed 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md @@ -99,6 +99,39 @@ Return whether NVIDIA NeMo Relay ATOF streaming is enabled. --- +### method `check_health` + +```python +async def check_health(*, timeout_seconds: float = 3.0) -> RuntimeHealth +``` + +Inspect liveness and readiness without changing runtime state. + +Negative health outcomes, including timeouts and unsupported adapter checks, are returned as structured data. Only invalid SDK usage or an inability to perform the inspection raises an exception. + + + +**Args:** + + - `timeout_seconds`: Positive finite deadline for the complete check. + + + +**Returns:** + A typed snapshot of runtime liveness, activity, readiness, and individual checks. + + + +**Raises:** + + - `FabricConfigError`: If ``timeout_seconds`` is not positive and finite. + - `FabricStateError`: If the runtime has already stopped. + - `FabricNativeUnavailableError`: If the native extension is missing. + - `FabricRuntimeError`: If health inspection cannot be performed. + +--- + + ### method `invoke` ```python diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index c6be002c9..86b747cc5 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -107,6 +107,7 @@ Capabilities describe what the selected runtime can support; callers should stil - `service`: Whether long-lived service handles are supported. - `streaming`: Whether event streaming is supported. + - `health`: Whether bounded runtime health inspection is supported. - `updates`: Whether runtime configuration updates are supported. - `cancellation`: Whether in-flight cancellation is supported. - `metadata`: Additional capability details. @@ -121,6 +122,7 @@ The mapping exposes the following typed fields: | --- | --- | | `service` | `bool` | | `streaming` | `bool` | +| `health` | `bool` | | `updates` | `bool` | | `cancellation` | `bool` | | `metadata` | `Mapping[str, Any]` | @@ -886,6 +888,184 @@ def __init__(mapping: Mapping[str, Any]) -> None +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + + +## class `HealthCheck` + +One timestamped runtime or adapter health observation. + + + +**Attributes:** + + - `name`: Stable, namespaced check name. + - `status`: One of ``ok``, ``failed``, ``unknown``, or ``unsupported``. + - `reason_code`: Stable machine-readable reason. + - `observed_at_millis`: Unix timestamp when the evidence was observed. + - `age_millis`: Age of the evidence when the report was assembled. + - `message`: Optional human-readable diagnostic detail. + - `metadata`: Additional non-sensitive check metadata. + + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `name` | `str` | +| `status` | `str` | +| `reason_code` | `str` | +| `observed_at_millis` | `int` | +| `age_millis` | `int` | +| `message` | `str \| None` | +| `metadata` | `Mapping[str, Any]` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + + +## class `RuntimeHealth` + +Bounded health report for one started runtime. + + + +**Attributes:** + + - `runtime_id`: Runtime represented by the report. + - `checked_at_millis`: Unix timestamp when the report completed. + - `duration_millis`: Total probe duration. + - `liveness`: Adapter-host liveness. + - `activity`: Current invocation and shutdown activity. + - `readiness`: Whether the runtime can currently accept work. + - `reason_code`: Stable reason for the readiness decision. + - `checks`: Ordered common and adapter-specific observations. + + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `runtime_id` | `str` | +| `checked_at_millis` | `int` | +| `duration_millis` | `int` | +| `liveness` | `str` | +| `activity` | `str` | +| `readiness` | `str` | +| `reason_code` | `str` | +| `checks` | `Sequence[HealthCheck]` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + --- ### property extra_fields diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index cd6a4847f..b82d7ac27 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 118 +position: 128 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index cae2a2644..295bbbfac 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 119 +position: 129 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 5893f339b..887b15e75 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 120 +position: 130 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index f6aeb152c..b10139738 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 121 +position: 131 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx index 0d972cb87..d1a628093 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n    pub streaming: bool,\n    pub updates: bool,\n    pub cancellation: bool,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
bool,\n    pub streaming: bool,\n    pub health: bool,\n    pub updates: bool,\n    pub cancellation: bool,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
Lifecycle behavior implemented by a resolved runtime path. @@ -23,6 +23,10 @@ Whether the selected runtime supports service lifecycle operations. Whether invocations can emit progressive output. +### `health: bool` + +Whether the selected runtime exposes bounded health observations. + ### `updates: bool` Whether a running runtime can accept config updates. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index 8f4297fd4..bf0b98c02 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 122 +position: 132 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index b4a351361..efed18bf7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 123 +position: 133 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index 4b159cfac..6b00f4d18 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 126 +position: 136 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index f67820946..9adbce873 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -92,13 +92,18 @@ Core config and runtime contract for NeMo Fabric. - `pub use doctor::doctor_plan;` - `pub use error::FabricError;` - `pub use error::Result;` +- `pub use runtime::AdapterHealthRequest;` +- `pub use runtime::AdapterHealthResult;` - `pub use runtime::AdapterInvocation;` +- `pub use runtime::AdapterReadiness;` - `pub use runtime::ArtifactManifest;` - `pub use runtime::ArtifactRef;` - `pub use runtime::EnvironmentHandle;` - `pub use runtime::ErrorInfo;` - `pub use runtime::ErrorStage;` - `pub use runtime::FabricEvent;` +- `pub use runtime::HealthCheck;` +- `pub use runtime::HealthCheckStatus;` - `pub use runtime::InvocationHandle;` - `pub use runtime::OpenAiChatCompletionChunk;` - `pub use runtime::OpenAiChatCompletionChunkChoice;` @@ -115,10 +120,15 @@ Core config and runtime contract for NeMo Fabric. - `pub use runtime::RunResult;` - `pub use runtime::RunStatus;` - `pub use runtime::RunUsage;` +- `pub use runtime::RuntimeActivity;` - `pub use runtime::RuntimeContext;` - `pub use runtime::RuntimeHandle;` +- `pub use runtime::RuntimeHealth;` +- `pub use runtime::RuntimeLiveness;` +- `pub use runtime::RuntimeReadiness;` - `pub use runtime::RuntimeTelemetryContext;` - `pub use runtime::TelemetryRef;` +- `pub use runtime::check_runtime_health;` - `pub use runtime::invoke_openai_stream;` - `pub use runtime::invoke_runtime;` - `pub use runtime::prepare_environment;` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx index 8c39c018f..db4c1b143 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx @@ -2,7 +2,7 @@ title: "Enum Error Stage" sidebar-title: "ErrorStage" description: "NeMo Fabric lifecycle stage associated with an error." -position: 21 +position: 26 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-healthcheckstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-healthcheckstatus.mdx new file mode 100644 index 000000000..6524ab6bb --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-healthcheckstatus.mdx @@ -0,0 +1,129 @@ +--- +title: "Enum Health Check Status" +sidebar-title: "HealthCheckStatus" +description: "Outcome of one runtime health check." +position: 27 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum HealthCheckStatus { + Ok, + Failed, + Unknown, + Unsupported, +} +``` + +Outcome of one runtime health check. + +## Variants + +### `Ok` + +
+ +The check passed. + +### `Failed` + +
+ +The check observed a failure. + +### `Unknown` + +
+ +The check could not establish a result. + +### `Unsupported` + +
+ +The selected adapter does not implement the check. + +## Trait Implementations + +### `impl Clone for HealthCheckStatus` + +
Clone for HealthCheckStatus"}} />
+ +#### `clone` + +
clone(&self) -> HealthCheckStatus"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for HealthCheckStatus` + +
Debug for HealthCheckStatus"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for HealthCheckStatus` + +
Deserialize<'de> for HealthCheckStatus"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for HealthCheckStatus` + +
HealthCheckStatus"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for HealthCheckStatus` + +
PartialEq for HealthCheckStatus"}} />
+ +#### `eq` + +
eq(&self, other: &HealthCheckStatus) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for HealthCheckStatus` + +
Serialize for HealthCheckStatus"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for HealthCheckStatus` + +
Copy for HealthCheckStatus"}} />
+ +### `impl Eq for HealthCheckStatus` + +
Eq for HealthCheckStatus"}} />
+ +### `impl StructuralPartialEq for HealthCheckStatus` + +
StructuralPartialEq for HealthCheckStatus"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx index bcc042797..7df46974b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Chat Completion Chunk Object" sidebar-title: "OpenAiChatCompletionChunkObject" description: "Exact OpenAI object discriminator accepted by the v1 chunk profile." -position: 22 +position: 28 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx index cd6acc0ed..fb1fc30ea 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Host" sidebar-title: "OpenAiStreamHost" description: "Supported native-streaming listener host." -position: 23 +position: 29 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx index 8d2ef88e4..870633f11 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Profile" sidebar-title: "OpenAiStreamProfile" description: "Supported OpenAI-compatible chunk profile." -position: 24 +position: 30 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx index e8b0a6423..83f95f3a6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Protocol Version" sidebar-title: "OpenAiStreamProtocolVersion" description: "Supported southbound native-streaming protocol version." -position: 25 +position: 31 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx index e2d65f637..725573f93 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Record" sidebar-title: "OpenAiStreamRecord" description: "One correlated NDJSON record on the adapter-native stream channel." -position: 26 +position: 32 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx index e1b66766a..89a7b1eca 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx @@ -2,7 +2,7 @@ title: "Enum RunStatus" sidebar-title: "RunStatus" description: "Runtime completion status." -position: 27 +position: 33 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimeactivity.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimeactivity.mdx new file mode 100644 index 000000000..730fea377 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimeactivity.mdx @@ -0,0 +1,129 @@ +--- +title: "Enum Runtime Activity" +sidebar-title: "RuntimeActivity" +description: "Current runtime activity observed by Fabric." +position: 34 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum RuntimeActivity { + Idle, + Busy, + Stopping, + Unknown, +} +``` + +Current runtime activity observed by Fabric. + +## Variants + +### `Idle` + +
+ +No invocation or stop is in progress. + +### `Busy` + +
+ +At least one invocation is in progress or waiting on the runtime. + +### `Stopping` + +
+ +Runtime shutdown is in progress. + +### `Unknown` + +
+ +Fabric could not establish activity. + +## Trait Implementations + +### `impl Clone for RuntimeActivity` + +
Clone for RuntimeActivity"}} />
+ +#### `clone` + +
clone(&self) -> RuntimeActivity"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RuntimeActivity` + +
Debug for RuntimeActivity"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for RuntimeActivity` + +
Deserialize<'de> for RuntimeActivity"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RuntimeActivity` + +
RuntimeActivity"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RuntimeActivity` + +
PartialEq for RuntimeActivity"}} />
+ +#### `eq` + +
eq(&self, other: &RuntimeActivity) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RuntimeActivity` + +
Serialize for RuntimeActivity"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for RuntimeActivity` + +
Copy for RuntimeActivity"}} />
+ +### `impl Eq for RuntimeActivity` + +
Eq for RuntimeActivity"}} />
+ +### `impl StructuralPartialEq for RuntimeActivity` + +
StructuralPartialEq for RuntimeActivity"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimeliveness.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimeliveness.mdx new file mode 100644 index 000000000..6d60e9298 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimeliveness.mdx @@ -0,0 +1,129 @@ +--- +title: "Enum Runtime Liveness" +sidebar-title: "RuntimeLiveness" +description: "Whether the adapter host can be reached independently of invocation traffic." +position: 35 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum RuntimeLiveness { + Responsive, + Unresponsive, + Exited, + Unknown, +} +``` + +Whether the adapter host can be reached independently of invocation traffic. + +## Variants + +### `Responsive` + +
+ +The adapter process and health control path responded within the deadline. + +### `Unresponsive` + +
+ +The process was observed running but its health control path did not respond. + +### `Exited` + +
+ +Fabric directly observed that the adapter process exited. + +### `Unknown` + +
+ +Fabric could not establish liveness. + +## Trait Implementations + +### `impl Clone for RuntimeLiveness` + +
Clone for RuntimeLiveness"}} />
+ +#### `clone` + +
clone(&self) -> RuntimeLiveness"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RuntimeLiveness` + +
Debug for RuntimeLiveness"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for RuntimeLiveness` + +
Deserialize<'de> for RuntimeLiveness"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RuntimeLiveness` + +
RuntimeLiveness"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RuntimeLiveness` + +
PartialEq for RuntimeLiveness"}} />
+ +#### `eq` + +
eq(&self, other: &RuntimeLiveness) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RuntimeLiveness` + +
Serialize for RuntimeLiveness"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for RuntimeLiveness` + +
Copy for RuntimeLiveness"}} />
+ +### `impl Eq for RuntimeLiveness` + +
Eq for RuntimeLiveness"}} />
+ +### `impl StructuralPartialEq for RuntimeLiveness` + +
StructuralPartialEq for RuntimeLiveness"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimereadiness.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimereadiness.mdx new file mode 100644 index 000000000..c48a5cfad --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimereadiness.mdx @@ -0,0 +1,122 @@ +--- +title: "Enum Runtime Readiness" +sidebar-title: "RuntimeReadiness" +description: "Whether Fabric knows the runtime can currently accept work." +position: 36 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum RuntimeReadiness { + Ready, + NotReady, + Unknown, +} +``` + +Whether Fabric knows the runtime can currently accept work. + +## Variants + +### `Ready` + +
+ +The runtime can accept work under its current invocation policy. + +### `NotReady` + +
+ +The runtime is known not to accept work. + +### `Unknown` + +
+ +Fabric lacks enough fresh evidence to decide. + +## Trait Implementations + +### `impl Clone for RuntimeReadiness` + +
Clone for RuntimeReadiness"}} />
+ +#### `clone` + +
clone(&self) -> RuntimeReadiness"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RuntimeReadiness` + +
Debug for RuntimeReadiness"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for RuntimeReadiness` + +
Deserialize<'de> for RuntimeReadiness"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RuntimeReadiness` + +
RuntimeReadiness"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RuntimeReadiness` + +
PartialEq for RuntimeReadiness"}} />
+ +#### `eq` + +
eq(&self, other: &RuntimeReadiness) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RuntimeReadiness` + +
Serialize for RuntimeReadiness"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for RuntimeReadiness` + +
Copy for RuntimeReadiness"}} />
+ +### `impl Eq for RuntimeReadiness` + +
Eq for RuntimeReadiness"}} />
+ +### `impl StructuralPartialEq for RuntimeReadiness` + +
StructuralPartialEq for RuntimeReadiness"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-check-runtime-health.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-check-runtime-health.mdx new file mode 100644 index 000000000..8b66c421b --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-check-runtime-health.mdx @@ -0,0 +1,14 @@ +--- +title: "Function check_runtime_health" +sidebar-title: "check_runtime_health" +description: "Inspect a started runtime without invoking the agent or changing its state." +position: 37 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
RunPlan,\n    runtime: &RuntimeHandle,\n    timeout: Duration,\n) -> Result<RuntimeHealth>"}} />
+ +Inspect a started runtime without invoking the agent or changing its state. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx index 6316f0e99..2bfa8c1b8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx @@ -2,7 +2,7 @@ title: "Function invoke_openai_stream" sidebar-title: "invoke_openai_stream" description: "Invoke a started harness runtime and pass through native OpenAI chat-completion chunks." -position: 28 +position: 38 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx index 9d8575830..dcc4e491f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx @@ -2,7 +2,7 @@ title: "Function invoke_runtime" sidebar-title: "invoke_runtime" description: "Invoke a started harness runtime." -position: 29 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-environment.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-environment.mdx index 8b5adb0f6..5ea374157 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-environment.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-environment.mdx @@ -2,7 +2,7 @@ title: "Function prepare_environment" sidebar-title: "prepare_environment" description: "Resolve or attach to the execution environment context for a run plan." -position: 30 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx index e15bdb27e..03b7f8283 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx @@ -2,7 +2,7 @@ title: "Function run_plan" sidebar-title: "run_plan" description: "Invoke a NeMo Fabric run plan." -position: 31 +position: 41 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx index 09c5a9f81..8c43c631d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime" sidebar-title: "start_runtime" description: "Start or connect to a harness runtime." -position: 32 +position: 42 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx index 8a96d55a6..58e334534 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx @@ -2,7 +2,7 @@ title: "Function stop_runtime" sidebar-title: "stop_runtime" description: "Stop or detach from a harness runtime." -position: 33 +position: 43 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index 0efd18b7c..97b51b6ee 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 124 +position: 134 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -13,12 +13,16 @@ Runtime invocation helpers. ## Structs +- [AdapterHealthRequest](struct-adapterhealthrequest.mdx): Request passed to an optional adapter health hook. +- [AdapterHealthResult](struct-adapterhealthresult.mdx): Optional adapter-specific contribution to a runtime health report. - [AdapterInvocation](struct-adapterinvocation.mdx): One invocation against an initialized adapter runtime. +- [AdapterReadiness](struct-adapterreadiness.mdx): Adapter-owned readiness observation. - [ArtifactManifest](struct-artifactmanifest.mdx): Manifest of run artifacts. - [ArtifactRef](struct-artifactref.mdx): Reference to one artifact. - [EnvironmentHandle](struct-environmenthandle.mdx): Resolved execution environment context. - [ErrorInfo](struct-errorinfo.mdx): Normalized error metadata. - [FabricEvent](struct-fabricevent.mdx): NeMo Fabric lifecycle or progress event. +- [HealthCheck](struct-healthcheck.mdx): One timestamped runtime or adapter health observation. - [InvocationHandle](struct-invocationhandle.mdx): One request sent to a runtime. - [OpenAiChatCompletionChunk](struct-openaichatcompletionchunk.mdx): OpenAI Chat Completions chunk accepted by the native streaming profile. - [OpenAiChatCompletionChunkChoice](struct-openaichatcompletionchunkchoice.mdx): One choice within an OpenAI Chat Completions streaming chunk. @@ -31,21 +35,27 @@ Runtime invocation helpers. - [RunUsage](struct-runusage.mdx): Normalized invocation usage exposed to NeMo Fabric consumers. - [RuntimeContext](struct-runtimecontext.mdx): Context generated for one invocation of a started runtime. - [RuntimeHandle](struct-runtimehandle.mdx): Active or resumable harness runtime. +- [RuntimeHealth](struct-runtimehealth.mdx): Bounded health report for one started runtime. - [RuntimeTelemetryContext](struct-runtimetelemetrycontext.mdx): Runtime telemetry config passed to adapters. - [TelemetryRef](struct-telemetryref.mdx): Reference to telemetry emitted by Relay or another configured telemetry path. ## Enums - [ErrorStage](enum-errorstage.mdx): NeMo Fabric lifecycle stage associated with an error. +- [HealthCheckStatus](enum-healthcheckstatus.mdx): Outcome of one runtime health check. - [OpenAiChatCompletionChunkObject](enum-openaichatcompletionchunkobject.mdx): Exact OpenAI object discriminator accepted by the v1 chunk profile. - [OpenAiStreamHost](enum-openaistreamhost.mdx): Supported native-streaming listener host. - [OpenAiStreamProfile](enum-openaistreamprofile.mdx): Supported OpenAI-compatible chunk profile. - [OpenAiStreamProtocolVersion](enum-openaistreamprotocolversion.mdx): Supported southbound native-streaming protocol version. - [OpenAiStreamRecord](enum-openaistreamrecord.mdx): One correlated NDJSON record on the adapter-native stream channel. - [RunStatus](enum-runstatus.mdx): Runtime completion status. +- [RuntimeActivity](enum-runtimeactivity.mdx): Current runtime activity observed by Fabric. +- [RuntimeLiveness](enum-runtimeliveness.mdx): Whether the adapter host can be reached independently of invocation traffic. +- [RuntimeReadiness](enum-runtimereadiness.mdx): Whether Fabric knows the runtime can currently accept work. ## Functions +- [check_runtime_health](fn-check-runtime-health.mdx): Inspect a started runtime without invoking the agent or changing its state. - [invoke_openai_stream](fn-invoke-openai-stream.mdx): Invoke a started harness runtime and pass through native OpenAI chat-completion chunks. - [invoke_runtime](fn-invoke-runtime.mdx): Invoke a started harness runtime. - [prepare_environment](fn-prepare-environment.mdx): Resolve or attach to the execution environment context for a run plan. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterhealthrequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterhealthrequest.mdx new file mode 100644 index 000000000..b3fdc0d80 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterhealthrequest.mdx @@ -0,0 +1,98 @@ +--- +title: "Struct Adapter Health Request" +sidebar-title: "AdapterHealthRequest" +description: "Request passed to an optional adapter health hook." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub timeout_millis: u64,\n}"}} />
+ +Request passed to an optional adapter health hook. + +## Fields + +### `runtime_id: String` + +Runtime being inspected. + +### `timeout_millis: u64` + +Remaining health budget in milliseconds. + +## Trait Implementations + +### `impl Clone for AdapterHealthRequest` + +
Clone for AdapterHealthRequest"}} />
+ +#### `clone` + +
clone(&self) -> AdapterHealthRequest"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for AdapterHealthRequest` + +
Debug for AdapterHealthRequest"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for AdapterHealthRequest` + +
Deserialize<'de> for AdapterHealthRequest"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for AdapterHealthRequest` + +
AdapterHealthRequest"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for AdapterHealthRequest` + +
PartialEq for AdapterHealthRequest"}} />
+ +#### `eq` + +
eq(&self, other: &AdapterHealthRequest) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for AdapterHealthRequest` + +
Serialize for AdapterHealthRequest"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for AdapterHealthRequest` + +
StructuralPartialEq for AdapterHealthRequest"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterhealthresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterhealthresult.mdx new file mode 100644 index 000000000..8633210b9 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterhealthresult.mdx @@ -0,0 +1,106 @@ +--- +title: "Struct Adapter Health Result" +sidebar-title: "AdapterHealthResult" +description: "Optional adapter-specific contribution to a runtime health report." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<AdapterReadiness>,\n    pub checks: Vec<HealthCheck>,\n}"}} />
+ +Optional adapter-specific contribution to a runtime health report. + +## Fields + +### `readiness: Option` + +Adapter-owned readiness override when one is known. + +### `checks: Vec` + +Adapter or dependency checks. + +## Trait Implementations + +### `impl Clone for AdapterHealthResult` + +
Clone for AdapterHealthResult"}} />
+ +#### `clone` + +
clone(&self) -> AdapterHealthResult"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for AdapterHealthResult` + +
Debug for AdapterHealthResult"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for AdapterHealthResult` + +
Default for AdapterHealthResult"}} />
+ +#### `default` + +
default() -> AdapterHealthResult"}} />
+ +### `impl<'de> Deserialize<'de> for AdapterHealthResult` + +
Deserialize<'de> for AdapterHealthResult"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for AdapterHealthResult` + +
AdapterHealthResult"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for AdapterHealthResult` + +
PartialEq for AdapterHealthResult"}} />
+ +#### `eq` + +
eq(&self, other: &AdapterHealthResult) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for AdapterHealthResult` + +
Serialize for AdapterHealthResult"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for AdapterHealthResult` + +
StructuralPartialEq for AdapterHealthResult"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterinvocation.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterinvocation.mdx index 3473e867c..ed333b022 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterinvocation.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterinvocation.mdx @@ -2,7 +2,7 @@ title: "Struct Adapter Invocation" sidebar-title: "AdapterInvocation" description: "One invocation against an initialized adapter runtime." -position: 1 +position: 3 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterreadiness.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterreadiness.mdx new file mode 100644 index 000000000..146daafff --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-adapterreadiness.mdx @@ -0,0 +1,98 @@ +--- +title: "Struct Adapter Readiness" +sidebar-title: "AdapterReadiness" +description: "Adapter-owned readiness observation." +position: 4 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
RuntimeReadiness,\n    pub reason_code: String,\n}"}} />
+ +Adapter-owned readiness observation. + +## Fields + +### `state: RuntimeReadiness` + +Adapter readiness state. + +### `reason_code: String` + +Stable machine-readable reason. + +## Trait Implementations + +### `impl Clone for AdapterReadiness` + +
Clone for AdapterReadiness"}} />
+ +#### `clone` + +
clone(&self) -> AdapterReadiness"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for AdapterReadiness` + +
Debug for AdapterReadiness"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for AdapterReadiness` + +
Deserialize<'de> for AdapterReadiness"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for AdapterReadiness` + +
AdapterReadiness"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for AdapterReadiness` + +
PartialEq for AdapterReadiness"}} />
+ +#### `eq` + +
eq(&self, other: &AdapterReadiness) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for AdapterReadiness` + +
Serialize for AdapterReadiness"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for AdapterReadiness` + +
StructuralPartialEq for AdapterReadiness"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-artifactmanifest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-artifactmanifest.mdx index acb71cbbd..966a92d90 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-artifactmanifest.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-artifactmanifest.mdx @@ -2,7 +2,7 @@ title: "Struct Artifact Manifest" sidebar-title: "ArtifactManifest" description: "Manifest of run artifacts." -position: 2 +position: 5 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-artifactref.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-artifactref.mdx index 51efa0cb9..fa8e01e17 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-artifactref.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-artifactref.mdx @@ -2,7 +2,7 @@ title: "Struct Artifact Ref" sidebar-title: "ArtifactRef" description: "Reference to one artifact." -position: 3 +position: 6 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx index 55d47378d..5cb7860cb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx @@ -2,7 +2,7 @@ title: "Struct Environment Handle" sidebar-title: "EnvironmentHandle" description: "Resolved execution environment context." -position: 4 +position: 7 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx index 9468739a0..1d674eab7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx @@ -2,7 +2,7 @@ title: "Struct Error Info" sidebar-title: "ErrorInfo" description: "Normalized error metadata." -position: 5 +position: 8 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx index 5330e632a..1d1960830 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx @@ -2,7 +2,7 @@ title: "Struct Fabric Event" sidebar-title: "FabricEvent" description: "NeMo Fabric lifecycle or progress event." -position: 6 +position: 9 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-healthcheck.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-healthcheck.mdx new file mode 100644 index 000000000..83d733ec8 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-healthcheck.mdx @@ -0,0 +1,118 @@ +--- +title: "Struct Health Check" +sidebar-title: "HealthCheck" +description: "One timestamped runtime or adapter health observation." +position: 10 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub status: HealthCheckStatus,\n    pub reason_code: String,\n    pub observed_at_millis: u128,\n    pub age_millis: u64,\n    pub message: Option<String>,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+ +One timestamped runtime or adapter health observation. + +## Fields + +### `name: String` + +Stable, namespaced check name. + +### `status: HealthCheckStatus` + +Structured check outcome. + +### `reason_code: String` + +Stable machine-readable reason. + +### `observed_at_millis: u128` + +Unix timestamp in milliseconds when the evidence was observed. + +### `age_millis: u64` + +Age of the evidence when this report was assembled. + +### `message: Option` + +Optional human-readable diagnostic detail. + +### `metadata: BTreeMap` + +Additional non-sensitive check metadata. + +## Trait Implementations + +### `impl Clone for HealthCheck` + +
Clone for HealthCheck"}} />
+ +#### `clone` + +
clone(&self) -> HealthCheck"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for HealthCheck` + +
Debug for HealthCheck"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for HealthCheck` + +
Deserialize<'de> for HealthCheck"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for HealthCheck` + +
HealthCheck"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for HealthCheck` + +
PartialEq for HealthCheck"}} />
+ +#### `eq` + +
eq(&self, other: &HealthCheck) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for HealthCheck` + +
Serialize for HealthCheck"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for HealthCheck` + +
StructuralPartialEq for HealthCheck"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx index 3f3ac0d42..0b0979ed0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx @@ -2,7 +2,7 @@ title: "Struct Invocation Handle" sidebar-title: "InvocationHandle" description: "One request sent to a runtime." -position: 7 +position: 11 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx index 026b2de72..142362f3f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk" sidebar-title: "OpenAiChatCompletionChunk" description: "OpenAI Chat Completions chunk accepted by the native streaming profile." -position: 8 +position: 12 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx index 94b2e70ee..4cfc3b74e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Choice" sidebar-title: "OpenAiChatCompletionChunkChoice" description: "One choice within an OpenAI Chat Completions streaming chunk." -position: 9 +position: 13 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx index 845afed45..83e364808 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Delta" sidebar-title: "OpenAiChatCompletionChunkDelta" description: "Incremental assistant message fields carried by one OpenAI choice." -position: 10 +position: 14 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx index 49b4c9c49..e55f344bc 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Invocation" sidebar-title: "OpenAiStreamInvocation" description: "One adapter-native OpenAI streaming invocation." -position: 11 +position: 15 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx index 8ad3827d5..9390fbcd4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Sink" sidebar-title: "OpenAiStreamSink" description: "Adapter-facing stream sink with invocation identity generated by NVIDIA NeMo Fabric." -position: 12 +position: 16 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx index 2ed1e5c62..157db8401 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Transport" sidebar-title: "OpenAiStreamTransport" description: "SDK-owned loopback transport for one native OpenAI streaming invocation." -position: 13 +position: 17 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx index dfaf781b0..86fe4e1a6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx @@ -2,7 +2,7 @@ title: "Struct RunRequest" sidebar-title: "RunRequest" description: "A request passed to a NeMo Fabric-managed harness runtime." -position: 14 +position: 18 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 1cb5bd052..b7fb265ab 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -2,7 +2,7 @@ title: "Struct RunResult" sidebar-title: "RunResult" description: "Result from a NeMo Fabric-managed harness invocation." -position: 15 +position: 19 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx index 54b33e2d6..542bcde9a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Context" sidebar-title: "RuntimeContext" description: "Context generated for one invocation of a started runtime." -position: 17 +position: 21 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx index d8147e5a3..88d1d8ac1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Handle" sidebar-title: "RuntimeHandle" description: "Active or resumable harness runtime." -position: 18 +position: 22 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehealth.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehealth.mdx new file mode 100644 index 000000000..417f02aa3 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehealth.mdx @@ -0,0 +1,122 @@ +--- +title: "Struct Runtime Health" +sidebar-title: "RuntimeHealth" +description: "Bounded health report for one started runtime." +position: 23 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub checked_at_millis: u128,\n    pub duration_millis: u128,\n    pub liveness: RuntimeLiveness,\n    pub activity: RuntimeActivity,\n    pub readiness: RuntimeReadiness,\n    pub reason_code: String,\n    pub checks: Vec<HealthCheck>,\n}"}} />
+ +Bounded health report for one started runtime. + +## Fields + +### `runtime_id: String` + +Runtime represented by this report. + +### `checked_at_millis: u128` + +Unix timestamp in milliseconds when the report completed. + +### `duration_millis: u128` + +Total probe duration in milliseconds. + +### `liveness: RuntimeLiveness` + +Adapter-host liveness. + +### `activity: RuntimeActivity` + +Current invocation and shutdown activity. + +### `readiness: RuntimeReadiness` + +Current admission readiness. + +### `reason_code: String` + +Stable reason for the top-level readiness decision. + +### `checks: Vec` + +Ordered common and adapter-specific observations. + +## Trait Implementations + +### `impl Clone for RuntimeHealth` + +
Clone for RuntimeHealth"}} />
+ +#### `clone` + +
clone(&self) -> RuntimeHealth"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RuntimeHealth` + +
Debug for RuntimeHealth"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for RuntimeHealth` + +
Deserialize<'de> for RuntimeHealth"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RuntimeHealth` + +
RuntimeHealth"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RuntimeHealth` + +
PartialEq for RuntimeHealth"}} />
+ +#### `eq` + +
eq(&self, other: &RuntimeHealth) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RuntimeHealth` + +
Serialize for RuntimeHealth"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for RuntimeHealth` + +
StructuralPartialEq for RuntimeHealth"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx index 1525ed89d..b4cb07fc0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Telemetry Context" sidebar-title: "RuntimeTelemetryContext" description: "Runtime telemetry config passed to adapters." -position: 19 +position: 24 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx index 432ce72d1..1b08fbc1a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx @@ -2,7 +2,7 @@ title: "Struct RunUsage" sidebar-title: "RunUsage" description: "Normalized invocation usage exposed to NeMo Fabric consumers." -position: 16 +position: 20 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx index 2173873a3..43a0c1e96 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Ref" sidebar-title: "TelemetryRef" description: "Reference to telemetry emitted by Relay or another configured telemetry path." -position: 20 +position: 25 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index 69454d076..1b8a4b4ad 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -19,11 +19,14 @@ pub enum SchemaName { AdapterTargetDescriptor, RunPlan, AdapterInvocation, + AdapterHealthRequest, + AdapterHealthResult, OpenAiStreamInvocation, OpenAiStreamRecord, RuntimeContext, EnvironmentHandle, RuntimeHandle, + RuntimeHealth, InvocationHandle, RunRequest, RunResult, @@ -85,6 +88,18 @@ Resolved run plan schema. Initialized-runtime invocation payload schema. +### `AdapterHealthRequest` + +
+ +Adapter-facing bounded health-hook request. + +### `AdapterHealthResult` + +
+ +Adapter-facing bounded health-hook result. + ### `OpenAiStreamInvocation`
@@ -115,6 +130,12 @@ Environment handle schema. Runtime handle schema. +### `RuntimeHealth` + +
+ +Bounded runtime health report. + ### `InvocationHandle`
@@ -159,7 +180,7 @@ NeMo Fabric lifecycle event schema. #### `ALL` -
19]"}} />
+
22]"}} />
All public schemas in stable output order. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index 27de96a85..a60dea2c9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 125 +position: 135 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 4f38bf40d..ede457dfc 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -158,6 +158,37 @@ objects. NeMo Fabric provides the runtime contract. Applications own scheduling, queues, retries, worker scaling, and the number of runtimes to run. +### Check Runtime Health + +Call `Runtime.check_health(...)` to get a bounded snapshot of a started local +runtime without invoking the agent or changing the runtime state: + +```python +async with await fabric.start_runtime(config, base_dir=base) as runtime: + health = await runtime.check_health(timeout_seconds=3.0) + if health.readiness == "ready": + result = await runtime.invoke(input="Inspect the repository") +``` + +The returned `RuntimeHealth` contains `liveness`, `activity`, `readiness`, a +top-level `reason_code`, and ordered `HealthCheck` observations. A probe timeout, +an unavailable control endpoint, and an unsupported adapter check are report +data rather than exceptions. Inspect `reason_code` and each check's `status` +instead of parsing diagnostic messages. + +Health checks use an authenticated loopback control endpoint that is separate +from the ordered lifecycle channel. You can therefore check a runtime while an +invocation is active. Under the one-invocation policy, the report uses +`activity="busy"`, `readiness="not_ready"`, and +`reason_code="invocation_in_progress"`. The health timeout does not cancel an +invocation, stop the runtime, or mark it failed. + +The SDK raises `FabricConfigError` for a nonpositive or nonfinite timeout and +`FabricStateError` after the runtime has stopped. Transport or native binding +errors raise `FabricRuntimeError`. Health inspection is currently implemented +for local Process and Python adapter hosts. Older hosts and adapters that do not +declare `capabilities.health` return explicit unknown or unsupported evidence. + ## Configure Agents In Code ### Normalized Configuration Compatibility diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 7819d8b82..955a8faf3 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -96,6 +96,10 @@ descriptor schema. and the projected southbound `agent-run-request`. This envelope is an internal transport detail; the common Python host passes its members to the adapter as typed arguments. +- `adapter-contract/adapter-health-request`: runtime identity and remaining + deadline passed to an optional adapter health hook. +- `adapter-contract/adapter-health-result`: optional adapter readiness and + timestamped check observations returned through the health-control endpoint. - `adapter-contract/openai-stream-invocation`: current native OpenAI stream payload sent to an initialized persistent local adapter host. It contains the per-turn runtime context and request plus a NeMo Fabric-owned @@ -120,6 +124,8 @@ descriptor schema. - `sdk/environment-handle`: prepared execution environment context. - `sdk/runtime-handle`: active harness runtime identity and opaque adapter binding. - `sdk/invocation-handle`: one request or turn sent to a runtime. +- `sdk/runtime-health`: bounded runtime liveness, activity, readiness, reason, + and ordered common or adapter-specific checks. ### Results, Artifacts, and Diagnostics diff --git a/schemas/adapter-contract/adapter-descriptor.schema.json b/schemas/adapter-contract/adapter-descriptor.schema.json index 637dfbf2a..4dda2e2e7 100644 --- a/schemas/adapter-contract/adapter-descriptor.schema.json +++ b/schemas/adapter-contract/adapter-descriptor.schema.json @@ -254,6 +254,11 @@ "description": "Whether an in-flight invocation can be cancelled.", "type": "boolean" }, + "health": { + "default": false, + "description": "Whether the selected runtime exposes bounded health observations.", + "type": "boolean" + }, "metadata": { "additionalProperties": true, "description": "Additional adapter-specific capability metadata.", @@ -296,6 +301,7 @@ "$ref": "#/$defs/RuntimeCapabilities", "default": { "cancellation": false, + "health": false, "service": false, "streaming": false, "updates": false diff --git a/schemas/adapter-contract/adapter-health-request.schema.json b/schemas/adapter-contract/adapter-health-request.schema.json new file mode 100644 index 000000000..db721f294 --- /dev/null +++ b/schemas/adapter-contract/adapter-health-request.schema.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "Request passed to an optional adapter health hook.", + "properties": { + "runtime_id": { + "description": "Runtime being inspected.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "timeout_millis": { + "description": "Remaining health budget in milliseconds.", + "format": "uint64", + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "runtime_id", + "timeout_millis" + ], + "title": "AdapterHealthRequest", + "type": "object" +} \ No newline at end of file diff --git a/schemas/adapter-contract/adapter-health-result.schema.json b/schemas/adapter-contract/adapter-health-result.schema.json new file mode 100644 index 000000000..7a998afe0 --- /dev/null +++ b/schemas/adapter-contract/adapter-health-result.schema.json @@ -0,0 +1,151 @@ +{ + "$defs": { + "AdapterReadiness": { + "additionalProperties": false, + "description": "Adapter-owned readiness observation.", + "properties": { + "reason_code": { + "description": "Stable machine-readable reason.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "state": { + "$ref": "#/$defs/RuntimeReadiness", + "description": "Adapter readiness state." + } + }, + "required": [ + "state", + "reason_code" + ], + "type": "object" + }, + "HealthCheck": { + "additionalProperties": false, + "description": "One timestamped runtime or adapter health observation.", + "properties": { + "age_millis": { + "description": "Age of the evidence when this report was assembled.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "message": { + "description": "Optional human-readable diagnostic detail.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "metadata": { + "additionalProperties": true, + "description": "Additional non-sensitive check metadata.", + "type": "object" + }, + "name": { + "description": "Stable, namespaced check name.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed_at_millis": { + "description": "Unix timestamp in milliseconds when the evidence was observed.", + "format": "uint128", + "minimum": 0, + "type": "integer" + }, + "reason_code": { + "description": "Stable machine-readable reason.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "status": { + "$ref": "#/$defs/HealthCheckStatus", + "description": "Structured check outcome." + } + }, + "required": [ + "name", + "status", + "reason_code", + "observed_at_millis", + "age_millis" + ], + "type": "object" + }, + "HealthCheckStatus": { + "description": "Outcome of one runtime health check.", + "oneOf": [ + { + "const": "ok", + "description": "The check passed.", + "type": "string" + }, + { + "const": "failed", + "description": "The check observed a failure.", + "type": "string" + }, + { + "const": "unknown", + "description": "The check could not establish a result.", + "type": "string" + }, + { + "const": "unsupported", + "description": "The selected adapter does not implement the check.", + "type": "string" + } + ] + }, + "RuntimeReadiness": { + "description": "Whether Fabric knows the runtime can currently accept work.", + "oneOf": [ + { + "const": "ready", + "description": "The runtime can accept work under its current invocation policy.", + "type": "string" + }, + { + "const": "not_ready", + "description": "The runtime is known not to accept work.", + "type": "string" + }, + { + "const": "unknown", + "description": "Fabric lacks enough fresh evidence to decide.", + "type": "string" + } + ] + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "Optional adapter-specific contribution to a runtime health report.", + "properties": { + "checks": { + "description": "Adapter or dependency checks.", + "items": { + "$ref": "#/$defs/HealthCheck" + }, + "type": "array" + }, + "readiness": { + "anyOf": [ + { + "$ref": "#/$defs/AdapterReadiness" + }, + { + "type": "null" + } + ], + "description": "Adapter-owned readiness override when one is known." + } + }, + "title": "AdapterHealthResult", + "type": "object" +} \ No newline at end of file diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index b98b26973..8d6680dd3 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -131,6 +131,7 @@ "$ref": "#/$defs/RuntimeCapabilities", "default": { "cancellation": false, + "health": false, "service": false, "streaming": false, "updates": false @@ -2701,6 +2702,11 @@ "description": "Whether an in-flight invocation can be cancelled.", "type": "boolean" }, + "health": { + "default": false, + "description": "Whether the selected runtime exposes bounded health observations.", + "type": "boolean" + }, "metadata": { "additionalProperties": true, "description": "Additional adapter-specific capability metadata.", diff --git a/schemas/sdk/runtime-health.schema.json b/schemas/sdk/runtime-health.schema.json new file mode 100644 index 000000000..8976713b1 --- /dev/null +++ b/schemas/sdk/runtime-health.schema.json @@ -0,0 +1,215 @@ +{ + "$defs": { + "HealthCheck": { + "additionalProperties": false, + "description": "One timestamped runtime or adapter health observation.", + "properties": { + "age_millis": { + "description": "Age of the evidence when this report was assembled.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "message": { + "description": "Optional human-readable diagnostic detail.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "metadata": { + "additionalProperties": true, + "description": "Additional non-sensitive check metadata.", + "type": "object" + }, + "name": { + "description": "Stable, namespaced check name.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed_at_millis": { + "description": "Unix timestamp in milliseconds when the evidence was observed.", + "format": "uint128", + "minimum": 0, + "type": "integer" + }, + "reason_code": { + "description": "Stable machine-readable reason.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "status": { + "$ref": "#/$defs/HealthCheckStatus", + "description": "Structured check outcome." + } + }, + "required": [ + "name", + "status", + "reason_code", + "observed_at_millis", + "age_millis" + ], + "type": "object" + }, + "HealthCheckStatus": { + "description": "Outcome of one runtime health check.", + "oneOf": [ + { + "const": "ok", + "description": "The check passed.", + "type": "string" + }, + { + "const": "failed", + "description": "The check observed a failure.", + "type": "string" + }, + { + "const": "unknown", + "description": "The check could not establish a result.", + "type": "string" + }, + { + "const": "unsupported", + "description": "The selected adapter does not implement the check.", + "type": "string" + } + ] + }, + "RuntimeActivity": { + "description": "Current runtime activity observed by Fabric.", + "oneOf": [ + { + "const": "idle", + "description": "No invocation or stop is in progress.", + "type": "string" + }, + { + "const": "busy", + "description": "At least one invocation is in progress or waiting on the runtime.", + "type": "string" + }, + { + "const": "stopping", + "description": "Runtime shutdown is in progress.", + "type": "string" + }, + { + "const": "unknown", + "description": "Fabric could not establish activity.", + "type": "string" + } + ] + }, + "RuntimeLiveness": { + "description": "Whether the adapter host can be reached independently of invocation traffic.", + "oneOf": [ + { + "const": "responsive", + "description": "The adapter process and health control path responded within the deadline.", + "type": "string" + }, + { + "const": "unresponsive", + "description": "The process was observed running but its health control path did not respond.", + "type": "string" + }, + { + "const": "exited", + "description": "Fabric directly observed that the adapter process exited.", + "type": "string" + }, + { + "const": "unknown", + "description": "Fabric could not establish liveness.", + "type": "string" + } + ] + }, + "RuntimeReadiness": { + "description": "Whether Fabric knows the runtime can currently accept work.", + "oneOf": [ + { + "const": "ready", + "description": "The runtime can accept work under its current invocation policy.", + "type": "string" + }, + { + "const": "not_ready", + "description": "The runtime is known not to accept work.", + "type": "string" + }, + { + "const": "unknown", + "description": "Fabric lacks enough fresh evidence to decide.", + "type": "string" + } + ] + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "Bounded health report for one started runtime.", + "properties": { + "activity": { + "$ref": "#/$defs/RuntimeActivity", + "description": "Current invocation and shutdown activity." + }, + "checked_at_millis": { + "description": "Unix timestamp in milliseconds when the report completed.", + "format": "uint128", + "minimum": 0, + "type": "integer" + }, + "checks": { + "description": "Ordered common and adapter-specific observations.", + "items": { + "$ref": "#/$defs/HealthCheck" + }, + "type": "array" + }, + "duration_millis": { + "description": "Total probe duration in milliseconds.", + "format": "uint128", + "minimum": 0, + "type": "integer" + }, + "liveness": { + "$ref": "#/$defs/RuntimeLiveness", + "description": "Adapter-host liveness." + }, + "readiness": { + "$ref": "#/$defs/RuntimeReadiness", + "description": "Current admission readiness." + }, + "reason_code": { + "description": "Stable reason for the top-level readiness decision.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "runtime_id": { + "description": "Runtime represented by this report.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + } + }, + "required": [ + "runtime_id", + "checked_at_millis", + "duration_millis", + "liveness", + "activity", + "readiness", + "reason_code", + "checks" + ], + "title": "RuntimeHealth", + "type": "object" +} \ No newline at end of file diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py index ec46893f7..97a263765 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py @@ -53,12 +53,14 @@ from nemo_fabric.types import DoctorReport from nemo_fabric.types import ErrorInfo from nemo_fabric.types import FabricEvent +from nemo_fabric.types import HealthCheck from nemo_fabric.types import RunOutput from nemo_fabric.types import RunPlan from nemo_fabric.types import RunResult from nemo_fabric.types import RunUsage from nemo_fabric.types import RuntimeCapabilities from nemo_fabric.types import RuntimeHandle +from nemo_fabric.types import RuntimeHealth from nemo_fabric.types import TelemetryRef __all__ = [ @@ -78,6 +80,7 @@ "FabricError", "FabricEvent", "HarnessConfig", + "HealthCheck", "InstructionConfig", "InstructionsConfig", "InvokeStream", @@ -109,6 +112,7 @@ "RunUsage", "RuntimeCapabilities", "RuntimeHandle", + "RuntimeHealth", "RuntimeConfig", "Runtime", "RuntimeStatus", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi index bc5254a96..f8867889d 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi @@ -30,4 +30,9 @@ def invoke_openai_stream( request_json: str, transport_json: str, ) -> str: ... +def check_runtime_health( + plan_json: str, + runtime_json: str, + timeout_millis: int, +) -> str: ... def stop_runtime(plan_json: str, runtime_json: str) -> str: ... diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index 2112f90b8..2550c196b 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -8,6 +8,7 @@ import asyncio import json import logging +import math from collections.abc import Mapping, Sequence from contextlib import AsyncExitStack from copy import deepcopy @@ -27,10 +28,11 @@ from nemo_fabric.models import RunRequest from nemo_fabric.openai_streaming import OpenAIInvokeStream from nemo_fabric.streaming import InvokeStream -from nemo_fabric.types import RunPlan, RunResult, RuntimeHandle +from nemo_fabric.types import RunPlan, RunResult, RuntimeHandle, RuntimeHealth logger = logging.getLogger(__name__) +_UINT64_MAX = (1 << 64) - 1 class RuntimeStatus(str, Enum): @@ -150,6 +152,62 @@ def supports_openai_streaming(self) -> bool: and descriptor_capabilities.get("streaming") is True ) + async def check_health(self, *, timeout_seconds: float = 3.0) -> RuntimeHealth: + """Inspect liveness and readiness without changing runtime state. + + Negative health outcomes, including timeouts and unsupported adapter + checks, are returned as structured data. Only invalid SDK usage or an + inability to perform the inspection raises an exception. + + Args: + timeout_seconds: Positive finite deadline for the complete check. + + Returns: + A typed snapshot of runtime liveness, activity, readiness, and + individual checks. + + Raises: + FabricConfigError: If ``timeout_seconds`` is not positive and finite. + FabricStateError: If the runtime has already stopped. + FabricNativeUnavailableError: If the native extension is missing. + FabricRuntimeError: If health inspection cannot be performed. + """ + + if ( + isinstance(timeout_seconds, bool) + or not isinstance(timeout_seconds, (int, float)) + or not math.isfinite(timeout_seconds) + or timeout_seconds <= 0 + ): + raise FabricConfigError("timeout_seconds must be positive and finite") + if self._status is RuntimeStatus.STOPPED: + raise FabricStateError("cannot check health of a stopped runtime") + + try: + native = self._client._require_native_module("health") + timeout_millis = ( + _UINT64_MAX + if timeout_seconds >= _UINT64_MAX / 1000 + else min( + _UINT64_MAX, + max(1, math.ceil(float(timeout_seconds) * 1000)), + ) + ) + + def check() -> dict[str, Any]: + encoded = native.check_runtime_health( + json.dumps(self._plan.to_mapping()), + json.dumps(self._runtime.to_mapping()), + timeout_millis, + ) + return json.loads(encoded) + + return RuntimeHealth.from_mapping(await _call_blocking(check)) + except FabricError: + raise + except Exception as error: + raise FabricRuntimeError(str(error), stage="health") from error + async def invoke( self, *, diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 0dd057473..2649260cf 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -1358,6 +1358,7 @@ class RuntimeCapabilities(FabricMapping): Attributes: service: Whether long-lived service handles are supported. streaming: Whether event streaming is supported. + health: Whether bounded runtime health inspection is supported. updates: Whether runtime configuration updates are supported. cancellation: Whether in-flight cancellation is supported. metadata: Additional capability details. @@ -1365,6 +1366,7 @@ class RuntimeCapabilities(FabricMapping): service: bool streaming: bool + health: bool updates: bool cancellation: bool metadata: Mapping[str, Any] @@ -1374,6 +1376,7 @@ class RuntimeCapabilities(FabricMapping): "streaming", "updates", "cancellation", + "health", "metadata", } ) @@ -1663,6 +1666,126 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: return data +class HealthCheck(FabricMapping): + """One timestamped runtime or adapter health observation. + + Attributes: + name: Stable, namespaced check name. + status: One of ``ok``, ``failed``, ``unknown``, or ``unsupported``. + reason_code: Stable machine-readable reason. + observed_at_millis: Unix timestamp when the evidence was observed. + age_millis: Age of the evidence when the report was assembled. + message: Optional human-readable diagnostic detail. + metadata: Additional non-sensitive check metadata. + """ + + name: str + status: str + reason_code: str + observed_at_millis: int + age_millis: int + message: str | None + metadata: Mapping[str, Any] + _fields = frozenset( + { + "name", + "status", + "reason_code", + "observed_at_millis", + "age_millis", + "message", + "metadata", + } + ) + _json_fields = frozenset({"metadata"}) + _omit_if_empty = frozenset({"metadata"}) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + data["name"] = _required_text(data.get("name"), "health check name") + data["status"] = _required_text(data.get("status"), "health check status") + if data["status"] not in {"ok", "failed", "unknown", "unsupported"}: + raise FabricConfigError("health check status is invalid") + data["reason_code"] = _required_text( + data.get("reason_code"), "health check reason code" + ) + for field in ("observed_at_millis", "age_millis"): + value = data.get(field) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise FabricConfigError( + f"{field.replace('_', ' ')} must be a nonnegative integer" + ) + message = data.get("message") + if message is not None and not isinstance(message, str): + raise FabricConfigError("health check message must be a string or null") + data["message"] = message + data["metadata"] = _mapping(data.get("metadata", {}), "health check metadata") + return data + + +class RuntimeHealth(FabricMapping): + """Bounded health report for one started runtime. + + Attributes: + runtime_id: Runtime represented by the report. + checked_at_millis: Unix timestamp when the report completed. + duration_millis: Total probe duration. + liveness: Adapter-host liveness. + activity: Current invocation and shutdown activity. + readiness: Whether the runtime can currently accept work. + reason_code: Stable reason for the readiness decision. + checks: Ordered common and adapter-specific observations. + """ + + runtime_id: str + checked_at_millis: int + duration_millis: int + liveness: str + activity: str + readiness: str + reason_code: str + checks: Sequence[HealthCheck] + _fields = frozenset( + { + "runtime_id", + "checked_at_millis", + "duration_millis", + "liveness", + "activity", + "readiness", + "reason_code", + "checks", + } + ) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + data["runtime_id"] = _required_text(data.get("runtime_id"), "runtime id") + for field in ("checked_at_millis", "duration_millis"): + value = data.get(field) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise FabricConfigError( + f"{field.replace('_', ' ')} must be a nonnegative integer" + ) + allowed = { + "liveness": {"responsive", "unresponsive", "exited", "unknown"}, + "activity": {"idle", "busy", "stopping", "unknown"}, + "readiness": {"ready", "not_ready", "unknown"}, + } + for field, values in allowed.items(): + data[field] = _required_text(data.get(field), field) + if data[field] not in values: + raise FabricConfigError(f"runtime health {field} is invalid") + data["reason_code"] = _required_text( + data.get("reason_code"), "runtime health reason code" + ) + checks = data.get("checks", []) + if not isinstance(checks, Sequence) or isinstance(checks, (str, bytes)): + raise FabricConfigError("runtime health checks must be a sequence") + data["checks"] = tuple(HealthCheck.from_mapping(check) for check in checks) + return data + + class RunOutput(FabricMapping): """Normalized adapter output. diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index ae8e2e97d..99133f351 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -73,6 +73,9 @@ translation: when the adapter implements native OpenAI Chat Completions streaming through `invoke_openai_stream`. Relay-backed ATOF streaming is independent and does not require this capability. +- Set `capabilities.health` only when the selected local host implements the + authenticated health-control protocol. The maintained Python and TypeScript + common hosts implement it. An adapter-specific health hook is optional. If the adapter loads registered targets, list their types in `target_types`. Create one `*.fabric-target.json` per target. The target record owns its @@ -207,6 +210,12 @@ Return `AgentRunStatus.FAILED` with an `AgentRunError` when the target completes with a failed outcome. Raise an exception when the adapter cannot produce a normalized terminal result. +The common host also serves health on an authenticated loopback endpoint that +does not share the ordered lifecycle channel. Optional health hooks must respect +`request.timeout_millis`, avoid inference or other billable probes, avoid +runtime mutation, return stable reason codes, and never expose credentials. +Treat hook failure and timeout as health data; do not fail the runtime. + ### Support Warm Session Continuation When later invocations must use earlier conversation state, retain that state diff --git a/skills/nemo-fabric-integrate/SKILL.md b/skills/nemo-fabric-integrate/SKILL.md index 05fcd17b9..d82b74676 100644 --- a/skills/nemo-fabric-integrate/SKILL.md +++ b/skills/nemo-fabric-integrate/SKILL.md @@ -184,6 +184,10 @@ Pick the smallest lifecycle the consumer needs: (`stop()` can raise `FabricRuntimeError`; see Consume Results And Handle Errors). A runtime accepts one active invocation at a time; overlapping calls raise `FabricStateError`. +- **Runtime health** — bounded liveness and readiness data for a started local + runtime. Call `await runtime.check_health(timeout_seconds=3.0)`. Treat + timeouts, unsupported checks, and not-ready results as `RuntimeHealth` data; + the check does not invoke the agent, stop the runtime, or mark it failed. - **Native OpenAI stream** — adapter-native OpenAI Chat Completions chunks plus a separate terminal normalized result. Check `runtime.supports_openai_streaming`, call diff --git a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md index 29861af58..4d293eaff 100644 --- a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md +++ b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md @@ -40,6 +40,7 @@ The following table lists the `Runtime` members for driving a stateful runtime. | `invoke(*, input=... \| request=...)` | Yes | One turn on an active runtime. One active invocation at a time; overlap raises `FabricStateError`. | | `invoke_openai_stream(*, input=... \| request=...)` | No | Start exactly one descriptor-gated native invocation and return an async `OpenAIInvokeStream` of `chat.completion.chunk` mappings. Await `stream.result()` for the separate terminal `RunResult`. | | `invoke_stream(*, input=... \| request=...)` | No | Start one NeMo Relay turn and return an async `InvokeStream` of raw ATOF records. Await `stream.result()` for the terminal `RunResult`. | +| `check_health(*, timeout_seconds=3.0)` | Yes | Return bounded `RuntimeHealth` data without invoking, stopping, or failing the runtime. Negative health and timeouts are report data. | | `stop()` | Yes | Stop the runtime. Called automatically by `async with`. | | `status` | No | `RuntimeStatus`: `ACTIVE`, `STOPPED`, or `FAILED`. | | `supports_openai_streaming` | No | `True` when the selected descriptor declares `capabilities.streaming` for native OpenAI Chat Completions chunks. | diff --git a/tests/adapter_contract/test_health.py b/tests/adapter_contract/test_health.py new file mode 100644 index 000000000..cfa21011e --- /dev/null +++ b/tests/adapter_contract/test_health.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for optional adapter health contracts.""" + +import pytest + +from nemo_fabric_adapter_contract.codec import ContractValidationError +from nemo_fabric_adapter_contract.models import AdapterHealthRequest +from nemo_fabric_adapter_contract.models import AdapterHealthResult +from nemo_fabric_adapter_contract.models import AdapterReadiness +from nemo_fabric_adapter_contract.models import HealthCheck +from nemo_fabric_adapter_contract.models import HealthCheckStatus +from nemo_fabric_adapter_contract.models import RuntimeReadiness + + +def test_adapter_health_models_round_trip(): + result = AdapterHealthResult( + readiness=AdapterReadiness( + state=RuntimeReadiness.READY, + reason_code="ready", + ), + checks=[ + HealthCheck( + name="dependency.cache", + status=HealthCheckStatus.OK, + reason_code="cache_ready", + observed_at_millis=10, + age_millis=0, + metadata={"region": "local"}, + ) + ], + ) + + restored = AdapterHealthResult.from_mapping(result.to_mapping()) + + assert restored == result + assert restored.checks[0].status is HealthCheckStatus.OK + + +def test_adapter_health_request_requires_positive_timeout(): + with pytest.raises(ContractValidationError, match="greater than zero"): + AdapterHealthRequest(runtime_id="runtime-1", timeout_millis=0) diff --git a/tests/adapters/test_mini_swe_agent.py b/tests/adapters/test_mini_swe_agent.py index 2ae4e6d4d..5bc97dfe7 100644 --- a/tests/adapters/test_mini_swe_agent.py +++ b/tests/adapters/test_mini_swe_agent.py @@ -275,6 +275,7 @@ def test_mini_swe_agent_descriptor_is_narrow_and_versioned(): assert descriptor["capabilities"] == { "service": False, "streaming": False, + "health": True, "updates": False, "cancellation": False, } diff --git a/tests/adapters/test_nooa_adapter.py b/tests/adapters/test_nooa_adapter.py index fcfc9854a..58044a4c1 100644 --- a/tests/adapters/test_nooa_adapter.py +++ b/tests/adapters/test_nooa_adapter.py @@ -222,6 +222,7 @@ def test_descriptor_and_registered_target_declare_the_shared_boundary(): "cancellation": False, "service": False, "streaming": False, + "health": True, "updates": False, } assert descriptor["telemetry"] == { diff --git a/tests/adapters/test_pi_adapter.py b/tests/adapters/test_pi_adapter.py index ef4c48cd1..9130d91f0 100644 --- a/tests/adapters/test_pi_adapter.py +++ b/tests/adapters/test_pi_adapter.py @@ -55,6 +55,7 @@ def test_pi_descriptor_declares_the_supported_surface(): assert descriptor["config"]["system_instruction_modes"] == ["replace"] assert descriptor["capabilities"] == { "streaming": False, + "health": True, "cancellation": False, "updates": False, "service": False, diff --git a/tests/python/test_adapter_lifecycle_health.py b/tests/python/test_adapter_lifecycle_health.py new file mode 100644 index 000000000..9330ed8a7 --- /dev/null +++ b/tests/python/test_adapter_lifecycle_health.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Health-control tests for the shared Python adapter host.""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any + +from nemo_fabric_adapter_contract.models import AdapterHealthRequest +from nemo_fabric_adapters.common.lifecycle import _adapter_health +from nemo_fabric_adapters.common.lifecycle import _close_health_server +from nemo_fabric_adapters.common.lifecycle import _HostState +from nemo_fabric_adapters.common.lifecycle import _start_health_server + + +class _Runtime: + async def start(self, payload: dict[str, Any]): + del payload + + async def invoke(self, request: Any, context: Any): + del request, context + raise AssertionError("not used") + + async def stop(self): + pass + + +class _SlowHealthRuntime(_Runtime): + async def health(self, request: AdapterHealthRequest): + del request + await asyncio.sleep(1) + raise AssertionError("health hook should time out") + + +async def test_python_health_control_reports_busy_without_lifecycle_channel(): + state = _HostState( + runtime=_Runtime(), + runtime_id="runtime-1", + invoking=True, + ) + output = await _start_health_server(state) + control = output["health_control"] + try: + reader, writer = await asyncio.open_connection( + control["host"], control["port"] + ) + writer.write( + json.dumps( + { + "protocol_version": control["protocol_version"], + "token": control["token"], + "runtime_id": "runtime-1", + "timeout_millis": 1_000, + } + ).encode() + + b"\n" + ) + await writer.drain() + response = json.loads(await reader.readline()) + writer.close() + await writer.wait_closed() + finally: + await _close_health_server(state) + + assert response["runtime_id"] == "runtime-1" + assert response["result"]["readiness"] == { + "state": "not_ready", + "reason_code": "invocation_in_progress", + } + assert [check["status"] for check in response["result"]["checks"]] == [ + "unsupported", + "unsupported", + ] + + +async def test_python_adapter_health_hook_timeout_is_data(): + state = _HostState(runtime=_SlowHealthRuntime(), runtime_id="runtime-1") + + result = await _adapter_health( + state, + AdapterHealthRequest(runtime_id="runtime-1", timeout_millis=51), + ) + + assert state.failed is False + assert result.readiness is not None + assert result.readiness.state.value == "ready" + assert result.checks[-1].reason_code == "adapter_health_timed_out" diff --git a/tests/python/test_runtime.py b/tests/python/test_runtime.py index 6f4aefa67..ffbc8d10e 100644 --- a/tests/python/test_runtime.py +++ b/tests/python/test_runtime.py @@ -23,11 +23,13 @@ FabricRuntimeError, FabricStateError, HarnessConfig, + HealthCheck, MetadataConfig, RunRequest, RunResult, RuntimeConfig, Runtime, + RuntimeHealth, RuntimeStatus, ) from nemo_fabric import client as client_mod @@ -77,6 +79,32 @@ def _runtime(runtime_id: str = "runtime-1") -> dict[str, Any]: } +def _health( + runtime_id: str = "runtime-1", + *, + readiness: str = "ready", + reason_code: str = "ready", +) -> dict[str, Any]: + return { + "runtime_id": runtime_id, + "checked_at_millis": 1_700_000_000_000, + "duration_millis": 2, + "liveness": "responsive", + "activity": "idle", + "readiness": readiness, + "reason_code": reason_code, + "checks": [ + { + "name": "adapter.control", + "status": "ok", + "reason_code": "probe_succeeded", + "observed_at_millis": 1_700_000_000_000, + "age_millis": 0, + } + ], + } + + def _config() -> FabricConfig: return FabricConfig( metadata=MetadataConfig(name="demo"), @@ -127,6 +155,11 @@ def invoke(plan_json: str, runtime_json: str, request_json: str) -> str: ) mock_native.invoke_runtime.side_effect = invoke + mock_native.check_runtime_health.side_effect = ( + lambda _plan, runtime_json, _timeout: json.dumps( + _health(json.loads(runtime_json)["runtime_id"]) + ) + ) mock_native.stop_runtime.return_value = json.dumps([]) return mock_native @@ -220,6 +253,58 @@ async def test_runtime_reuses_runtime_and_orders_turns(mock_native: MagicMock): assert len(runtime.invocations) == 2 +async def test_runtime_health_returns_typed_report(mock_native: MagicMock): + runtime = _runtime_wrapper(mock_native) + + health = await runtime.check_health(timeout_seconds=0.125) + + assert isinstance(health, RuntimeHealth) + assert health.runtime_id == "runtime-1" + assert health.liveness == "responsive" + assert health.readiness == "ready" + assert isinstance(health.checks[0], HealthCheck) + assert health.checks[0].reason_code == "probe_succeeded" + assert mock_native.check_runtime_health.call_args.args[2] == 125 + assert runtime.status is RuntimeStatus.ACTIVE + + +async def test_negative_runtime_health_is_data(mock_native: MagicMock): + mock_native.check_runtime_health.return_value = json.dumps( + _health(readiness="unknown", reason_code="probe_timed_out") + ) + mock_native.check_runtime_health.side_effect = None + runtime = _runtime_wrapper(mock_native) + + health = await runtime.check_health() + + assert health.readiness == "unknown" + assert health.reason_code == "probe_timed_out" + assert runtime.status is RuntimeStatus.ACTIVE + + +@pytest.mark.parametrize("timeout", [0, -1, float("inf"), float("nan"), True]) +async def test_runtime_health_rejects_invalid_timeout( + mock_native: MagicMock, + timeout: Any, +): + runtime = _runtime_wrapper(mock_native) + + with pytest.raises(FabricConfigError, match="positive and finite"): + await runtime.check_health(timeout_seconds=timeout) + + mock_native.check_runtime_health.assert_not_called() + + +async def test_runtime_health_rejects_stopped_runtime(mock_native: MagicMock): + runtime = _runtime_wrapper(mock_native) + await runtime.stop() + + with pytest.raises(FabricStateError, match="stopped"): + await runtime.check_health() + + mock_native.check_runtime_health.assert_not_called() + + async def test_native_invoke_failure_marks_runtime_failed(mock_native: MagicMock): mock_native.invoke_runtime.side_effect = RuntimeError("invoke failed") runtime = _runtime_wrapper(mock_native) diff --git a/tests/python/test_sdk_contract.py b/tests/python/test_sdk_contract.py index f7df930b7..f0ace9dac 100644 --- a/tests/python/test_sdk_contract.py +++ b/tests/python/test_sdk_contract.py @@ -28,6 +28,7 @@ from nemo_fabric import FabricRuntimeError from nemo_fabric import FabricStateError from nemo_fabric import HarnessConfig +from nemo_fabric import HealthCheck from nemo_fabric import InstructionConfig from nemo_fabric import InstructionsConfig from nemo_fabric import McpAuthenticationConfig @@ -53,6 +54,7 @@ from nemo_fabric import RuntimeCapabilities from nemo_fabric import RuntimeConfig from nemo_fabric import RuntimeHandle +from nemo_fabric import RuntimeHealth from nemo_fabric import SkillConfig from nemo_fabric import TelemetryConfig from nemo_fabric import ToolDefinitionConfig @@ -1490,6 +1492,7 @@ def test_inspection_models_are_typed_read_only_mappings(): assert plan.adapter.harness == "hermes" assert "harness_type" not in plan.adapter assert plan.adapter.extra_fields["future"] == "value" + assert plan.capabilities.health is False assert plan.capabilities.extra_fields["future_capability"] == "declared" resolved = plan.to_mapping() plan.config.metadata.name = "mutated" @@ -1599,6 +1602,40 @@ def test_runtime_capabilities_reject_non_boolean_values(): RuntimeCapabilities.from_mapping({"streaming": "false"}) +def test_runtime_health_models_are_typed_and_validate_states(): + report = RuntimeHealth.from_mapping( + { + "runtime_id": "runtime-1", + "checked_at_millis": 10, + "duration_millis": 2, + "liveness": "responsive", + "activity": "idle", + "readiness": "ready", + "reason_code": "ready", + "checks": [ + { + "name": "adapter.process", + "status": "ok", + "reason_code": "process_running", + "observed_at_millis": 9, + "age_millis": 1, + } + ], + } + ) + + assert isinstance(report.checks[0], HealthCheck) + assert report.to_mapping()["checks"][0]["status"] == "ok" + + with pytest.raises(FabricConfigError, match="liveness"): + RuntimeHealth.from_mapping( + { + **report.to_mapping(), + "liveness": "alive", + } + ) + + def test_doctor_report_and_errors_expose_typed_contract_fields(): report = DoctorReport.from_mapping( { From 53db297004987339ce5d66cacf0ad0c35e6ae28d Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 15 Sep 2026 16:44:28 -0700 Subject: [PATCH 2/3] fix: harden runtime health checks Signed-off-by: Yuchen Zhang --- .../nemo_fabric_adapter_contract/models.py | 2 +- .../schemas/adapter-health-result.schema.json | 2 +- .../python/claude/claude.fabric-adapter.json | 3 + .../python/codex/codex.fabric-adapter.json | 3 + adapters/python/common/README.md | 15 +- .../nemo_fabric_adapters/common/lifecycle.py | 25 +- .../deepagents/deepagents.fabric-adapter.json | 3 + .../python/hermes/hermes.fabric-adapter.json | 3 + adapters/typescript/common/README.md | 17 +- adapters/typescript/common/src/lifecycle.ts | 38 ++- .../typescript/common/test/lifecycle.test.mjs | 161 +++++++++- .../claude/claude.fabric-adapter.json | 3 + .../adapters/codex/codex.fabric-adapter.json | 3 + .../deepagents/deepagents.fabric-adapter.json | 3 + .../hermes/hermes.fabric-adapter.json | 3 + crates/fabric-core/src/runtime.rs | 302 ++++++++++++++---- .../runtime/struct-healthcheck.mdx | 4 +- .../runtime/struct-runtimehealth.mdx | 6 +- docs/sdk/python.mdx | 7 + .../claude/claude.fabric-adapter.json | 3 + .../hermes/hermes.fabric-adapter.json | 3 + .../adapter-health-result.schema.json | 2 +- schemas/sdk/runtime-health.schema.json | 6 +- skills/nemo-fabric-build-adapter/SKILL.md | 3 + tests/adapters/test_claude_adapter.py | 1 + tests/adapters/test_codex_adapter.py | 1 + tests/adapters/test_deepagents.py | 1 + tests/adapters/test_hermes_adapter.py | 1 + tests/python/test_adapter_lifecycle_health.py | 150 +++++++-- 29 files changed, 627 insertions(+), 147 deletions(-) diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py index 24b647e2b..2bec6cea9 100644 --- a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py @@ -566,7 +566,7 @@ class HealthCheck(ContractModel): def _validate(self) -> None: _nonblank(self.name, "name") _nonblank(self.reason_code, "reason_code") - _bounded_int(self.observed_at_millis, "observed_at_millis", (1 << 128) - 1) + _bounded_int(self.observed_at_millis, "observed_at_millis", (1 << 64) - 1) _bounded_int(self.age_millis, "age_millis", (1 << 64) - 1) if self.message is not None: _nonblank(self.message, "message") diff --git a/adapter-contract/typescript/schemas/adapter-health-result.schema.json b/adapter-contract/typescript/schemas/adapter-health-result.schema.json index 7a998afe0..07c7ab303 100644 --- a/adapter-contract/typescript/schemas/adapter-health-result.schema.json +++ b/adapter-contract/typescript/schemas/adapter-health-result.schema.json @@ -53,7 +53,7 @@ }, "observed_at_millis": { "description": "Unix timestamp in milliseconds when the evidence was observed.", - "format": "uint128", + "format": "uint64", "minimum": 0, "type": "integer" }, diff --git a/adapters/python/claude/claude.fabric-adapter.json b/adapters/python/claude/claude.fabric-adapter.json index bc761d0ce..b507daf17 100644 --- a/adapters/python/claude/claude.fabric-adapter.json +++ b/adapters/python/claude/claude.fabric-adapter.json @@ -90,6 +90,9 @@ ], "system_instruction_modes": ["replace", "append"] }, + "capabilities": { + "health": true + }, "telemetry": { "providers": { "relay": { diff --git a/adapters/python/codex/codex.fabric-adapter.json b/adapters/python/codex/codex.fabric-adapter.json index 03819ee12..acc29cc46 100644 --- a/adapters/python/codex/codex.fabric-adapter.json +++ b/adapters/python/codex/codex.fabric-adapter.json @@ -107,6 +107,9 @@ ], "system_instruction_modes": ["replace"] }, + "capabilities": { + "health": true + }, "telemetry": { "providers": { "relay": { diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index a7a33d9c6..a5a7ce4b5 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -118,13 +118,14 @@ runtime. ## Runtime Health -The common host starts an authenticated loopback health endpoint and returns -its connection metadata to NeMo Fabric during lifecycle startup. The endpoint -is independent of the ordered lifecycle channel, so it responds while -`invoke` is running. Do not log or persist the endpoint token. - -The host always reports lifecycle activity and marks inference probing as -unsupported. An adapter can optionally implement +When the adapter descriptor declares `capabilities.health`, the common host +starts an authenticated loopback health endpoint and returns its connection +metadata to NeMo Fabric during lifecycle startup. The endpoint is independent +of the ordered lifecycle channel, so it responds while `invoke` is running. Do +not log or persist the endpoint token. + +The host reports lifecycle activity and marks inference probing as unsupported. +An adapter can optionally implement the `AdapterHealthRuntime` protocol method `async health(request) -> AdapterHealthResult` to add fast, adapter-owned checks. Respect `request.timeout_millis`; do not invoke the agent, contact a model merely to test availability, consume quota, or mutate runtime state. A diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/lifecycle.py b/adapters/python/common/src/nemo_fabric_adapters/common/lifecycle.py index 5d06da5b4..fc08880a0 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/lifecycle.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/lifecycle.py @@ -54,6 +54,13 @@ async def stop(self) -> None: """Release all resources owned by the runtime.""" +class AdapterHealthRuntime(Protocol): + """Optional adapter extension for runtime-specific health evidence.""" + + async def health(self, request: AdapterHealthRequest) -> AdapterHealthResult: + """Return adapter-owned health observations within the request budget.""" + + RuntimeFactory = Callable[[], AdapterRuntime] ConfigLoader = Callable[[Any], Any] OpenAIChunkEmitter = Callable[[Mapping[str, Any]], Awaitable[None]] @@ -418,9 +425,6 @@ def clear(self) -> None: self.failed = False self.invoking = False self.stopping = False - self.health_server = None - self.health_token = None - self.health_tasks.clear() def _error( @@ -490,10 +494,7 @@ def _invocation_environment(payload: dict[str, Any]) -> Iterator[None]: async def _adapter_call(operation: str, call: Callable[[], Awaitable[Any]]) -> Any: try: - # Protocol stdout is reserved for exactly one JSON response per line. - # Keep incidental adapter and library output as host diagnostics. - with redirect_stdout(sys.stderr): - return await call() + return await call() except LifecycleError as error: raise _AdapterCallError( error.code, @@ -677,9 +678,7 @@ async def _adapter_health( readiness = AdapterReadiness( state=( - RuntimeReadiness.NOT_READY - if state.invoking - else RuntimeReadiness.READY + RuntimeReadiness.NOT_READY if state.invoking else RuntimeReadiness.READY ), reason_code=("invocation_in_progress" if state.invoking else "ready"), ) @@ -735,7 +734,7 @@ async def _adapter_health( ) else: checks.extend(result.checks) - if not state.invoking and result.readiness is not None: + if result.readiness is not None: readiness = result.readiness return AdapterHealthResult(readiness=readiness, checks=checks) @@ -808,6 +807,8 @@ async def _handle_start( state.runtime = candidate state.runtime_id = message_runtime_id state.failed = False + if payload.get("health_enabled") is not True: + return _response("start") try: output = await _start_health_server(state) except Exception: @@ -914,9 +915,9 @@ async def _handle_stop( ) -> dict[str, Any]: state.stopping = True try: - await _close_health_server(state) await _adapter_call("stop", runtime.stop) finally: + await _close_health_server(state) state.clear() return _response("stop") diff --git a/adapters/python/deepagents/deepagents.fabric-adapter.json b/adapters/python/deepagents/deepagents.fabric-adapter.json index 12eb69898..1297849bb 100644 --- a/adapters/python/deepagents/deepagents.fabric-adapter.json +++ b/adapters/python/deepagents/deepagents.fabric-adapter.json @@ -211,6 +211,9 @@ ], "system_instruction_modes": ["replace"] }, + "capabilities": { + "health": true + }, "telemetry": { "providers": { "relay": { diff --git a/adapters/python/hermes/hermes.fabric-adapter.json b/adapters/python/hermes/hermes.fabric-adapter.json index 7ae06401d..1f528ff2e 100644 --- a/adapters/python/hermes/hermes.fabric-adapter.json +++ b/adapters/python/hermes/hermes.fabric-adapter.json @@ -87,6 +87,9 @@ ], "system_instruction_modes": ["replace"] }, + "capabilities": { + "health": true + }, "telemetry": { "providers": { "relay": { diff --git a/adapters/typescript/common/README.md b/adapters/typescript/common/README.md index 4d3310377..bd4f64cb8 100644 --- a/adapters/typescript/common/README.md +++ b/adapters/typescript/common/README.md @@ -26,14 +26,15 @@ await serve(() => new MyAdapterRuntime()); The factory may return a runtime directly or resolve one asynchronously. The host begins reading lifecycle input before it awaits asynchronous adapter setup. -The host also exposes an authenticated loopback health endpoint that is -independent of ordered lifecycle traffic. It reports idle or busy activity even -when the runtime omits the optional -`health(request): Promise` method. Implement that method -only for fast adapter-owned checks, respect `request.timeout_millis`, and do not -invoke the agent or probe an inference model. Missing, failed, and timed-out -hooks become structured health data without failing the runtime. Do not log or -persist health-control credentials. +When the adapter descriptor declares `capabilities.health`, the host exposes an +authenticated loopback health endpoint that is independent of ordered lifecycle +traffic. It reports idle or busy activity even when the runtime omits the +optional `health(request, signal): Promise` method. +Implement that method only for fast adapter-owned checks, respect +`request.timeout_millis` and the optional abort signal, and do not invoke the +agent or probe an inference model. Missing, failed, and timed-out hooks become +structured health data without failing the runtime. Do not log or persist +health-control credentials. This package is intended to be published as the shared runtime dependency for TypeScript adapters. Its public API will be versioned independently from the diff --git a/adapters/typescript/common/src/lifecycle.ts b/adapters/typescript/common/src/lifecycle.ts index 1964bb758..54d902a35 100644 --- a/adapters/typescript/common/src/lifecycle.ts +++ b/adapters/typescript/common/src/lifecycle.ts @@ -28,6 +28,7 @@ export interface AdapterStartInput { baseDir: string; config: AgentConfig; runtimeContext: RuntimeContext; + healthEnabled: boolean; capabilityPlan?: JsonObject; telemetryPlan?: JsonObject; } @@ -35,7 +36,7 @@ export interface AdapterStartInput { export interface AdapterRuntime { start(input: AdapterStartInput): Promise; invoke(request: AgentRunRequest, context: RuntimeContext): Promise; - health?(request: AdapterHealthRequest): Promise; + health?(request: AdapterHealthRequest, signal?: AbortSignal): Promise; stop(): Promise; } @@ -107,10 +108,6 @@ ajv.addFormat("uint64", { type: "number", validate: (value: number) => Number.isSafeInteger(value) && value >= 0, }); -ajv.addFormat("uint128", { - type: "number", - validate: (value: number) => Number.isSafeInteger(value) && value >= 0, -}); ajv.addFormat("double", { type: "number", validate: (value: number) => Number.isFinite(value), @@ -205,6 +202,7 @@ function decodeStart(payload: Record): AdapterStartInput { baseDir: payload.base_dir, config, runtimeContext: context, + healthEnabled: payload.health_enabled === true, capabilityPlan: capabilityPlan as JsonObject | undefined, telemetryPlan: telemetryPlan as JsonObject | undefined, }; @@ -324,7 +322,8 @@ async function startHealthServer(state: HostState): Promise { function readHealthLine(socket: Socket): Promise { return new Promise((resolve, reject) => { - let encoded = ""; + const chunks: Buffer[] = []; + let encodedLength = 0; const cleanup = (): void => { socket.off("data", onData); socket.off("error", onError); @@ -332,16 +331,18 @@ function readHealthLine(socket: Socket): Promise { socket.off("timeout", onTimeout); }; const onData = (chunk: Buffer): void => { - encoded += chunk.toString("utf8"); - if (Buffer.byteLength(encoded) > HEALTH_REQUEST_LIMIT) { + const newline = chunk.indexOf(0x0a); + const recordChunk = newline >= 0 ? chunk.subarray(0, newline) : chunk; + encodedLength += recordChunk.length; + if (encodedLength > HEALTH_REQUEST_LIMIT) { cleanup(); reject(new Error("health request exceeds the size limit")); return; } - const newline = encoded.indexOf("\n"); + chunks.push(recordChunk); if (newline >= 0) { cleanup(); - resolve(encoded.slice(0, newline)); + resolve(Buffer.concat(chunks, encodedLength).toString("utf8")); } }; const onError = (error: Error): void => { @@ -452,11 +453,15 @@ async function adapterHealth( return { readiness, checks }; } let timeout: ReturnType | undefined; + const controller = new AbortController(); try { const result = await Promise.race([ - callAdapter("health", () => hook.call(runtime, request)), + callAdapter("health", () => hook.call(runtime, request, controller.signal)), new Promise((_resolve, reject) => { - timeout = setTimeout(() => reject(new HealthHookTimeout()), hookBudgetMillis); + timeout = setTimeout(() => { + controller.abort(); + reject(new HealthHookTimeout()); + }, hookBudgetMillis); }), ]); validate( @@ -466,7 +471,7 @@ async function adapterHealth( "Adapter health hook returned an invalid result", ); checks.push(...(result.checks ?? [])); - if (!state.invoking && result.readiness !== undefined && result.readiness !== null) { + if (result.readiness !== undefined && result.readiness !== null) { readiness = result.readiness; } } catch (error) { @@ -522,13 +527,14 @@ async function dispatch( try { candidate = await callAdapter("start", factory); const active = candidate; - await callAdapter("start", () => active.start(decodeStart(request.payload))); + const startInput = decodeStart(request.payload); + await callAdapter("start", () => active.start(startInput)); state.runtime = active; state.runtimeId = messageRuntimeId; state.failed = false; state.invoking = false; state.stopping = false; - const output = await startHealthServer(state); + const output = startInput.healthEnabled ? await startHealthServer(state) : null; return success("start", output); } catch (error) { await closeHealthServer(state); @@ -551,7 +557,6 @@ async function dispatch( if (request.operation === "stop") { const active = state.runtime; state.stopping = true; - await closeHealthServer(state); try { await callAdapter("stop", () => active.stop()); state.runtime = undefined; @@ -559,6 +564,7 @@ async function dispatch( state.failed = false; return success("stop"); } finally { + await closeHealthServer(state); state.stopping = false; } } diff --git a/adapters/typescript/common/test/lifecycle.test.mjs b/adapters/typescript/common/test/lifecycle.test.mjs index c549446b1..87cbbb665 100644 --- a/adapters/typescript/common/test/lifecycle.test.mjs +++ b/adapters/typescript/common/test/lifecycle.test.mjs @@ -23,13 +23,14 @@ function context(runtimeId, invocationId = "invocation-1") { }; } -function start(runtimeId) { +function start(runtimeId, healthEnabled = true) { return { operation: "start", payload: { agent_name: "test-agent", base_dir: "/tmp", config: {}, + health_enabled: healthEnabled, runtime_context: context(runtimeId, "start"), }, }; @@ -138,6 +139,11 @@ test("serves health independently while an invocation is busy", async () => { await blocked; return { status: "succeeded", output: null }; }, + async health() { + return { + readiness: { state: "ready", reason_code: "concurrent_invocations_supported" }, + }; + }, async stop() {}, }; const serving = serve(() => runtime, { input, output, diagnostics }); @@ -149,13 +155,12 @@ test("serves health independently while an invocation is busy", async () => { await started; const healthResponse = await checkHealth(control, "runtime-1"); - assert.equal(healthResponse.result.readiness.state, "not_ready"); - assert.equal(healthResponse.result.readiness.reason_code, "invocation_in_progress"); + assert.equal(healthResponse.result.readiness.state, "ready"); + assert.equal(healthResponse.result.readiness.reason_code, "concurrent_invocations_supported"); assert.deepEqual( healthResponse.result.checks.map((check) => [check.name, check.status]), [ ["dependency.inference", "unsupported"], - ["adapter.health", "unsupported"], ], ); @@ -167,6 +172,154 @@ test("serves health independently while an invocation is busy", async () => { await serving; }); +test("reports stop in progress while adapter cleanup is running", async () => { + const input = new PassThrough(); + const output = new PassThrough(); + const diagnostics = new PassThrough(); + const nextResponse = responseReader(output); + let releaseStop; + let stopStarted; + const started = new Promise((resolve) => { + stopStarted = resolve; + }); + const blocked = new Promise((resolve) => { + releaseStop = resolve; + }); + const runtime = { + async start() {}, + async invoke() { + return { status: "succeeded", output: null }; + }, + async stop() { + stopStarted(); + await blocked; + }, + }; + const serving = serve(() => runtime, { input, output, diagnostics }); + input.write(`${JSON.stringify(start("runtime-1"))}\n`); + const startResponse = await nextResponse(); + const control = startResponse.outcome.output.health_control; + + input.write(`${JSON.stringify(stop("runtime-1"))}\n`); + await started; + const healthResponse = await checkHealth(control, "runtime-1"); + + assert.deepEqual(healthResponse.result.readiness, { + state: "not_ready", + reason_code: "stop_in_progress", + }); + releaseStop(); + await nextResponse(); + input.end(); + await serving; +}); + +test("aborts a timed-out adapter health hook", async () => { + const input = new PassThrough(); + const output = new PassThrough(); + const diagnostics = new PassThrough(); + const nextResponse = responseReader(output); + let aborted = false; + const runtime = { + async start() {}, + async invoke() { + return { status: "succeeded", output: null }; + }, + async health(_request, signal) { + await new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => { + aborted = true; + reject(new Error("aborted")); + }, { once: true }); + }); + }, + async stop() {}, + }; + const serving = serve(() => runtime, { input, output, diagnostics }); + input.write(`${JSON.stringify(start("runtime-1"))}\n`); + const startResponse = await nextResponse(); + + const healthResponse = await checkHealth( + startResponse.outcome.output.health_control, + "runtime-1", + 60, + ); + + assert.equal(aborted, true); + assert.equal(healthResponse.result.checks.at(-1).reason_code, "adapter_health_timed_out"); + input.write(`${JSON.stringify(stop("runtime-1"))}\n`); + await nextResponse(); + input.end(); + await serving; +}); + +test("decodes a health request after a UTF-8 code point is split across chunks", async () => { + const input = new PassThrough(); + const output = new PassThrough(); + const diagnostics = new PassThrough(); + const nextResponse = responseReader(output); + const runtimeId = "runtime-é"; + const runtime = { + async start() {}, + async invoke() { + return { status: "succeeded", output: null }; + }, + async stop() {}, + }; + const serving = serve(() => runtime, { input, output, diagnostics }); + input.write(`${JSON.stringify(start(runtimeId))}\n`); + const startResponse = await nextResponse(); + const control = startResponse.outcome.output.health_control; + const socket = connect(control.port, control.host); + const response = new Promise((resolve, reject) => { + let encoded = ""; + socket.on("data", (chunk) => { + encoded += chunk.toString("utf8"); + const newline = encoded.indexOf("\n"); + if (newline >= 0) { + resolve(JSON.parse(encoded.slice(0, newline))); + } + }); + socket.once("error", reject); + }); + await new Promise((resolve) => socket.once("connect", resolve)); + const request = Buffer.from(`${JSON.stringify({ + protocol_version: "fabric.health/v1alpha1", + token: control.token, + runtime_id: runtimeId, + timeout_millis: 1000, + })}\n`); + const codePoint = Buffer.from("é"); + const splitAt = request.indexOf(codePoint) + 1; + socket.write(request.subarray(0, splitAt)); + socket.write(request.subarray(splitAt)); + + const healthResponse = await response; + + assert.equal(healthResponse.runtime_id, runtimeId); + input.write(`${JSON.stringify(stop(runtimeId))}\n`); + await nextResponse(); + input.end(); + await serving; +}); + +test("does not start health control when the capability is disabled", async () => { + const runtime = { + async start() {}, + async invoke() { + return { status: "succeeded", output: null }; + }, + async stop() {}, + }; + + const responses = await exchange( + () => runtime, + [start("runtime-1", false), stop("runtime-1")], + ); + + assert.equal(responses[0].outcome.output, null); +}); + test("serves two ordered invocations and stops one runtime", async () => { const calls = []; const runtime = { diff --git a/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json b/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json index bc761d0ce..b507daf17 100644 --- a/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json @@ -90,6 +90,9 @@ ], "system_instruction_modes": ["replace", "append"] }, + "capabilities": { + "health": true + }, "telemetry": { "providers": { "relay": { diff --git a/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json b/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json index 03819ee12..acc29cc46 100644 --- a/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json @@ -107,6 +107,9 @@ ], "system_instruction_modes": ["replace"] }, + "capabilities": { + "health": true + }, "telemetry": { "providers": { "relay": { diff --git a/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json b/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json index 12eb69898..1297849bb 100644 --- a/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json @@ -211,6 +211,9 @@ ], "system_instruction_modes": ["replace"] }, + "capabilities": { + "health": true + }, "telemetry": { "providers": { "relay": { diff --git a/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json b/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json index 7ae06401d..1f528ff2e 100644 --- a/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json @@ -87,6 +87,9 @@ ], "system_instruction_modes": ["replace"] }, + "capabilities": { + "health": true + }, "telemetry": { "providers": { "relay": { diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 6f49d1847..d88c22bdf 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -6,7 +6,7 @@ use std::collections::BTreeMap; use std::ffi::OsString; use std::fs::File; -use std::io::{BufRead, BufReader, ErrorKind, Write}; +use std::io::{BufRead, BufReader, ErrorKind, Read, Write}; use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpStream}; use std::path::{Path, PathBuf}; use std::process::{Child, ChildStdin, Command, Stdio}; @@ -42,7 +42,7 @@ const LOCAL_HOST_START_TIMEOUT: Duration = Duration::from_secs(90); const LOCAL_HOST_INVOKE_TIMEOUT: Duration = Duration::from_secs(60 * 60); const LOCAL_HOST_STOP_TIMEOUT: Duration = Duration::from_secs(10); const HEALTH_CONTROL_PROTOCOL: &str = "fabric.health/v1alpha1"; -const HEALTH_CONTROL_RESPONSE_LIMIT: u64 = 1024 * 1024; +const HEALTH_CONTROL_RESPONSE_LIMIT: usize = 1024 * 1024; const LOCAL_HOST_EXIT_GRACE: Duration = Duration::from_secs(2); const LOCAL_HOST_DIAGNOSTIC_LIMIT: usize = 16 * 1024; #[cfg(test)] @@ -389,7 +389,7 @@ pub struct HealthCheck { #[schemars(length(min = 1), regex(pattern = r"\S"))] pub reason_code: String, /// Unix timestamp in milliseconds when the evidence was observed. - pub observed_at_millis: u128, + pub observed_at_millis: u64, /// Age of the evidence when this report was assembled. pub age_millis: u64, /// Optional human-readable diagnostic detail. @@ -409,9 +409,9 @@ pub struct RuntimeHealth { #[schemars(length(min = 1), regex(pattern = r"\S"))] pub runtime_id: String, /// Unix timestamp in milliseconds when the report completed. - pub checked_at_millis: u128, + pub checked_at_millis: u64, /// Total probe duration in milliseconds. - pub duration_millis: u128, + pub duration_millis: u64, /// Adapter-host liveness. pub liveness: RuntimeLiveness, /// Current invocation and shutdown activity. @@ -763,6 +763,7 @@ struct AdapterLifecycleStart { base_dir: PathBuf, config: AgentConfig, runtime_context: RuntimeContext, + health_enabled: bool, capability_plan: CapabilityPlan, #[serde(skip_serializing_if = "Option::is_none")] telemetry_plan: Option, @@ -863,6 +864,7 @@ struct LocalAdapterHost { stderr_offset: usize, artifacts: ArtifactManifest, relay_config: Option, + process_exited: Arc, } struct LocalAdapterHostHandle { @@ -870,6 +872,7 @@ struct LocalAdapterHostHandle { health_control: Option, active_invocations: AtomicUsize, stopping: AtomicBool, + process_exited: Arc, } impl LocalAdapterHostHandle { @@ -1121,7 +1124,7 @@ pub fn check_runtime_health( } let started = Instant::now(); - let observed_at = now_millis(); + let observed_at = health_now_millis(); let Some(host) = local_hosts().get(&runtime.runtime_id).cloned() else { return Ok(finish_health_report( runtime, @@ -1206,7 +1209,7 @@ pub fn check_runtime_health( )); } - if !runtime_health_capability(plan) { + if !plan.capabilities.health { checks.push(health_check( "adapter.control", HealthCheckStatus::Unsupported, @@ -1261,22 +1264,24 @@ pub fn check_runtime_health( "adapter.control", HealthCheckStatus::Ok, "probe_succeeded", - now_millis(), + health_now_millis(), )); checks.extend(result.checks); - let (readiness, reason_code) = match activity { - RuntimeActivity::Busy => ( - RuntimeReadiness::NotReady, - "invocation_in_progress".to_string(), - ), - RuntimeActivity::Stopping => { - (RuntimeReadiness::NotReady, "stop_in_progress".to_string()) - } - RuntimeActivity::Idle | RuntimeActivity::Unknown => result - .readiness - .map(|readiness| (readiness.state, readiness.reason_code)) - .unwrap_or((RuntimeReadiness::Unknown, "readiness_unknown".to_string())), - }; + let (readiness, reason_code) = result + .readiness + .map(|readiness| (readiness.state, readiness.reason_code)) + .unwrap_or_else(|| match activity { + RuntimeActivity::Busy => ( + RuntimeReadiness::NotReady, + "invocation_in_progress".to_string(), + ), + RuntimeActivity::Stopping => { + (RuntimeReadiness::NotReady, "stop_in_progress".to_string()) + } + RuntimeActivity::Idle | RuntimeActivity::Unknown => { + (RuntimeReadiness::Unknown, "readiness_unknown".to_string()) + } + }); Ok(finish_health_report( runtime, started, @@ -1292,7 +1297,7 @@ pub fn check_runtime_health( "adapter.control", HealthCheckStatus::Unknown, "probe_timed_out", - now_millis(), + health_now_millis(), )); Ok(finish_health_report( runtime, @@ -1321,7 +1326,7 @@ pub fn check_runtime_health( "adapter.control", HealthCheckStatus::Unknown, reason_code, - now_millis(), + health_now_millis(), )); Ok(finish_health_report( runtime, @@ -1356,6 +1361,9 @@ enum ProcessObservation { } fn inspect_local_host_process(host: &LocalAdapterHostHandle) -> ProcessObservation { + if host.process_exited.load(Ordering::Acquire) { + return ProcessObservation::Exited; + } match host.host.try_lock() { Ok(mut host) => match host.child.try_wait() { Ok(Some(_)) => ProcessObservation::Exited, @@ -1370,18 +1378,10 @@ fn inspect_local_host_process(host: &LocalAdapterHostHandle) -> ProcessObservati Err(_) => ProcessObservation::Unknown, } } - Err(std::sync::TryLockError::WouldBlock) => ProcessObservation::Unknown, + Err(std::sync::TryLockError::WouldBlock) => ProcessObservation::Running, } } -fn runtime_health_capability(plan: &RunPlan) -> bool { - plan.capabilities.health - && plan - .adapter_descriptor - .as_ref() - .is_some_and(|adapter| adapter.descriptor.capabilities.health) -} - enum HealthControlProbe { Succeeded(AdapterHealthResult), TimedOut, @@ -1436,24 +1436,34 @@ fn probe_health_control( HealthControlProbe::Unavailable("control_unavailable") }; } - let mut line = String::new(); - let mut limited = - std::io::Read::take(BufReader::new(stream), HEALTH_CONTROL_RESPONSE_LIMIT + 1); - let read_result = limited.read_line(&mut line); - if let Err(error) = read_result { - return if is_timeout_error(&error) { - HealthControlProbe::TimedOut - } else { - HealthControlProbe::Unavailable("control_unavailable") + let mut line = Vec::new(); + let mut buffer = [0_u8; 4096]; + loop { + let Some(remaining) = timeout.checked_sub(started.elapsed()) else { + return HealthControlProbe::TimedOut; }; + if remaining.is_zero() || stream.set_read_timeout(Some(remaining)).is_err() { + return HealthControlProbe::TimedOut; + } + let read = match stream.read(&mut buffer) { + Ok(0) => return HealthControlProbe::Unavailable("control_protocol_error"), + Ok(read) => read, + Err(error) if is_timeout_error(&error) => return HealthControlProbe::TimedOut, + Err(_) => return HealthControlProbe::Unavailable("control_unavailable"), + }; + let record_end = buffer[..read] + .iter() + .position(|byte| *byte == b'\n') + .map_or(read, |index| index + 1); + if line.len().saturating_add(record_end) > HEALTH_CONTROL_RESPONSE_LIMIT { + return HealthControlProbe::Unavailable("control_protocol_error"); + } + line.extend_from_slice(&buffer[..record_end]); + if line.ends_with(b"\n") { + break; + } } - if line.len() as u64 > HEALTH_CONTROL_RESPONSE_LIMIT || !line.ends_with('\n') { - return HealthControlProbe::Unavailable("control_protocol_error"); - } - if line.len() > 1024 * 1024 { - return HealthControlProbe::Unavailable("control_protocol_error"); - } - let response: HealthControlResponse = match serde_json::from_str(&line) { + let response: HealthControlResponse = match serde_json::from_slice(&line) { Ok(response) => response, Err(_) => return HealthControlProbe::Unavailable("control_protocol_error"), }; @@ -1490,7 +1500,7 @@ fn health_check( name: impl Into, status: HealthCheckStatus, reason_code: impl Into, - observed_at_millis: u128, + observed_at_millis: u64, ) -> HealthCheck { HealthCheck { name: name.into(), @@ -1512,16 +1522,14 @@ fn finish_health_report( reason_code: impl Into, mut checks: Vec, ) -> RuntimeHealth { - let checked_at_millis = now_millis(); + let checked_at_millis = health_now_millis(); for check in &mut checks { - check.age_millis = checked_at_millis - .saturating_sub(check.observed_at_millis) - .min(u128::from(u64::MAX)) as u64; + check.age_millis = checked_at_millis.saturating_sub(check.observed_at_millis); } RuntimeHealth { runtime_id: runtime.runtime_id.clone(), checked_at_millis, - duration_millis: started.elapsed().as_millis(), + duration_millis: u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX), liveness, activity, readiness, @@ -1825,10 +1833,8 @@ impl RuntimeAdapter for LocalHostAdapter { return Err(error); } }; - let start_output: AdapterLifecycleStartOutput = if start_output.is_null() { - AdapterLifecycleStartOutput::default() - } else { - match serde_json::from_value(start_output) { + let start_output: AdapterLifecycleStartOutput = match start_output { + Value::Object(output) => match serde_json::from_value(Value::Object(output)) { Ok(output) => output, Err(source) => { let error = lifecycle_error( @@ -1842,13 +1848,16 @@ impl RuntimeAdapter for LocalHostAdapter { let _ = remove_local_host_files(&host); return Err(error); } - } + }, + _ => AdapterLifecycleStartOutput::default(), }; + let process_exited = Arc::clone(&host.process_exited); let handle = LocalAdapterHostHandle { host: Mutex::new(host), health_control: start_output.health_control, active_invocations: AtomicUsize::new(0), stopping: AtomicBool::new(false), + process_exited, }; local_hosts().insert(runtime.runtime_id.clone(), Arc::new(handle)); Ok(runtime) @@ -1874,29 +1883,33 @@ impl RuntimeAdapter for LocalHostAdapter { } fn stop(&self, runtime: &RuntimeHandle) -> Result> { - let Some(host) = local_hosts().remove(&runtime.runtime_id) else { + let Some(host) = local_hosts().get(&runtime.runtime_id).cloned() else { return Ok(vec![local_host_stop_event(runtime, true, false)]); }; - host.stopping.store(true, Ordering::Release); - let mut host = host.host.lock().unwrap_or_else(|error| error.into_inner()); + if host.stopping.swap(true, Ordering::AcqRel) { + return Ok(vec![local_host_stop_event(runtime, true, false)]); + } + let mut host_guard = host.host.lock().unwrap_or_else(|error| error.into_inner()); let request = AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Stop(AdapterLifecycleStop { runtime_id: runtime.runtime_id.clone(), })); let result = exchange_lifecycle_message( - &mut host, + &mut host_guard, &runtime.runtime_id, &request, LOCAL_HOST_STOP_TIMEOUT, ); - let termination = terminate_local_host(&mut host); - let diagnostics = local_host_diagnostics(&host); - let removal = remove_local_host_files(&host); + let termination = terminate_local_host(&mut host_guard); + let diagnostics = local_host_diagnostics(&host_guard); + let removal = remove_local_host_files(&host_guard); let host_crashed = matches!( &result, Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "host_crashed" ); + drop(host_guard); + local_hosts().remove(&runtime.runtime_id); if !host_crashed { result?; } @@ -2026,6 +2039,15 @@ fn run_local_host_invocation_with_timeout( })?; host.active_invocations.fetch_add(1, Ordering::AcqRel); let _activity = InvocationActivityGuard(&host); + if host.stopping.load(Ordering::Acquire) { + return Err(lifecycle_error( + operation, + &runtime.runtime_id, + "stop_in_progress", + "persistent local adapter host is stopping", + "", + )); + } let exchange_result = { let mut host_guard = host.host.lock().unwrap_or_else(|error| error.into_inner()); @@ -2430,6 +2452,8 @@ fn spawn_local_host( )); }; let (sender, responses) = mpsc::channel(); + let process_exited = Arc::new(AtomicBool::new(false)); + let reader_process_exited = Arc::clone(&process_exited); if let Err(source) = thread::Builder::new() .name(format!("fabric-host-{}", runtime.runtime_id)) .spawn(move || { @@ -2452,6 +2476,7 @@ fn spawn_local_host( } } } + reader_process_exited.store(true, Ordering::Release); }) { let _ = child.kill(); @@ -2472,6 +2497,7 @@ fn spawn_local_host( stderr_offset: 0, artifacts, relay_config, + process_exited, }) } @@ -2847,6 +2873,7 @@ fn adapter_lifecycle_start( artifacts, relay_config, ), + health_enabled: plan.capabilities.health, capability_plan: plan.capability_plan.clone(), telemetry_plan: plan.telemetry_plan.clone(), }) @@ -3608,6 +3635,10 @@ fn now_millis() -> u128 { .unwrap_or_default() } +fn health_now_millis() -> u64 { + u64::try_from(now_millis()).unwrap_or(u64::MAX) +} + #[cfg(test)] mod tests { use std::fs; @@ -3783,7 +3814,16 @@ def start_health_control(runtime_id): "runtime_id": runtime_id, "result": result, } - client.sendall(json.dumps(response).encode() + b"\n") + encoded = json.dumps(response).encode() + b"\n" + if MODE == "health_dribble": + for byte in encoded: + try: + client.sendall(bytes([byte])) + except BrokenPipeError: + break + time.sleep(0.05) + return + client.sendall(encoded) listener.close() threading.Thread(target=serve_health, daemon=True).start() @@ -3806,8 +3846,8 @@ for line in sys.stdin: sys.exit(16) output = ( start_health_control(message["payload"]["runtime_context"]["runtime_id"]) - if MODE in {"health_success", "health_timeout", "health_busy"} - else None + if MODE in {"health_success", "health_timeout", "health_dribble", "health_busy"} + else ("legacy-start-output" if MODE == "legacy_start_output" else None) ) response("start", output=output) if MODE == "crash_after_start": @@ -3820,6 +3860,9 @@ for line in sys.stdin: if MODE == "health_busy": print("busy invocation accepted", file=sys.stderr, flush=True) time.sleep(1) + if MODE == "crash_during_invoke": + print("crash invocation accepted", file=sys.stderr, flush=True) + os._exit(19) if MODE == "invoke_stderr": print(f"diagnostic-{invocations}", file=sys.stderr, flush=True) if MODE == "invoke_timeout": @@ -3900,6 +3943,9 @@ for line in sys.stdin: result = {"status": "succeeded", "output": output} response(operation, output=result) elif operation == "stop": + if MODE == "stop_slow": + print("stop accepted", file=sys.stderr, flush=True) + time.sleep(1) if MODE == "stop_failure": response("stop", error=failure("stop", "fake_stop", "stop rejected")) sys.exit(18) @@ -4106,6 +4152,21 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn local_host_health_timeout_is_one_total_deadline() { + let (root, plan) = local_host_plan("health_dribble"); + let runtime = start_runtime(&plan).expect("start local host"); + let started = Instant::now(); + + let health = check_runtime_health(&plan, &runtime, Duration::from_millis(75)) + .expect("health timeout is a report"); + + assert!(started.elapsed() < Duration::from_millis(500)); + assert_eq!(health.reason_code, "probe_timed_out"); + stop_runtime(&plan, &runtime).expect("stop local host"); + let _ = fs::remove_dir_all(root); + } + #[test] fn local_host_health_uses_control_path_during_invocation() { let (root, plan) = local_host_plan("health_busy"); @@ -4142,13 +4203,112 @@ for line in sys.stdin: assert_eq!(health.liveness, RuntimeLiveness::Responsive); assert_eq!(health.activity, RuntimeActivity::Busy); - assert_eq!(health.readiness, RuntimeReadiness::NotReady); - assert_eq!(health.reason_code, "invocation_in_progress"); + assert_eq!(health.readiness, RuntimeReadiness::Ready); + assert_eq!(health.reason_code, "ready"); invocation.join().expect("join invocation").expect("invoke"); stop_runtime(&plan, &runtime).expect("stop local host"); let _ = fs::remove_dir_all(root); } + #[test] + fn local_host_health_reports_stop_in_progress() { + let (root, plan) = local_host_plan("stop_slow"); + let runtime = start_runtime(&plan).expect("start local host"); + let stderr_path = local_hosts()[&runtime.runtime_id] + .host + .lock() + .expect("local host") + .stderr_path + .clone(); + let stop_plan = plan.clone(); + let stop_runtime_handle = runtime.clone(); + let stopping = thread::spawn(move || stop_runtime(&stop_plan, &stop_runtime_handle)); + let accepted_deadline = Instant::now() + Duration::from_secs(2); + while !fs::read_to_string(&stderr_path) + .expect("read host stderr") + .contains("stop accepted") + { + assert!(Instant::now() < accepted_deadline, "stop was not accepted"); + thread::sleep(Duration::from_millis(10)); + } + + let health = check_runtime_health(&plan, &runtime, Duration::from_millis(500)) + .expect("check stopping runtime health"); + + assert_eq!(health.activity, RuntimeActivity::Stopping); + assert_eq!(health.readiness, RuntimeReadiness::NotReady); + assert_eq!(health.reason_code, "stop_in_progress"); + stopping + .join() + .expect("join stop") + .expect("stop local host"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn local_host_health_observes_process_exit_during_invocation() { + let (root, plan) = local_host_plan("crash_during_invoke"); + let runtime = start_runtime(&plan).expect("start local host"); + let stderr_path = local_hosts()[&runtime.runtime_id] + .host + .lock() + .expect("local host") + .stderr_path + .clone(); + let invoke_plan = plan.clone(); + let invoke_runtime_handle = runtime.clone(); + let invocation = thread::spawn(move || { + invoke_runtime( + &invoke_plan, + &invoke_runtime_handle, + RunRequest::text("crash"), + ) + }); + let accepted_deadline = Instant::now() + Duration::from_secs(2); + while !fs::read_to_string(&stderr_path) + .expect("read host stderr") + .contains("crash invocation accepted") + { + assert!( + Instant::now() < accepted_deadline, + "crashing invocation was not accepted" + ); + thread::sleep(Duration::from_millis(10)); + } + let health_deadline = Instant::now() + Duration::from_secs(2); + let health = loop { + let health = check_runtime_health(&plan, &runtime, Duration::from_millis(100)) + .expect("check crashed runtime health"); + if health.reason_code == "process_exited" { + break health; + } + assert!( + Instant::now() < health_deadline, + "process exit was not observed" + ); + thread::sleep(Duration::from_millis(10)); + }; + + assert_eq!(health.liveness, RuntimeLiveness::Exited); + assert_eq!(health.readiness, RuntimeReadiness::NotReady); + invocation + .join() + .expect("join invocation") + .expect_err("invocation should observe host crash"); + stop_runtime(&plan, &runtime).expect("stop crashed local host"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn local_host_tolerates_legacy_scalar_start_output() { + let (root, plan) = local_host_plan("legacy_start_output"); + + let runtime = start_runtime(&plan).expect("start local host"); + + stop_runtime(&plan, &runtime).expect("stop local host"); + let _ = fs::remove_dir_all(root); + } + #[test] fn adapter_lifecycle_always_receives_southbound_agent_config() { let (root, plan) = local_host_plan("success"); diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-healthcheck.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-healthcheck.mdx index 83d733ec8..3a879d6c3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-healthcheck.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-healthcheck.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub status: HealthCheckStatus,\n    pub reason_code: String,\n    pub observed_at_millis: u128,\n    pub age_millis: u64,\n    pub message: Option<String>,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub status: HealthCheckStatus,\n    pub reason_code: String,\n    pub observed_at_millis: u64,\n    pub age_millis: u64,\n    pub message: Option<String>,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
One timestamped runtime or adapter health observation. @@ -27,7 +27,7 @@ Structured check outcome. Stable machine-readable reason. -### `observed_at_millis: u128` +### `observed_at_millis: u64` Unix timestamp in milliseconds when the evidence was observed. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehealth.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehealth.mdx index 417f02aa3..8760acc09 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehealth.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehealth.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub checked_at_millis: u128,\n    pub duration_millis: u128,\n    pub liveness: RuntimeLiveness,\n    pub activity: RuntimeActivity,\n    pub readiness: RuntimeReadiness,\n    pub reason_code: String,\n    pub checks: Vec<HealthCheck>,\n}"}} />
+
String,\n    pub checked_at_millis: u64,\n    pub duration_millis: u64,\n    pub liveness: RuntimeLiveness,\n    pub activity: RuntimeActivity,\n    pub readiness: RuntimeReadiness,\n    pub reason_code: String,\n    pub checks: Vec<HealthCheck>,\n}"}} />
Bounded health report for one started runtime. @@ -19,11 +19,11 @@ Bounded health report for one started runtime. Runtime represented by this report. -### `checked_at_millis: u128` +### `checked_at_millis: u64` Unix timestamp in milliseconds when the report completed. -### `duration_millis: u128` +### `duration_millis: u64` Total probe duration in milliseconds. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index ede457dfc..c41709825 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -176,6 +176,13 @@ an unavailable control endpoint, and an unsupported adapter check are report data rather than exceptions. Inspect `reason_code` and each check's `status` instead of parsing diagnostic messages. +Each check records `observed_at_millis`, the Unix timestamp in milliseconds when +its evidence was collected. The report recomputes `age_millis` at +`checked_at_millis`, when the complete report is assembled. Common process and +control checks are fresh observations and normally have an age near zero. An +adapter hook that returns cached evidence must preserve the original observation +timestamp so its age identifies the evidence as cached rather than fresh. + Health checks use an authenticated loopback control endpoint that is separate from the ordered lifecycle channel. You can therefore check a runtime while an invocation is active. Under the one-invocation policy, the report uses diff --git a/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json b/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json index bc761d0ce..b507daf17 100644 --- a/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json +++ b/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json @@ -90,6 +90,9 @@ ], "system_instruction_modes": ["replace", "append"] }, + "capabilities": { + "health": true + }, "telemetry": { "providers": { "relay": { diff --git a/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json b/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json index 7ae06401d..1f528ff2e 100644 --- a/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json +++ b/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json @@ -87,6 +87,9 @@ ], "system_instruction_modes": ["replace"] }, + "capabilities": { + "health": true + }, "telemetry": { "providers": { "relay": { diff --git a/schemas/adapter-contract/adapter-health-result.schema.json b/schemas/adapter-contract/adapter-health-result.schema.json index 7a998afe0..07c7ab303 100644 --- a/schemas/adapter-contract/adapter-health-result.schema.json +++ b/schemas/adapter-contract/adapter-health-result.schema.json @@ -53,7 +53,7 @@ }, "observed_at_millis": { "description": "Unix timestamp in milliseconds when the evidence was observed.", - "format": "uint128", + "format": "uint64", "minimum": 0, "type": "integer" }, diff --git a/schemas/sdk/runtime-health.schema.json b/schemas/sdk/runtime-health.schema.json index 8976713b1..a2fadc4b1 100644 --- a/schemas/sdk/runtime-health.schema.json +++ b/schemas/sdk/runtime-health.schema.json @@ -32,7 +32,7 @@ }, "observed_at_millis": { "description": "Unix timestamp in milliseconds when the evidence was observed.", - "format": "uint128", + "format": "uint64", "minimum": 0, "type": "integer" }, @@ -162,7 +162,7 @@ }, "checked_at_millis": { "description": "Unix timestamp in milliseconds when the report completed.", - "format": "uint128", + "format": "uint64", "minimum": 0, "type": "integer" }, @@ -175,7 +175,7 @@ }, "duration_millis": { "description": "Total probe duration in milliseconds.", - "format": "uint128", + "format": "uint64", "minimum": 0, "type": "integer" }, diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 99133f351..73366de07 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -214,6 +214,9 @@ The common host also serves health on an authenticated loopback endpoint that does not share the ordered lifecycle channel. Optional health hooks must respect `request.timeout_millis`, avoid inference or other billable probes, avoid runtime mutation, return stable reason codes, and never expose credentials. +TypeScript hooks should also stop promptly when their optional `AbortSignal` is +aborted. Cached checks must preserve their original `observed_at_millis` so +NeMo Fabric can report their age accurately. Treat hook failure and timeout as health data; do not fail the runtime. ### Support Warm Session Continuation diff --git a/tests/adapters/test_claude_adapter.py b/tests/adapters/test_claude_adapter.py index 07c24d6af..781b07fad 100644 --- a/tests/adapters/test_claude_adapter.py +++ b/tests/adapters/test_claude_adapter.py @@ -237,6 +237,7 @@ def test_claude_descriptor_is_narrow_and_versioned(): ], "system_instruction_modes": ["replace", "append"], }, + "capabilities": {"health": True}, "telemetry": { "providers": { "relay": { diff --git a/tests/adapters/test_codex_adapter.py b/tests/adapters/test_codex_adapter.py index 52bfd3cd0..8c0c9ab87 100644 --- a/tests/adapters/test_codex_adapter.py +++ b/tests/adapters/test_codex_adapter.py @@ -1706,6 +1706,7 @@ def test_descriptor_has_no_codex_binary_requirement(): "skills", ] assert descriptor["config"]["system_instruction_modes"] == ["replace"] + assert descriptor["capabilities"]["health"] is True assert descriptor["model_schema"]["if"]["properties"]["provider"] == { "const": "openai" } diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 71567f8b1..7e19fd6f5 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -49,6 +49,7 @@ def test_descriptor_declares_supported_normalized_config(): ) assert descriptor["config"]["system_instruction_modes"] == ["replace"] + assert descriptor["capabilities"]["health"] is True assert "runtime.max_turns" in descriptor["config"]["accepts"] assert descriptor["settings_schema"]["properties"]["deepagents"]["properties"][ "backend" diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index 1efd0edc8..b6195ba14 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -143,6 +143,7 @@ def test_descriptor_uses_the_typed_agent_config_contract(): ] assert "model" not in descriptor["extension_schemas"] assert descriptor["config"]["system_instruction_modes"] == ["replace"] + assert descriptor["capabilities"]["health"] is True async def test_runtime_start_rejects_append_system_instruction(tmp_path: Path): diff --git a/tests/python/test_adapter_lifecycle_health.py b/tests/python/test_adapter_lifecycle_health.py index 9330ed8a7..138ef3a2d 100644 --- a/tests/python/test_adapter_lifecycle_health.py +++ b/tests/python/test_adapter_lifecycle_health.py @@ -7,11 +7,18 @@ import asyncio import json +import sys from typing import Any from nemo_fabric_adapter_contract.models import AdapterHealthRequest +from nemo_fabric_adapter_contract.models import AdapterHealthResult +from nemo_fabric_adapter_contract.models import AdapterReadiness +from nemo_fabric_adapter_contract.models import RuntimeReadiness +from nemo_fabric_adapters.common.lifecycle import _adapter_call from nemo_fabric_adapters.common.lifecycle import _adapter_health from nemo_fabric_adapters.common.lifecycle import _close_health_server +from nemo_fabric_adapters.common.lifecycle import _handle_start +from nemo_fabric_adapters.common.lifecycle import _handle_stop from nemo_fabric_adapters.common.lifecycle import _HostState from nemo_fabric_adapters.common.lifecycle import _start_health_server @@ -35,6 +42,47 @@ async def health(self, request: AdapterHealthRequest): raise AssertionError("health hook should time out") +class _ReadyWhileBusyRuntime(_Runtime): + async def health(self, request: AdapterHealthRequest): + del request + return AdapterHealthResult( + readiness=AdapterReadiness( + state=RuntimeReadiness.READY, + reason_code="concurrent_invocations_supported", + ) + ) + + +class _BlockingStopRuntime(_Runtime): + def __init__(self, started: asyncio.Event, release: asyncio.Event) -> None: + self._started = started + self._release = release + + async def stop(self): + self._started.set() + await self._release.wait() + + +async def _check_health(control: dict[str, Any], runtime_id: str) -> dict[str, Any]: + reader, writer = await asyncio.open_connection(control["host"], control["port"]) + writer.write( + json.dumps( + { + "protocol_version": control["protocol_version"], + "token": control["token"], + "runtime_id": runtime_id, + "timeout_millis": 1_000, + } + ).encode() + + b"\n" + ) + await writer.drain() + response = json.loads(await reader.readline()) + writer.close() + await writer.wait_closed() + return response + + async def test_python_health_control_reports_busy_without_lifecycle_channel(): state = _HostState( runtime=_Runtime(), @@ -44,24 +92,7 @@ async def test_python_health_control_reports_busy_without_lifecycle_channel(): output = await _start_health_server(state) control = output["health_control"] try: - reader, writer = await asyncio.open_connection( - control["host"], control["port"] - ) - writer.write( - json.dumps( - { - "protocol_version": control["protocol_version"], - "token": control["token"], - "runtime_id": "runtime-1", - "timeout_millis": 1_000, - } - ).encode() - + b"\n" - ) - await writer.drain() - response = json.loads(await reader.readline()) - writer.close() - await writer.wait_closed() + response = await _check_health(control, "runtime-1") finally: await _close_health_server(state) @@ -88,3 +119,86 @@ async def test_python_adapter_health_hook_timeout_is_data(): assert result.readiness is not None assert result.readiness.state.value == "ready" assert result.checks[-1].reason_code == "adapter_health_timed_out" + + +async def test_python_adapter_health_preserves_busy_runtime_readiness(): + state = _HostState( + runtime=_ReadyWhileBusyRuntime(), + runtime_id="runtime-1", + invoking=True, + ) + + result = await _adapter_health( + state, + AdapterHealthRequest(runtime_id="runtime-1", timeout_millis=1_000), + ) + + assert result.readiness is not None + assert result.readiness.state is RuntimeReadiness.READY + assert result.readiness.reason_code == "concurrent_invocations_supported" + + +async def test_python_health_reports_stop_in_progress(): + stop_started = asyncio.Event() + release_stop = asyncio.Event() + runtime = _BlockingStopRuntime(stop_started, release_stop) + state = _HostState(runtime=runtime, runtime_id="runtime-1") + control = (await _start_health_server(state))["health_control"] + stopping = asyncio.create_task(_handle_stop(state, runtime)) + await stop_started.wait() + + response = await _check_health(control, "runtime-1") + + assert response["result"]["readiness"] == { + "state": "not_ready", + "reason_code": "stop_in_progress", + } + release_stop.set() + await stopping + + +async def test_python_host_skips_health_server_when_capability_is_disabled(): + state = _HostState() + + response = await _handle_start( + state, + _Runtime, + {"health_enabled": False}, + "runtime-1", + None, + ) + + assert response["outcome"]["output"] is None + assert state.health_server is None + assert state.runtime is not None + await _handle_stop(state, state.runtime) + + +async def test_adapter_calls_do_not_rebind_stdout_when_they_overlap(): + original_stdout = sys.stdout + first_started = asyncio.Event() + second_started = asyncio.Event() + release_first = asyncio.Event() + release_second = asyncio.Event() + + async def first_call(): + first_started.set() + await release_first.wait() + + async def second_call(): + second_started.set() + await release_second.wait() + + try: + first = asyncio.create_task(_adapter_call("first", first_call)) + await first_started.wait() + second = asyncio.create_task(_adapter_call("second", second_call)) + await second_started.wait() + release_first.set() + await first + release_second.set() + await second + + assert sys.stdout is original_stdout + finally: + sys.stdout = original_stdout From d3aebd464458dcddeaf2965249154dbc4353c93e Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 16 Sep 2026 11:03:22 -0700 Subject: [PATCH 3/3] fix: address runtime health review feedback Signed-off-by: Yuchen Zhang --- .../nemo_fabric_adapters/common/lifecycle.py | 3 +- .../typescript/common/test/lifecycle.test.mjs | 121 ++++++++++++---- crates/fabric-core/src/runtime.rs | 15 +- crates/fabric-core/src/schema.rs | 77 ++++++---- docs/adapter-contract/tutorials/execution.md | 2 +- .../runtime/enum-runtimeactivity.mdx | 6 +- .../runtime/enum-runtimeliveness.mdx | 8 +- .../runtime/enum-runtimereadiness.mdx | 6 +- .../nemo-fabric-core/runtime/index.mdx | 6 +- docs/sdk/python.mdx | 10 +- .../adapter-health-result.schema.json | 4 +- schemas/sdk/runtime-health.schema.json | 14 +- .../src/nemo_fabric/runtime.py | 6 +- tests/python/test_adapter_lifecycle_health.py | 131 +++++++++++------- tests/python/test_runtime.py | 26 ++++ 15 files changed, 295 insertions(+), 140 deletions(-) diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/lifecycle.py b/adapters/python/common/src/nemo_fabric_adapters/common/lifecycle.py index fc08880a0..027cb6e3e 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/lifecycle.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/lifecycle.py @@ -557,13 +557,14 @@ async def _close_health_server(state: _HostState) -> None: state.health_token = None if server is not None: server.close() - await server.wait_closed() tasks = tuple(state.health_tasks) state.health_tasks.clear() for task in tasks: task.cancel() if tasks: await asyncio.gather(*tasks, return_exceptions=True) + if server is not None: + await server.wait_closed() def _schedule_health_connection( diff --git a/adapters/typescript/common/test/lifecycle.test.mjs b/adapters/typescript/common/test/lifecycle.test.mjs index 87cbbb665..3b5a8dfb5 100644 --- a/adapters/typescript/common/test/lifecycle.test.mjs +++ b/adapters/typescript/common/test/lifecycle.test.mjs @@ -70,15 +70,34 @@ async function exchange(factory, messages) { function responseReader(output) { let encoded = ""; + let ended = false; + let failure; let wake; + const notify = () => { + wake?.(); + wake = undefined; + }; output.setEncoding("utf8"); output.on("data", (chunk) => { encoded += chunk; - wake?.(); - wake = undefined; + notify(); + }); + output.once("end", () => { + ended = true; + notify(); + }); + output.once("error", (error) => { + failure = error; + notify(); }); return async () => { while (!encoded.includes("\n")) { + if (failure !== undefined) { + throw failure; + } + if (ended) { + throw new Error("lifecycle output ended without a complete response"); + } await new Promise((resolve) => { wake = resolve; }); @@ -90,9 +109,43 @@ function responseReader(output) { }; } +test("response reader settles when lifecycle output terminates", async () => { + const endedOutput = new PassThrough(); + const endedResponse = responseReader(endedOutput)(); + endedOutput.end(); + await assert.rejects(endedResponse, /ended without a complete response/); + + const failedOutput = new PassThrough(); + const failedResponse = responseReader(failedOutput)(); + failedOutput.destroy(new Error("lifecycle output failed")); + await assert.rejects(failedResponse, /lifecycle output failed/); +}); + async function checkHealth(control, runtimeId, timeoutMillis = 1000) { const socket = connect(control.port, control.host); socket.setEncoding("utf8"); + await new Promise((resolve, reject) => { + const cleanup = () => { + clearTimeout(timer); + socket.off("connect", onConnect); + socket.off("error", onError); + }; + const onConnect = () => { + cleanup(); + resolve(); + }; + const onError = (error) => { + cleanup(); + reject(error); + }; + const timer = setTimeout(() => { + cleanup(); + socket.destroy(); + reject(new Error("health connection timed out")); + }, 1000); + socket.once("connect", onConnect); + socket.once("error", onError); + }); let encoded = ""; const response = new Promise((resolve, reject) => { socket.on("data", (chunk) => { @@ -109,7 +162,6 @@ async function checkHealth(control, runtimeId, timeoutMillis = 1000) { } }); }); - await new Promise((resolve) => socket.once("connect", resolve)); socket.write(`${JSON.stringify({ protocol_version: "fabric.health/v1alpha1", token: control.token, @@ -153,23 +205,28 @@ test("serves health independently while an invocation is busy", async () => { input.write(`${JSON.stringify(invoke("runtime-1", "one", "one"))}\n`); await started; - const healthResponse = await checkHealth(control, "runtime-1"); - - assert.equal(healthResponse.result.readiness.state, "ready"); - assert.equal(healthResponse.result.readiness.reason_code, "concurrent_invocations_supported"); - assert.deepEqual( - healthResponse.result.checks.map((check) => [check.name, check.status]), - [ - ["dependency.inference", "unsupported"], - ], - ); - - releaseInvocation(); - await nextResponse(); - input.write(`${JSON.stringify(stop("runtime-1"))}\n`); - await nextResponse(); - input.end(); - await serving; + try { + const healthResponse = await checkHealth(control, "runtime-1"); + + assert.equal(healthResponse.result.readiness.state, "ready"); + assert.equal(healthResponse.result.readiness.reason_code, "concurrent_invocations_supported"); + assert.deepEqual( + healthResponse.result.checks.map((check) => [check.name, check.status]), + [ + ["dependency.inference", "unsupported"], + ], + ); + } finally { + releaseInvocation(); + try { + await nextResponse(); + input.write(`${JSON.stringify(stop("runtime-1"))}\n`); + await nextResponse(); + } finally { + input.end(); + await serving; + } + } }); test("reports stop in progress while adapter cleanup is running", async () => { @@ -202,16 +259,22 @@ test("reports stop in progress while adapter cleanup is running", async () => { input.write(`${JSON.stringify(stop("runtime-1"))}\n`); await started; - const healthResponse = await checkHealth(control, "runtime-1"); + try { + const healthResponse = await checkHealth(control, "runtime-1"); - assert.deepEqual(healthResponse.result.readiness, { - state: "not_ready", - reason_code: "stop_in_progress", - }); - releaseStop(); - await nextResponse(); - input.end(); - await serving; + assert.deepEqual(healthResponse.result.readiness, { + state: "not_ready", + reason_code: "stop_in_progress", + }); + } finally { + releaseStop(); + try { + await nextResponse(); + } finally { + input.end(); + await serving; + } + } }); test("aborts a timed-out adapter health hook", async () => { diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index d88c22bdf..a65e76354 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -322,7 +322,8 @@ pub struct RuntimeHandle { pub environment: EnvironmentHandle, } -/// Whether the adapter host can be reached independently of invocation traffic. +/// Whether NVIDIA NeMo Fabric can reach the adapter host independently of +/// invocation traffic. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum RuntimeLiveness { @@ -330,13 +331,13 @@ pub enum RuntimeLiveness { Responsive, /// The process was observed running but its health control path did not respond. Unresponsive, - /// Fabric directly observed that the adapter process exited. + /// NeMo Fabric directly observed that the adapter process exited. Exited, - /// Fabric could not establish liveness. + /// NeMo Fabric could not establish liveness. Unknown, } -/// Current runtime activity observed by Fabric. +/// Current runtime activity observed by NVIDIA NeMo Fabric. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum RuntimeActivity { @@ -346,11 +347,11 @@ pub enum RuntimeActivity { Busy, /// Runtime shutdown is in progress. Stopping, - /// Fabric could not establish activity. + /// NeMo Fabric could not establish activity. Unknown, } -/// Whether Fabric knows the runtime can currently accept work. +/// Whether NVIDIA NeMo Fabric knows the runtime can currently accept work. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum RuntimeReadiness { @@ -358,7 +359,7 @@ pub enum RuntimeReadiness { Ready, /// The runtime is known not to accept work. NotReady, - /// Fabric lacks enough fresh evidence to decide. + /// NeMo Fabric lacks enough fresh evidence to decide. Unknown, } diff --git a/crates/fabric-core/src/schema.rs b/crates/fabric-core/src/schema.rs index 21c9692c3..5c77bbc45 100644 --- a/crates/fabric-core/src/schema.rs +++ b/crates/fabric-core/src/schema.rs @@ -302,37 +302,64 @@ mod tests { #[test] fn schemas_use_contract_boundary_snapshot_paths() { - for schema in [ - SchemaName::AdapterDescriptor, - SchemaName::AgentConfig, - SchemaName::AgentRunRequest, - SchemaName::AgentRunResult, - SchemaName::AdapterInvocation, - SchemaName::OpenAiStreamInvocation, - SchemaName::OpenAiStreamRecord, - SchemaName::RuntimeContext, + for (schema, expected) in [ + ( + SchemaName::AdapterDescriptor, + "adapter-descriptor.schema.json", + ), + (SchemaName::AgentConfig, "agent-config.schema.json"), + (SchemaName::AgentRunRequest, "agent-run-request.schema.json"), + (SchemaName::AgentRunResult, "agent-run-result.schema.json"), + ( + SchemaName::AdapterInvocation, + "adapter-invocation.schema.json", + ), + ( + SchemaName::AdapterHealthRequest, + "adapter-health-request.schema.json", + ), + ( + SchemaName::AdapterHealthResult, + "adapter-health-result.schema.json", + ), + ( + SchemaName::OpenAiStreamInvocation, + "openai-stream-invocation.schema.json", + ), + ( + SchemaName::OpenAiStreamRecord, + "openai-stream-record.schema.json", + ), + (SchemaName::RuntimeContext, "runtime-context.schema.json"), ] { assert_eq!( schema.relative_path(), - PathBuf::from("adapter-contract").join(schema.filename()) + PathBuf::from("adapter-contract").join(expected) ); } - for schema in [ - SchemaName::Agent, - SchemaName::RunPlan, - SchemaName::EnvironmentHandle, - SchemaName::RuntimeHandle, - SchemaName::InvocationHandle, - SchemaName::RunRequest, - SchemaName::RunResult, - SchemaName::ArtifactManifest, - SchemaName::ErrorInfo, - SchemaName::FabricEvent, + for (schema, expected) in [ + (SchemaName::Agent, "agent.schema.json"), + (SchemaName::RunPlan, "run-plan.schema.json"), + ( + SchemaName::EnvironmentHandle, + "environment-handle.schema.json", + ), + (SchemaName::RuntimeHandle, "runtime-handle.schema.json"), + (SchemaName::RuntimeHealth, "runtime-health.schema.json"), + ( + SchemaName::InvocationHandle, + "invocation-handle.schema.json", + ), + (SchemaName::RunRequest, "run-request.schema.json"), + (SchemaName::RunResult, "run-result.schema.json"), + ( + SchemaName::ArtifactManifest, + "artifact-manifest.schema.json", + ), + (SchemaName::ErrorInfo, "error-info.schema.json"), + (SchemaName::FabricEvent, "fabric-event.schema.json"), ] { - assert_eq!( - schema.relative_path(), - PathBuf::from("sdk").join(schema.filename()) - ); + assert_eq!(schema.relative_path(), PathBuf::from("sdk").join(expected)); } } diff --git a/docs/adapter-contract/tutorials/execution.md b/docs/adapter-contract/tutorials/execution.md index 9c8fae07b..bc1993f5d 100644 --- a/docs/adapter-contract/tutorials/execution.md +++ b/docs/adapter-contract/tutorials/execution.md @@ -184,7 +184,7 @@ capability. The maintained common hosts expose an authenticated loopback health endpoint that remains independent of ordered lifecycle traffic. Set -`capabilities.health: true` when the adapter uses that host. Fabric reports +`capabilities.health: true` when the adapter uses that host. NeMo Fabric reports process and control-path observations even when the runtime class does not implement an adapter-specific hook. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimeactivity.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimeactivity.mdx index 730fea377..1afb27daf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimeactivity.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimeactivity.mdx @@ -1,7 +1,7 @@ --- title: "Enum Runtime Activity" sidebar-title: "RuntimeActivity" -description: "Current runtime activity observed by Fabric." +description: "Current runtime activity observed by NVIDIA NeMo Fabric." position: 34 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. @@ -18,7 +18,7 @@ pub enum RuntimeActivity { } ``` -Current runtime activity observed by Fabric. +Current runtime activity observed by NVIDIA NeMo Fabric. ## Variants @@ -44,7 +44,7 @@ Runtime shutdown is in progress.
-Fabric could not establish activity. +NeMo Fabric could not establish activity. ## Trait Implementations diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimeliveness.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimeliveness.mdx index 6d60e9298..e2ef94272 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimeliveness.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimeliveness.mdx @@ -1,7 +1,7 @@ --- title: "Enum Runtime Liveness" sidebar-title: "RuntimeLiveness" -description: "Whether the adapter host can be reached independently of invocation traffic." +description: "Whether NVIDIA NeMo Fabric can reach the adapter host independently of invocation traffic." position: 35 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. @@ -18,7 +18,7 @@ pub enum RuntimeLiveness { } ``` -Whether the adapter host can be reached independently of invocation traffic. +Whether NVIDIA NeMo Fabric can reach the adapter host independently of invocation traffic. ## Variants @@ -38,13 +38,13 @@ The process was observed running but its health control path did not respond.
-Fabric directly observed that the adapter process exited. +NeMo Fabric directly observed that the adapter process exited. ### `Unknown`
-Fabric could not establish liveness. +NeMo Fabric could not establish liveness. ## Trait Implementations diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimereadiness.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimereadiness.mdx index c48a5cfad..684db2913 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimereadiness.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runtimereadiness.mdx @@ -1,7 +1,7 @@ --- title: "Enum Runtime Readiness" sidebar-title: "RuntimeReadiness" -description: "Whether Fabric knows the runtime can currently accept work." +description: "Whether NVIDIA NeMo Fabric knows the runtime can currently accept work." position: 36 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. @@ -17,7 +17,7 @@ pub enum RuntimeReadiness { } ``` -Whether Fabric knows the runtime can currently accept work. +Whether NVIDIA NeMo Fabric knows the runtime can currently accept work. ## Variants @@ -37,7 +37,7 @@ The runtime is known not to accept work.
-Fabric lacks enough fresh evidence to decide. +NeMo Fabric lacks enough fresh evidence to decide. ## Trait Implementations diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index 97b51b6ee..1915bda17 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -49,9 +49,9 @@ Runtime invocation helpers. - [OpenAiStreamProtocolVersion](enum-openaistreamprotocolversion.mdx): Supported southbound native-streaming protocol version. - [OpenAiStreamRecord](enum-openaistreamrecord.mdx): One correlated NDJSON record on the adapter-native stream channel. - [RunStatus](enum-runstatus.mdx): Runtime completion status. -- [RuntimeActivity](enum-runtimeactivity.mdx): Current runtime activity observed by Fabric. -- [RuntimeLiveness](enum-runtimeliveness.mdx): Whether the adapter host can be reached independently of invocation traffic. -- [RuntimeReadiness](enum-runtimereadiness.mdx): Whether Fabric knows the runtime can currently accept work. +- [RuntimeActivity](enum-runtimeactivity.mdx): Current runtime activity observed by NVIDIA NeMo Fabric. +- [RuntimeLiveness](enum-runtimeliveness.mdx): Whether NVIDIA NeMo Fabric can reach the adapter host independently of invocation traffic. +- [RuntimeReadiness](enum-runtimereadiness.mdx): Whether NVIDIA NeMo Fabric knows the runtime can currently accept work. ## Functions diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index c41709825..5d2a4d264 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -191,10 +191,12 @@ invocation is active. Under the one-invocation policy, the report uses invocation, stop the runtime, or mark it failed. The SDK raises `FabricConfigError` for a nonpositive or nonfinite timeout and -`FabricStateError` after the runtime has stopped. Transport or native binding -errors raise `FabricRuntimeError`. Health inspection is currently implemented -for local Process and Python adapter hosts. Older hosts and adapters that do not -declare `capabilities.health` return explicit unknown or unsupported evidence. +`FabricStateError` after the runtime has stopped. If the native extension is +unavailable, the SDK raises `FabricNativeUnavailableError`. Other health +inspection failures raise `FabricRuntimeError`. Health inspection is currently +implemented for local Process and Python adapter hosts. Older hosts and adapters +that do not declare `capabilities.health` return explicit unknown or unsupported +evidence. ## Configure Agents In Code diff --git a/schemas/adapter-contract/adapter-health-result.schema.json b/schemas/adapter-contract/adapter-health-result.schema.json index 07c7ab303..f4f9bd532 100644 --- a/schemas/adapter-contract/adapter-health-result.schema.json +++ b/schemas/adapter-contract/adapter-health-result.schema.json @@ -103,7 +103,7 @@ ] }, "RuntimeReadiness": { - "description": "Whether Fabric knows the runtime can currently accept work.", + "description": "Whether NVIDIA NeMo Fabric knows the runtime can currently accept work.", "oneOf": [ { "const": "ready", @@ -117,7 +117,7 @@ }, { "const": "unknown", - "description": "Fabric lacks enough fresh evidence to decide.", + "description": "NeMo Fabric lacks enough fresh evidence to decide.", "type": "string" } ] diff --git a/schemas/sdk/runtime-health.schema.json b/schemas/sdk/runtime-health.schema.json index a2fadc4b1..b111c0a93 100644 --- a/schemas/sdk/runtime-health.schema.json +++ b/schemas/sdk/runtime-health.schema.json @@ -82,7 +82,7 @@ ] }, "RuntimeActivity": { - "description": "Current runtime activity observed by Fabric.", + "description": "Current runtime activity observed by NVIDIA NeMo Fabric.", "oneOf": [ { "const": "idle", @@ -101,13 +101,13 @@ }, { "const": "unknown", - "description": "Fabric could not establish activity.", + "description": "NeMo Fabric could not establish activity.", "type": "string" } ] }, "RuntimeLiveness": { - "description": "Whether the adapter host can be reached independently of invocation traffic.", + "description": "Whether NVIDIA NeMo Fabric can reach the adapter host independently of\ninvocation traffic.", "oneOf": [ { "const": "responsive", @@ -121,18 +121,18 @@ }, { "const": "exited", - "description": "Fabric directly observed that the adapter process exited.", + "description": "NeMo Fabric directly observed that the adapter process exited.", "type": "string" }, { "const": "unknown", - "description": "Fabric could not establish liveness.", + "description": "NeMo Fabric could not establish liveness.", "type": "string" } ] }, "RuntimeReadiness": { - "description": "Whether Fabric knows the runtime can currently accept work.", + "description": "Whether NVIDIA NeMo Fabric knows the runtime can currently accept work.", "oneOf": [ { "const": "ready", @@ -146,7 +146,7 @@ }, { "const": "unknown", - "description": "Fabric lacks enough fresh evidence to decide.", + "description": "NeMo Fabric lacks enough fresh evidence to decide.", "type": "string" } ] diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index 2550c196b..eb25511a8 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -202,11 +202,15 @@ def check() -> dict[str, Any]: ) return json.loads(encoded) - return RuntimeHealth.from_mapping(await _call_blocking(check)) + report = await _call_blocking(check) except FabricError: raise except Exception as error: raise FabricRuntimeError(str(error), stage="health") from error + try: + return RuntimeHealth.from_mapping(report) + except FabricConfigError as error: + raise FabricRuntimeError(str(error), stage="health") from error async def invoke( self, diff --git a/tests/python/test_adapter_lifecycle_health.py b/tests/python/test_adapter_lifecycle_health.py index 138ef3a2d..350b2e9c7 100644 --- a/tests/python/test_adapter_lifecycle_health.py +++ b/tests/python/test_adapter_lifecycle_health.py @@ -9,6 +9,8 @@ import json import sys from typing import Any +from unittest.mock import AsyncMock +from unittest.mock import MagicMock from nemo_fabric_adapter_contract.models import AdapterHealthRequest from nemo_fabric_adapter_contract.models import AdapterHealthResult @@ -23,44 +25,17 @@ from nemo_fabric_adapters.common.lifecycle import _start_health_server -class _Runtime: - async def start(self, payload: dict[str, Any]): - del payload - - async def invoke(self, request: Any, context: Any): - del request, context - raise AssertionError("not used") - - async def stop(self): - pass - - -class _SlowHealthRuntime(_Runtime): - async def health(self, request: AdapterHealthRequest): - del request - await asyncio.sleep(1) - raise AssertionError("health hook should time out") - - -class _ReadyWhileBusyRuntime(_Runtime): - async def health(self, request: AdapterHealthRequest): - del request - return AdapterHealthResult( - readiness=AdapterReadiness( - state=RuntimeReadiness.READY, - reason_code="concurrent_invocations_supported", - ) - ) - - -class _BlockingStopRuntime(_Runtime): - def __init__(self, started: asyncio.Event, release: asyncio.Event) -> None: - self._started = started - self._release = release - - async def stop(self): - self._started.set() - await self._release.wait() +def _runtime_mock(*, health: AsyncMock | None = None) -> MagicMock: + methods = ["start", "invoke", "stop"] + if health is not None: + methods.append("health") + runtime = MagicMock(spec_set=methods) + runtime.start = AsyncMock() + runtime.invoke = AsyncMock(side_effect=AssertionError("not used")) + runtime.stop = AsyncMock() + if health is not None: + runtime.health = health + return runtime async def _check_health(control: dict[str, Any], runtime_id: str) -> dict[str, Any]: @@ -85,7 +60,7 @@ async def _check_health(control: dict[str, Any], runtime_id: str) -> dict[str, A async def test_python_health_control_reports_busy_without_lifecycle_channel(): state = _HostState( - runtime=_Runtime(), + runtime=_runtime_mock(), runtime_id="runtime-1", invoking=True, ) @@ -108,7 +83,15 @@ async def test_python_health_control_reports_busy_without_lifecycle_channel(): async def test_python_adapter_health_hook_timeout_is_data(): - state = _HostState(runtime=_SlowHealthRuntime(), runtime_id="runtime-1") + async def slow_health(request: AdapterHealthRequest): + del request + await asyncio.sleep(1) + raise AssertionError("health hook should time out") + + state = _HostState( + runtime=_runtime_mock(health=AsyncMock(side_effect=slow_health)), + runtime_id="runtime-1", + ) result = await _adapter_health( state, @@ -122,8 +105,14 @@ async def test_python_adapter_health_hook_timeout_is_data(): async def test_python_adapter_health_preserves_busy_runtime_readiness(): + readiness = AdapterHealthResult( + readiness=AdapterReadiness( + state=RuntimeReadiness.READY, + reason_code="concurrent_invocations_supported", + ) + ) state = _HostState( - runtime=_ReadyWhileBusyRuntime(), + runtime=_runtime_mock(health=AsyncMock(return_value=readiness)), runtime_id="runtime-1", invoking=True, ) @@ -141,28 +130,37 @@ async def test_python_adapter_health_preserves_busy_runtime_readiness(): async def test_python_health_reports_stop_in_progress(): stop_started = asyncio.Event() release_stop = asyncio.Event() - runtime = _BlockingStopRuntime(stop_started, release_stop) + + async def blocking_stop(): + stop_started.set() + await release_stop.wait() + + runtime = _runtime_mock() + runtime.stop.side_effect = blocking_stop state = _HostState(runtime=runtime, runtime_id="runtime-1") control = (await _start_health_server(state))["health_control"] stopping = asyncio.create_task(_handle_stop(state, runtime)) await stop_started.wait() - response = await _check_health(control, "runtime-1") + try: + response = await _check_health(control, "runtime-1") - assert response["result"]["readiness"] == { - "state": "not_ready", - "reason_code": "stop_in_progress", - } - release_stop.set() - await stopping + assert response["result"]["readiness"] == { + "state": "not_ready", + "reason_code": "stop_in_progress", + } + finally: + release_stop.set() + await stopping async def test_python_host_skips_health_server_when_capability_is_disabled(): state = _HostState() + runtime = _runtime_mock() response = await _handle_start( state, - _Runtime, + lambda: runtime, {"health_enabled": False}, "runtime-1", None, @@ -174,6 +172,39 @@ async def test_python_host_skips_health_server_when_capability_is_disabled(): await _handle_stop(state, state.runtime) +async def test_python_close_health_server_drains_connections_before_wait_closed(): + events: list[str] = [] + connection_started = asyncio.Event() + release_connection = asyncio.Event() + + async def connection(): + connection_started.set() + try: + await release_connection.wait() + finally: + events.append("connection_closed") + + task = asyncio.create_task(connection()) + await connection_started.wait() + server = MagicMock() + server.close.side_effect = lambda: events.append("server_closed") + + async def wait_closed(): + assert task.done() + events.append("server_waited") + + server.wait_closed = AsyncMock(side_effect=wait_closed) + state = _HostState(health_server=server, health_token="secret") + state.health_tasks.add(task) + + await _close_health_server(state) + + assert events == ["server_closed", "connection_closed", "server_waited"] + assert state.health_server is None + assert state.health_token is None + assert not state.health_tasks + + async def test_adapter_calls_do_not_rebind_stdout_when_they_overlap(): original_stdout = sys.stdout first_started = asyncio.Event() diff --git a/tests/python/test_runtime.py b/tests/python/test_runtime.py index ffbc8d10e..492e2ccaf 100644 --- a/tests/python/test_runtime.py +++ b/tests/python/test_runtime.py @@ -268,6 +268,32 @@ async def test_runtime_health_returns_typed_report(mock_native: MagicMock): assert runtime.status is RuntimeStatus.ACTIVE +async def test_runtime_health_maps_malformed_native_report_to_runtime_error( + mock_native: MagicMock, +): + mock_native.check_runtime_health.side_effect = None + mock_native.check_runtime_health.return_value = json.dumps( + {"runtime_id": "runtime-1"} + ) + runtime = _runtime_wrapper(mock_native) + + with pytest.raises(FabricRuntimeError, match="checked at millis") as caught: + await runtime.check_health() + + assert caught.value.stage == "health" + + +async def test_runtime_health_preserves_native_fabric_errors(mock_native: MagicMock): + expected = FabricStateError("native health state") + mock_native.check_runtime_health.side_effect = expected + runtime = _runtime_wrapper(mock_native) + + with pytest.raises(FabricStateError) as caught: + await runtime.check_health() + + assert caught.value is expected + + async def test_negative_runtime_health_is_data(mock_native: MagicMock): mock_native.check_runtime_health.return_value = json.dumps( _health(readiness="unknown", reason_code="probe_timed_out")