diff --git a/.env.example b/.env.example index 956dd2a..12a6897 100644 --- a/.env.example +++ b/.env.example @@ -56,6 +56,9 @@ MODELPORT_ADMIN_PASSWORD=replace-with-a-long-random-admin-password # Optional for a confidential client; omit for a supported public client. # MODELPORT_OIDC_CLIENT_SECRET=replace-with-client-secret # MODELPORT_OIDC_LABEL=Company SSO +# After verifying a linked SSO administrator, optionally enforce SSO/MFA class: +# MODELPORT_PASSWORD_LOGIN_ENABLED=0 +# MODELPORT_OIDC_REQUIRED_ACR=urn:example:authentication:mfa # Automatic user provisioning is off by default; pre-create users initially. # MODELPORT_OIDC_AUTO_PROVISION=0 # MODELPORT_OIDC_USERNAME_CLAIM=preferred_username diff --git a/.github/MAINTAINERS.md b/.github/MAINTAINERS.md index 85d1042..29b30cd 100644 --- a/.github/MAINTAINERS.md +++ b/.github/MAINTAINERS.md @@ -25,6 +25,17 @@ Protect `main` with: - restricted force pushes and branch deletion; - signed commits and tags where the signing setup is available. +The Beta repository enforces PRs, the five GitHub Actions checks (including both +CodeQL languages), current branches and resolved conversations for administrators +too. Force pushes and branch deletion are disabled. The independent-approval +count is currently zero: a second release/security maintainer has not completed +the continuity gate below. Do not describe this as enforced two-person review. +Raise the required review count and enable CODEOWNERS review after that handoff. +The active version-tag ruleset prohibits updating or deleting `v*` tags without +a bypass. Private vulnerability reporting, automatic Dependabot security +updates and immutable publication are enabled. Existing older Releases are not +retroactively made immutable by that setting. + Routine CI uses the pinned Rust and Node versions, locked dependencies, fmt/test/clippy, dashboard checks, shell syntax, examples, and documentation links through `scripts/check-all.sh`. Paid Provider tests require an explicit, diff --git a/.github/SECURITY.md b/.github/SECURITY.md index aada161..8af1178 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -12,10 +12,13 @@ Older snapshots may not receive backports unless a release notice says so. ## Dependency Audit Exceptions -Rust and dashboard dependency audits fail CI. A temporary exception is allowed -only in `dashboard/npm-audit-exceptions.json`, with an exact advisory/package -pair, an expiry date, and a deployment-specific exposure analysis. The audit -fails for every unlisted advisory, expired exception, or stale exception. +Rust and dashboard dependency audits fail CI. Dashboard exceptions live in +`dashboard/npm-audit-exceptions.json`, with an exact advisory/package pair, +an expiry date, and a deployment-specific exposure analysis. Rust exceptions +are recorded in `.cargo/audit.toml` and `deny.toml`; the current RSA exception +covers OIDC public-key verification, with the dependency path and removal +condition documented there. Unlisted advisories fail CI; dashboard exceptions +also fail when expired or stale. Exceptions are risk acceptance records, not claims that an affected package is generally safe. @@ -63,6 +66,11 @@ Do not place exploit details, provider keys, session tokens, backups, or a full process-local and reset on restart. - Session cookies are HttpOnly and SameSite=Lax. Set `MODELPORT_ADMIN_COOKIE_SECURE=1` whenever the dashboard is served over HTTPS. +- `MODELPORT_PASSWORD_LOGIN_ENABLED=0` enforces OIDC-only login at the backend. + Startup requires an active administrator already linked to that issuer. + `MODELPORT_OIDC_REQUIRED_ACR` additionally requires an exact signed `acr` + claim and forbids password fallback. Configure the identity provider's MFA + policy for that class; ModelPort does not implement an MFA factor itself. - Dashboard writes require a session, `X-ModelPort-CSRF`, and an allowed Origin/Referer when present. `MODELPORT_ALLOWED_ORIGINS` extends that write check; it does not enable browser CORS. @@ -91,10 +99,10 @@ Do not place exploit details, provider keys, session tokens, backups, or a full internal upstream; local/custom runtimes retain HTTP support for controlled local integration. The HTTP override does not disable private/metadata-IP protection. -- Hostnames are not currently pinned or revalidated after DNS resolution. A - hostname that resolves to an internal address is outside the current SSRF - guard. Use outbound firewall rules or an allowlist when administrators are not - fully trusted. +- Provider hostnames are resolved and every resulting address is checked before + egress. Validated addresses are pinned to the HTTP connection, environment + proxies are disabled, and private/metadata addresses require explicit policy. + Keep outbound firewall rules or an allowlist as an independent control. - Upstream HTTP redirects are disabled and every GET/POST/SSE handshake requires 2xx. A 3xx is treated as an upstream failure and mapped to client-facing 502, not followed or exposed as a client redirect. Upstream non-stream bodies and diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bfb83d6..19a679a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,6 +170,7 @@ jobs: permissions: contents: read + attestations: read services: postgres: @@ -268,6 +269,36 @@ jobs: sed -n '1,200p' "${RUNNER_TEMP}/modelport-e2e.log" exit 1 + - name: Verify previous release rollback artifact + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p "$RUNNER_TEMP/modelport-rollback" + gh release download v0.1.1 --repo tiammomo/ModelPort \ + --pattern model-port-v0.1.1-linux-amd64.tar.gz --pattern SHA256SUMS \ + --dir "$RUNNER_TEMP/modelport-rollback" + cd "$RUNNER_TEMP/modelport-rollback" + sha256sum --check --ignore-missing SHA256SUMS + gh attestation verify model-port-v0.1.1-linux-amd64.tar.gz --repo tiammomo/ModelPort + tar -xzf model-port-v0.1.1-linux-amd64.tar.gz \ + model-port-v0.1.1-linux-amd64/model-port + + - name: Run isolated authentication, protocol and recovery acceptance + env: + MODELPORT_TEST_BINARY: ${{ github.workspace }}/target/debug/model-port + MODELPORT_ROLLBACK_TEST_BINARY: ${{ runner.temp }}/modelport-rollback/model-port-v0.1.1-linux-amd64/model-port + MODELPORT_ASSURANCE_OUTPUT_DIR: ${{ runner.temp }}/modelport-assurance + run: ./scripts/acceptance.sh --isolated + + - name: Upload isolated acceptance evidence + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: isolated-production-acceptance + path: ${{ runner.temp }}/modelport-assurance + if-no-files-found: ignore + retention-days: 14 + - name: Run dashboard E2E run: npm --prefix dashboard run e2e diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dc50837..b8646aa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,6 +25,14 @@ jobs: uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.0.2 with: persist-credentials: false + fetch-depth: 0 + + - name: Require a commit on protected main + env: + GH_TOKEN: ${{ github.token }} + run: | + git merge-base --is-ancestor HEAD origin/main + test "$(gh api "repos/$GITHUB_REPOSITORY/branches/main" --jq '.protected')" = true - name: Match tag to backend and dashboard versions run: | @@ -50,6 +58,7 @@ jobs: uses: ./.github/workflows/ci.yml permissions: contents: read + attestations: read binary: name: Build and attest binary @@ -305,4 +314,10 @@ jobs: --verify-tag \ --generate-notes \ --title "$release_title" \ + --draft \ "${release_flags[@]}" + # All assets must be present before immutable publication locks them. + expected="$(find dist -maxdepth 1 -type f | wc -l)" + actual="$(gh release view "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" --json assets --jq '.assets | length')" + test "$actual" -eq "$expected" + gh release edit "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" --draft=false diff --git a/CHANGELOG.md b/CHANGELOG.md index 8452b52..84b5060 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,33 @@ All notable ModelPort changes are recorded here. The project follows ## [Unreleased] +## [0.1.2] - 2026-09-07 + +### Authentication and production acceptance + +- Add optional backend-enforced SSO-only login and exact signed OIDC `acr` + requirements. Reject password fallback when assurance is required and refuse + SSO-only startup without an active administrator linked to the configured + issuer. Existing deployments retain password login unless explicitly changed. +- Add isolated signed OIDC/PKCE, Messages/Chat Completions Tool Use, distinct-user + concurrent streaming, cancellation, database interruption, restore and actual + v0.1.1 binary rollback acceptance to the PostgreSQL CI gate. Record synthetic + evidence separately from real Provider, GPU, IdP and production RTO/RPO claims. +- Require release commits on protected main and upload draft assets before + immutable publication. Repository branch/tag protection, vulnerability + reporting and automated security-update settings are enabled separately. +- Correct stale DNS-pinning, migration and production-readiness documentation. + +### Internal maintenance + +- Group repository policies, fixtures and deployment assets by ownership. +- Separate ledger reporting and provider credential management, share + credential readiness and correct credential-pool readiness in route summaries. +- Consolidate native lifecycle and checks in `scripts/dev.sh`, preserve legacy + entries, resolve commands from their checkout and stop only owned processes. +- Include embedded resources and workspace inputs in native build freshness + checks; centralize repeated development and troubleshooting procedures. + ## [0.1.1] - 2026-09-06 ### Release correction @@ -116,6 +143,7 @@ Back up PostgreSQL and run a restore drill before upgrading. Compose still uses the PostgreSQL 18 volume `modelport_modelport-postgres-18`; export any older volume before removing it. -[Unreleased]: https://github.com/tiammomo/ModelPort/compare/v0.1.1...HEAD +[Unreleased]: https://github.com/tiammomo/ModelPort/compare/v0.1.2...HEAD +[0.1.2]: https://github.com/tiammomo/ModelPort/releases/tag/v0.1.2 [0.1.1]: https://github.com/tiammomo/ModelPort/releases/tag/v0.1.1 [0.1.0]: https://github.com/tiammomo/ModelPort/tree/v0.1.0 diff --git a/Cargo.lock b/Cargo.lock index 765308b..81e009e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1391,7 +1391,7 @@ dependencies = [ [[package]] name = "model-port" -version = "0.1.1" +version = "0.1.2" dependencies = [ "argon2", "async-stream", @@ -1419,7 +1419,7 @@ dependencies = [ [[package]] name = "modelport-ops-agent" -version = "0.1.1" +version = "0.1.2" dependencies = [ "anyhow", "axum", @@ -1436,7 +1436,7 @@ dependencies = [ [[package]] name = "modelport-ops-protocol" -version = "0.1.1" +version = "0.1.2" dependencies = [ "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 0fc909a..5dec980 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "model-port" -version = "0.1.1" +version = "0.1.2" edition = "2024" rust-version = "1.96" description = "A self-hosted multi-protocol model gateway for Anthropic and OpenAI-compatible workflows." @@ -29,7 +29,7 @@ axum = { version = "0.8", features = ["macros"] } futures-util = "0.3" httpdate = "1" jsonschema = { version = "0.48", default-features = false } -modelport-ops-protocol = { version = "0.1.1", path = "crates/ops-protocol" } +modelport-ops-protocol = { version = "0.1.2", path = "crates/ops-protocol" } openidconnect = { version = "4.0.1", default-features = false, features = ["reqwest", "rustls-tls"] } rand_core = { version = "0.6", features = ["getrandom"] } reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] } diff --git a/Dockerfile b/Dockerfile index 27dfcbc..03af71d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 ARG RUST_VERSION=1.96.0 -ARG MODELPORT_VERSION=0.1.1 +ARG MODELPORT_VERSION=0.1.2 ARG MODELPORT_SOURCE_REVISION=unknown ARG MODELPORT_SOURCE_STATE=unknown ARG MODELPORT_BUILD_DATE=unknown diff --git a/crates/ops-agent/Cargo.toml b/crates/ops-agent/Cargo.toml index dd28e39..ce8b48b 100644 --- a/crates/ops-agent/Cargo.toml +++ b/crates/ops-agent/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "modelport-ops-agent" -version = "0.1.1" +version = "0.1.2" edition = "2024" rust-version = "1.96" description = "Deterministic, read-only operations agent for ModelPort." @@ -10,7 +10,7 @@ publish = false [dependencies] anyhow = "1" axum = "0.8" -modelport-ops-protocol = { version = "0.1.1", path = "../ops-protocol" } +modelport-ops-protocol = { version = "0.1.2", path = "../ops-protocol" } reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } serde_json = "1" sha2 = "0.10" diff --git a/crates/ops-agent/Dockerfile b/crates/ops-agent/Dockerfile index 5b30b58..9318785 100644 --- a/crates/ops-agent/Dockerfile +++ b/crates/ops-agent/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 ARG RUST_VERSION=1.96.0 -ARG MODELPORT_VERSION=0.1.1 +ARG MODELPORT_VERSION=0.1.2 ARG MODELPORT_SOURCE_REVISION=unknown ARG MODELPORT_SOURCE_STATE=unknown ARG MODELPORT_BUILD_DATE=unknown diff --git a/crates/ops-protocol/Cargo.toml b/crates/ops-protocol/Cargo.toml index 3b88d18..c143317 100644 --- a/crates/ops-protocol/Cargo.toml +++ b/crates/ops-protocol/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "modelport-ops-protocol" -version = "0.1.1" +version = "0.1.2" edition = "2024" rust-version = "1.96" license = "MIT" diff --git a/dashboard/Dockerfile b/dashboard/Dockerfile index 85dca42..323bf75 100644 --- a/dashboard/Dockerfile +++ b/dashboard/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG MODELPORT_VERSION=0.1.1 +ARG MODELPORT_VERSION=0.1.2 ARG MODELPORT_SOURCE_REVISION=unknown ARG MODELPORT_SOURCE_STATE=unknown ARG MODELPORT_BUILD_DATE=unknown diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json index 9b48a3e..f86b791 100644 --- a/dashboard/package-lock.json +++ b/dashboard/package-lock.json @@ -1,12 +1,12 @@ { "name": "dashboard", - "version": "0.1.1", + "version": "0.1.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dashboard", - "version": "0.1.1", + "version": "0.1.2", "dependencies": { "@radix-ui/react-avatar": "^1.1.12", "@radix-ui/react-dialog": "^1.1.16", diff --git a/dashboard/package.json b/dashboard/package.json index 61e1cb1..7b674d2 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -1,7 +1,7 @@ { "name": "dashboard", "private": true, - "version": "0.1.1", + "version": "0.1.2", "type": "module", "scripts": { "dev": "vite", diff --git a/deploy/docker/modelport.env.example b/deploy/docker/modelport.env.example index 5f944d2..8caf6f6 100644 --- a/deploy/docker/modelport.env.example +++ b/deploy/docker/modelport.env.example @@ -42,6 +42,9 @@ MODELPORT_ADMIN_SESSION_TTL_SECONDS=43200 # Optional confidential-client secret. # MODELPORT_OIDC_CLIENT_SECRET=replace-with-client-secret # MODELPORT_OIDC_LABEL=Company SSO +# After verifying a linked SSO administrator, optionally enforce SSO/MFA class: +# MODELPORT_PASSWORD_LOGIN_ENABLED=0 +# MODELPORT_OIDC_REQUIRED_ACR=urn:example:authentication:mfa # Automatic provisioning is off by default; pre-create users initially. # MODELPORT_OIDC_AUTO_PROVISION=0 # MODELPORT_OIDC_USERNAME_CLAIM=preferred_username diff --git a/deploy/release/compose.yml b/deploy/release/compose.yml index 61884c3..605db91 100644 --- a/deploy/release/compose.yml +++ b/deploy/release/compose.yml @@ -31,7 +31,7 @@ services: max-file: "${MODELPORT_LOG_MAX_FILES:-5}" modelport: - image: ${MODELPORT_IMAGE:-ghcr.io/tiammomo/modelport:0.1.1} + image: ${MODELPORT_IMAGE:-ghcr.io/tiammomo/modelport:0.1.2} pull_policy: ${MODELPORT_PULL_POLICY:-missing} init: true read_only: true @@ -86,7 +86,7 @@ services: max-file: "${MODELPORT_LOG_MAX_FILES:-5}" dashboard: - image: ${MODELPORT_DASHBOARD_IMAGE:-ghcr.io/tiammomo/modelport-dashboard:0.1.1} + image: ${MODELPORT_DASHBOARD_IMAGE:-ghcr.io/tiammomo/modelport-dashboard:0.1.2} pull_policy: ${MODELPORT_PULL_POLICY:-missing} init: true read_only: true @@ -108,7 +108,7 @@ services: ops-agent: profiles: ["ops-agent"] - image: ${MODELPORT_OPS_AGENT_IMAGE:-ghcr.io/tiammomo/modelport-ops-agent:0.1.1} + image: ${MODELPORT_OPS_AGENT_IMAGE:-ghcr.io/tiammomo/modelport-ops-agent:0.1.2} pull_policy: ${MODELPORT_PULL_POLICY:-missing} init: true read_only: true diff --git a/deploy/systemd/modelport.env.example b/deploy/systemd/modelport.env.example index 3084468..62f9059 100644 --- a/deploy/systemd/modelport.env.example +++ b/deploy/systemd/modelport.env.example @@ -29,6 +29,9 @@ MODELPORT_ADMIN_SESSION_TTL_SECONDS=43200 # Optional confidential-client secret. # MODELPORT_OIDC_CLIENT_SECRET=replace-with-client-secret # MODELPORT_OIDC_LABEL=Company SSO +# After verifying a linked SSO administrator, optionally enforce SSO/MFA class: +# MODELPORT_PASSWORD_LOGIN_ENABLED=0 +# MODELPORT_OIDC_REQUIRED_ACR=urn:example:authentication:mfa # Automatic provisioning is off by default; pre-create users initially. # MODELPORT_OIDC_AUTO_PROVISION=0 # MODELPORT_OIDC_USERNAME_CLAIM=preferred_username diff --git a/docker-compose.yml b/docker-compose.yml index 86eba5a..86886ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ services: context: . dockerfile: Dockerfile args: - MODELPORT_VERSION: ${MODELPORT_VERSION:-0.1.1} + MODELPORT_VERSION: ${MODELPORT_VERSION:-0.1.2} MODELPORT_SOURCE_REVISION: ${MODELPORT_SOURCE_REVISION:-unknown} MODELPORT_SOURCE_STATE: ${MODELPORT_SOURCE_STATE:-unknown} MODELPORT_BUILD_DATE: ${MODELPORT_BUILD_DATE:-unknown} @@ -99,7 +99,7 @@ services: context: . dockerfile: dashboard/Dockerfile args: - MODELPORT_VERSION: ${MODELPORT_VERSION:-0.1.1} + MODELPORT_VERSION: ${MODELPORT_VERSION:-0.1.2} MODELPORT_SOURCE_REVISION: ${MODELPORT_SOURCE_REVISION:-unknown} MODELPORT_SOURCE_STATE: ${MODELPORT_SOURCE_STATE:-unknown} MODELPORT_BUILD_DATE: ${MODELPORT_BUILD_DATE:-unknown} @@ -129,7 +129,7 @@ services: context: . dockerfile: crates/ops-agent/Dockerfile args: - MODELPORT_VERSION: ${MODELPORT_VERSION:-0.1.1} + MODELPORT_VERSION: ${MODELPORT_VERSION:-0.1.2} MODELPORT_SOURCE_REVISION: ${MODELPORT_SOURCE_REVISION:-unknown} MODELPORT_SOURCE_STATE: ${MODELPORT_SOURCE_STATE:-unknown} MODELPORT_BUILD_DATE: ${MODELPORT_BUILD_DATE:-unknown} diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index dbf169d..7ba129c 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -401,12 +401,14 @@ direct path and cannot govern its usage or balance. | `MODELPORT_ADMIN_EMAIL` | `admin@modelport.local` | First-admin bootstrap email. | | `MODELPORT_ADMIN_SESSION_TTL_SECONDS` | `43200` | Dashboard session lifetime. | | `MODELPORT_ADMIN_COOKIE_SECURE` | off | Add `Secure` to the dashboard cookie. Set to `1` behind HTTPS. | +| `MODELPORT_PASSWORD_LOGIN_ENABLED` | `1` | Set to `0` to reject password login at the backend. Requires configured OIDC and an active administrator already linked to that issuer at startup. | | `MODELPORT_REQUIRE_DUAL_APPROVAL` | off; always on in enterprise mode | Require an approved change request from two distinct administrators before high-risk identity, Provider, model, or hard-budget writes. Small-Team mode otherwise relies on the administrator session, CSRF protection, and audit trail so a one-admin first install remains operable. | | `MODELPORT_OIDC_ISSUER` | unset | OIDC issuer discovery URL. OIDC console sign-in stays disabled when no OIDC values are configured. | | `MODELPORT_OIDC_CLIENT_ID` | unset | OIDC client identifier; required with issuer and redirect URI when OIDC is enabled. | | `MODELPORT_OIDC_CLIENT_SECRET` | unset | Optional confidential-client secret. Leave unset only when the identity provider accepts the supported public-client code exchange. | | `MODELPORT_OIDC_REDIRECT_URI` | unset | Exact external callback URL; its path must be `/admin/auth/oidc/callback` with no query or fragment. | | `MODELPORT_OIDC_LABEL` | `Single sign-on` | Login-button label. | +| `MODELPORT_OIDC_REQUIRED_ACR` | unset | Exact signed OIDC authentication-class claim required for login. Sends `acr_values` and rejects missing/mismatched claims. Requires password login disabled; the identity provider must enforce the corresponding MFA policy. | | `MODELPORT_OIDC_AUTO_PROVISION` | off | Create missing ordinary users after a valid OIDC login. Keep off initially and pre-create users; it never grants administrator access. | | `MODELPORT_OIDC_USERNAME_CLAIM` | `preferred_username` | ID-token claim used as the ModelPort username. | | `MODELPORT_OIDC_EMAIL_CLAIM` | `email` | ID-token claim read as the ModelPort email. Initial linking/JIT requires the standard `email` claim plus `email_verified=true`; verification is not inherited by a custom claim name. | diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index ca4138f..9d3d8b0 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -212,9 +212,16 @@ Runtime verification: scripts/dev.sh doctor scripts/smoke-test.sh scripts/acceptance.sh +scripts/acceptance.sh --isolated scripts/tool-use-acceptance.sh ``` +`--isolated` creates a disposable PostgreSQL database and loopback identity/model +servers, then exercises signed OIDC, both client protocols, bounded load and +recovery. It needs Docker and Node and builds the current gateway with Cargo. +See [production acceptance](PRODUCTION.md#automated-acceptance) for evidence and +rollback options. It does not load the local deployment's environment file. + Commands with `--upstream`, plus `provider-matrix.sh`, make real provider calls and may incur cost. Use mock-backed Tool Use acceptance for routine adapter work. @@ -224,7 +231,7 @@ and may incur cost. Use mock-backed Tool Use acceptance for routine adapter work | --- | --- | | Protocol/request/response mapping | Rust tests, smoke; provider matrix for the affected provider. | | SSE or Tool Use | Rust stream tests and `tool-use-acceptance.sh`; real upstream only for certification. | -| Auth/policy/quota | Rust tests and `acceptance.sh`. | +| Auth/policy/quota | Rust tests, `acceptance.sh` and `acceptance.sh --isolated`. | | Provider catalog/defaults | Config validation, `/v1/models`, provider matrix, docs catalog update. | | Dashboard behavior | lint, build, affected Playwright specs. | | Docker/systemd/reverse proxy | Render/build the deployment and run smoke through the deployed origin. | diff --git a/docs/OIDC.md b/docs/OIDC.md index c441eaa..941f9b7 100644 --- a/docs/OIDC.md +++ b/docs/OIDC.md @@ -113,6 +113,37 @@ the user or to a server-side BFF. ## Operational Notes +### Require SSO And A Verified Authentication Class + +First exercise OIDC with password login still enabled, bind an ordinary user +through its verified email, and explicitly promote that linked identity to +administrator. Verify the administrator's SSO access before setting: + +```env +MODELPORT_PASSWORD_LOGIN_ENABLED=0 +# Use the exact class your identity provider defines and enforces with MFA. +MODELPORT_OIDC_REQUIRED_ACR=urn:example:authentication:mfa +``` + +The backend refuses password login, including direct API requests. Startup +fails if there is no active administrator linked to the configured issuer. +When an authentication class is set, the authorization request includes +`acr_values`; the returned, signature-verified ID token must contain that exact +`acr` value. Missing or lower/different claims fail before session creation. +This setting cannot be combined with enabled password login. An `acr` string +has meaning only under the operator's verified identity-provider policy; it +does not itself prove an MFA factor was configured correctly. + +`MODELPORT_PASSWORD_LOGIN_ENABLED` defaults to `1` for existing deployments. +Changes require a restart. An operator with server configuration access can +restore the bootstrap password path by setting it back to `1` and removing +the required ACR setting, then restarting during an approved recovery window. +Keep that access controlled and audit the recovery. No HTTP recovery bypass +is provided. The login-method probe may offer a password retry during a network +failure; the backend still enforces the configured policy. + +### Session And Provider Boundaries + - OIDC authorization state and ModelPort console sessions are process-local in the current release. A restart invalidates in-progress login flows and active sessions. @@ -131,6 +162,17 @@ the user or to a server-side BFF. - Keep Provider API keys in the ModelPort server environment or an external secret manager. Never expose them to the browser. +### Automated And Real-Provider Acceptance + +`scripts/acceptance.sh --isolated` runs a real authorization-code/PKCE exchange +against a loopback identity provider with an ephemeral RSA signing key. It +checks issuer, audience, nonce, expiry, signature, access-token hash, browser +binding, replay rejection, disabled identities, and SSO/ACR enforcement. +Use [Production](PRODUCTION.md#deployment-specific-evidence) to record the +separate acceptance of your actual identity provider, MFA policy, account +offboarding and operator recovery. Identity-provider single logout remains +outside this release's supported contract. + ## Troubleshooting | Symptom | Check | diff --git a/docs/PRODUCTION.md b/docs/PRODUCTION.md index 985d0c8..59be836 100644 --- a/docs/PRODUCTION.md +++ b/docs/PRODUCTION.md @@ -23,9 +23,10 @@ name of a fail-closed configuration switch, not an enterprise-readiness claim. The accepted forty-user hybrid-routing target is defined in [ADR-0005](adr/0005-forty-user-hybrid-routing-baseline.md). Its first phase -still uses one ModelPort instance. Routing modes, per-user queue fairness, -managed secrets, and active-active operation remain target behavior until their -individual implementation and acceptance gates pass. +still uses one ModelPort instance. Routing modes and per-user queue rules have +implementation and automated acceptance. Managed-secret injection is operator +owned, and active-active operation remains unsupported. These implemented +rules do not establish a particular real-model throughput or latency. ## Go-Live Checklist @@ -49,6 +50,30 @@ individual implementation and acceptance gates pass. ## Automated Acceptance +Run the isolated runtime gate on a Linux host with Docker, Node and the pinned +Rust toolchain: + +```bash +MODELPORT_ASSURANCE_OUTPUT_DIR=/tmp/modelport-assurance scripts/acceptance.sh --isolated +``` + +It creates and removes its own PostgreSQL container, signs OIDC tokens with an +ephemeral key, and uses only loopback synthetic model responses. It tests both +Messages and Chat Completions text, live streams and complete Tool Use turns; +40 distinct scoped users across 400 paced requests; stream cancellation and +truncation; process restart; database interruption without unrecorded egress; +and restored auth/control fingerprints plus ledger row counts. CI additionally +verifies the checksum and GitHub attestation of v0.1.1, then runs that actual +binary against the restored database for paired application rollback. + +Evidence contains commit/source state, latency distributions, rejection counts +and recovery outcomes, with no credentials or conversation content. Set +`MODELPORT_ASSURANCE_LOAD_SECONDS=60` for a longer paced run (1–120 seconds; +the request budget remains 400). This synthetic gate validates gateway behavior, +not production model capacity or production RTO/RPO. `capacity-acceptance.sh` +separately checks policy unit invariants. Neither script certifies a real GPU +or cloud Provider. + Run configuration validation before starting or restarting the candidate: ```bash @@ -111,6 +136,23 @@ Keep: A fixture-backed pass supports a controlled gateway trial. A dated Provider pass supports only the exact model, path, account conditions, and commit tested. +## Deployment-Specific Evidence + +Keep the following as pending until the named deployment has produced and +retained the evidence. Repository CI cannot complete these rows for an operator. + +| Gate | Required evidence | +| --- | --- | +| Identity | Actual issuer/client/callback and HTTPS proxy; enforced MFA class; disabled-user behavior; session lifetime; tested operator recovery. | +| Provider protocols | Exact image digest, model, endpoint, account and date; both required client protocols; text, stream completion, tool-result continuation, failures and cancellation; an explicit request/cost cap. | +| Capacity | Actual host/GPU/model and workload; 40 separate identities; sustained concurrency, semantic TTFT/P95, queue/rejection counts, CPU/memory/DB pool use; agreed pass thresholds. | +| Recovery | Encrypted off-host backup, managed PostgreSQL TLS/PITR, timed restore and paired application rollback; operator-approved RTO/RPO. | +| Operations | Alerts delivered to an assigned owner, backup operator and maintenance window; candidate digest recorded and post-deployment smoke passed. | + +Routine acceptance uses fixtures. Run paid upstream checks only for a named +account/model with an explicit budget. A published Release or merged `main` +commit does not establish which digest a production host is running. + ## Reliability Objectives ModelPort does not publish a universal end-to-end SLO because Provider diff --git a/docs/PRODUCTION_BASELINE_40_USERS.zh-CN.md b/docs/PRODUCTION_BASELINE_40_USERS.zh-CN.md index eb5d7e3..acda583 100644 --- a/docs/PRODUCTION_BASELINE_40_USERS.zh-CN.md +++ b/docs/PRODUCTION_BASELINE_40_USERS.zh-CN.md @@ -44,6 +44,12 @@ Linux/WSL2 中运行不产生真实模型请求的容量基线: ./scripts/capacity-acceptance.sh ``` +这条命令只验证准入规则。实际 HTTP、鉴权、流式、40 个独立用户的并发与恢复验收使用 +`scripts/acceptance.sh --isolated`;可设置 `MODELPORT_ASSURANCE_LOAD_SECONDS=60` +进行 60 秒的分批持续运行。它使用隔离 PostgreSQL 和本地合成响应,不能作为真实 GPU、 +云模型吞吐或生产 RTO/RPO 的证明。实际部署所需证据统一见 +[投产验收](PRODUCTION.md#deployment-specific-evidence)。 + ## 第一阶段已经建立的保护 ### 1. 数据库更新先检查 diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 05a5e59..d560982 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -19,13 +19,17 @@ with a 30-day upgrade window for the previous Beta. There is no LTS or SLA. CodeQL, dependency review, and Scorecard checks are green. Any temporary dashboard audit exception must still match its documented deployment exposure and remain unexpired. -4. A clean PostgreSQL migration and the old-row rejection fixture have been +4. A clean PostgreSQL migration and historical-row preservation have been verified. 5. Documentation, configuration examples, Compose, systemd, and dashboard use the same versioned contract. 6. Any real-Provider claim has a dated, secret-free, commit-bound evidence artifact. Routine release CI makes no paid Provider calls. -7. The version in `deploy/release/compose.yml` matches the tag, and Linux x86_64 +7. The isolated OIDC/protocol/load/database-fault/restore gate passes, including + rollback to the attested v0.1.1 binary. Evidence is stored by CI as + `isolated-production-acceptance`; this does not replace deployment-specific + Provider, MFA, capacity and RTO/RPO evidence. +8. The version in `deploy/release/compose.yml` matches the tag, and Linux x86_64 install, safe stop, backup/restore, upgrade, and rollback acceptance passes. ## Version And Tag @@ -65,9 +69,14 @@ The release workflow: - publishes versioned backend and dashboard images to GHCR; - publishes Linux x86_64 container SBOMs, signs immutable image digests with keyless Cosign, and attaches GitHub provenance/SBOM attestations; -- records both immutable image references as Release assets; +- records all three immutable image references as Release assets; - creates the GitHub Release from the existing tag. +The tag must resolve to a commit on protected `main`. Publication first creates +a draft and uploads all assets, checks the asset count, then publishes it under +the repository's immutable-release policy. Do not delete or retag a failed +published version; issue a new patch version after fixing its cause. + Release workflows use least-privilege job permissions and pin third-party Actions to complete commit SHAs. @@ -102,9 +111,10 @@ Application rollback and database rollback are separate decisions. backup until production acceptance passes. - Never point an older release at a database after a migration unless that downgrade path was explicitly tested. -- The current clean operational baseline does not import old request/attempt or - JSON state. Roll back by restoring the previous database and application - together. +- The operational migrations preserve historical request/attempt records and + versioned auth/control state. Roll back by restoring the reviewed database + snapshot and compatible application together; do not infer downgrade safety + from a successful forward migration. - The PostgreSQL 18 Compose baseline uses a new `modelport_modelport-postgres-18` volume and the versioned `/var/lib/postgresql/18/docker` data directory. It intentionally does not diff --git a/scripts/acceptance.sh b/scripts/acceptance.sh index 800fe59..f3ac407 100755 --- a/scripts/acceptance.sh +++ b/scripts/acceptance.sh @@ -5,6 +5,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1091 source "$SCRIPT_DIR/lib.sh" +if [[ "${1:-}" == "--isolated" ]]; then + shift + [[ $# -eq 0 ]] || die "--isolated accepts no additional arguments" + exec "$ROOT_DIR/tests/runtime/run.sh" +fi + upstream=0 case "${1:-}" in "") @@ -14,11 +20,13 @@ case "${1:-}" in ;; -h|--help) cat <<'USAGE' -Usage: scripts/acceptance.sh [--upstream] +Usage: scripts/acceptance.sh [--isolated | --upstream] Runs a lightweight production acceptance check for personal and small-team deployments. Default mode does not call the upstream model provider. Use --upstream to also make one real /v1/messages request through the created API key. +Use --isolated for signed OIDC, protocol, load and recovery tests with disposable +PostgreSQL and loopback fixtures; no existing deployment or Provider is contacted. USAGE exit 0 ;; diff --git a/scripts/capacity-acceptance.sh b/scripts/capacity-acceptance.sh index 6bad8e2..cb0251d 100755 --- a/scripts/capacity-acceptance.sh +++ b/scripts/capacity-acceptance.sh @@ -16,12 +16,13 @@ main() { cd "$ROOT_DIR" cargo test --locked governance::tests:: -- --nocapture printf '%s\n' \ - '[modelport-capacity] 40-user admission baseline passed:' \ + '[modelport-capacity] admission policy unit invariants passed (not a runtime load test):' \ ' per-user local execution=1, queued=2' \ ' global interactive queue=16' \ ' local_first/balanced overflow threshold=5s' \ ' local_strict timeout=60s with HTTP 429 Retry-After' \ - ' batch queue=independent low priority' + ' batch queue=independent low priority' \ + ' runtime concurrency/recovery: scripts/acceptance.sh --isolated' } main "$@" diff --git a/scripts/check-all.sh b/scripts/check-all.sh index 82c9f59..0d07f4d 100755 --- a/scripts/check-all.sh +++ b/scripts/check-all.sh @@ -62,7 +62,7 @@ check_shell_lint() { while IFS= read -r -d '' file; do files+=("$file") done < <( - find "$ROOT_DIR/scripts" -type f -name '*.sh' -print0 | sort -z + find "$ROOT_DIR/scripts" "$ROOT_DIR/tests/runtime" -type f -name '*.sh' -print0 | sort -z ) if command -v shellcheck >/dev/null 2>&1; then diff --git a/src/auth.rs b/src/auth.rs index a526ac6..2f6caca 100644 --- a/src/auth.rs +++ b/src/auth.rs @@ -770,6 +770,22 @@ impl AuthStore { .count() } + pub fn has_active_federated_admin(&self, issuer: &str) -> bool { + self.inner + .lock() + .expect("auth lock poisoned") + .users + .values() + .any(|user| { + user.role == "admin" + && user.status == "active" + && user + .federated_identities + .iter() + .any(|identity| identity.issuer == issuer) + }) + } + pub fn session_cookie(&self, token: &str) -> String { let mut cookie = format!( "{ADMIN_SESSION_COOKIE}={token}; Path=/; HttpOnly; SameSite=Lax; Max-Age={}", diff --git a/src/oidc.rs b/src/oidc.rs index b39135f..536332c 100644 --- a/src/oidc.rs +++ b/src/oidc.rs @@ -93,6 +93,7 @@ type ReadyOidcClient = OidcClient< pub struct OidcService { config: Option, + password_enabled: bool, pending: Mutex>, metadata_cache: Mutex>, http_client: SecureHttpClient, @@ -110,6 +111,7 @@ struct OidcConfig { username_claim: String, email_claim: String, allow_insecure_loopback: bool, + required_acr: Option, } struct PendingAuthorization { @@ -228,19 +230,22 @@ impl std::fmt::Debug for OidcService { impl OidcService { pub fn from_env() -> Result { let config = OidcConfig::from_env()?; - Self::new(config) + let password_enabled = password_login_enabled()?; + validate_login_policy(config.as_ref(), password_enabled)?; + Self::new(config, password_enabled) } pub fn validate_configuration() -> Result<(), AppError> { - OidcConfig::from_env().map(|_| ()) + let config = OidcConfig::from_env()?; + validate_login_policy(config.as_ref(), password_login_enabled()?) } #[cfg(test)] pub fn disabled() -> Self { - Self::new(None).expect("disabled OIDC service should always initialize") + Self::new(None, true).expect("disabled OIDC service should always initialize") } - fn new(config: Option) -> Result { + fn new(config: Option, password_enabled: bool) -> Result { let allow_insecure_loopback = config .as_ref() .is_some_and(|config| config.allow_insecure_loopback); @@ -251,6 +256,7 @@ impl OidcService { .map_err(|_| AppError::Config("failed to initialize OIDC HTTP client".to_owned()))?; Ok(Self { config, + password_enabled, pending: Mutex::new(HashMap::new()), metadata_cache: Mutex::new(None), http_client: SecureHttpClient { @@ -263,7 +269,7 @@ impl OidcService { pub fn methods(&self) -> AuthenticationMethods { AuthenticationMethods { - password_enabled: true, + password_enabled: self.password_enabled, oidc: OidcMethod { enabled: self.config.is_some(), label: self @@ -276,12 +282,26 @@ impl OidcService { } } + pub fn validate_console_access(&self, auth: &crate::auth::AuthStore) -> Result<(), AppError> { + if !self.password_enabled + && !self + .config + .as_ref() + .is_some_and(|config| auth.has_active_federated_admin(&config.issuer)) + { + return Err(AppError::Config( + "SSO-only startup requires an active administrator already linked to this OIDC issuer; verify SSO and promote that identity before disabling password login".to_owned(), + )); + } + Ok(()) + } + pub async fn start(&self, return_to: Option<&str>) -> Result { let config = self.config.as_ref().ok_or(OidcFlowError::Disabled)?; let return_to = validate_return_to(return_to.unwrap_or("/"))?; let client = self.ready_client(config).await?; let (pkce_challenge, pkce_verifier) = PkceCodeChallenge::new_random_sha256(); - let (authorization_url, csrf_state, nonce) = client + let mut authorization = client .authorize_url( CoreAuthenticationFlow::AuthorizationCode, CsrfToken::new_random, @@ -289,8 +309,11 @@ impl OidcService { ) .add_scope(Scope::new("profile".to_owned())) .add_scope(Scope::new("email".to_owned())) - .set_pkce_challenge(pkce_challenge) - .url(); + .set_pkce_challenge(pkce_challenge); + if let Some(acr) = config.required_acr.as_deref() { + authorization = authorization.add_extra_param("acr_values", acr); + } + let (authorization_url, csrf_state, nonce) = authorization.url(); let browser_flow = CsrfToken::new_random(); self.insert_pending( @@ -350,6 +373,11 @@ impl OidcService { } let claims_value = serde_json::to_value(claims).map_err(|_| OidcFlowError::InvalidToken)?; + if config.required_acr.as_deref().is_some_and(|required| { + claims_value.get("acr").and_then(Value::as_str) != Some(required) + }) { + return Err(OidcFlowError::InvalidToken); + } Ok(CompletedOidcLogin { issuer: config.issuer.clone(), subject: claims.subject().as_str().to_owned(), @@ -497,10 +525,12 @@ impl OidcConfig { let client_id = env_optional("MODELPORT_OIDC_CLIENT_ID"); let client_secret = env_optional("MODELPORT_OIDC_CLIENT_SECRET"); let redirect_uri = env_optional("MODELPORT_OIDC_REDIRECT_URI"); + let required_acr = env_optional("MODELPORT_OIDC_REQUIRED_ACR"); let any_configured = issuer.is_some() || client_id.is_some() || client_secret.is_some() - || redirect_uri.is_some(); + || redirect_uri.is_some() + || required_acr.is_some(); if !any_configured { return Ok(None); } @@ -565,6 +595,14 @@ impl OidcConfig { .unwrap_or_else(|| DEFAULT_EMAIL_CLAIM.to_owned()); validate_claim_name(&username_claim, "MODELPORT_OIDC_USERNAME_CLAIM")?; validate_claim_name(&email_claim, "MODELPORT_OIDC_EMAIL_CLAIM")?; + if required_acr.as_ref().is_some_and(|acr| { + acr.len() > 256 || acr.chars().any(|ch| ch.is_control() || ch.is_whitespace()) + }) { + return Err(AppError::Config( + "MODELPORT_OIDC_REQUIRED_ACR must be one non-whitespace value of at most 256 bytes" + .to_owned(), + )); + } Ok(Some(Self { issuer, @@ -576,10 +614,38 @@ impl OidcConfig { username_claim, email_claim, allow_insecure_loopback, + required_acr, })) } } +fn password_login_enabled() -> Result { + match env_optional("MODELPORT_PASSWORD_LOGIN_ENABLED").as_deref() { + None | Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") => Ok(true), + Some("0" | "false" | "FALSE" | "no" | "NO" | "off" | "OFF") => Ok(false), + Some(_) => Err(AppError::Config( + "MODELPORT_PASSWORD_LOGIN_ENABLED must be a boolean".to_owned(), + )), + } +} + +fn validate_login_policy( + config: Option<&OidcConfig>, + password_enabled: bool, +) -> Result<(), AppError> { + if !password_enabled && config.is_none() { + return Err(AppError::Config( + "disabling password login requires a configured OIDC provider".to_owned(), + )); + } + if password_enabled && config.is_some_and(|config| config.required_acr.is_some()) { + return Err(AppError::Config( + "MODELPORT_OIDC_REQUIRED_ACR requires MODELPORT_PASSWORD_LOGIN_ENABLED=0 so password login cannot bypass the assurance policy".to_owned(), + )); + } + Ok(()) +} + impl<'c> AsyncHttpClient<'c> for SecureHttpClient { type Error = SecureHttpError; type Future = diff --git a/src/routes.rs b/src/routes.rs index 968c049..242143c 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -1013,6 +1013,11 @@ async fn admin_login( State(state): State, Json(input): Json, ) -> Result { + if !state.oidc.methods().password_enabled { + return Err(AppError::Forbidden( + "password login is disabled; use single sign-on".to_owned(), + )); + } let _permit = tokio::time::timeout( std::time::Duration::from_secs(5), ADMIN_LOGIN_WORKERS.acquire(), diff --git a/src/server.rs b/src/server.rs index 29696a5..2cec665 100644 --- a/src/server.rs +++ b/src/server.rs @@ -89,6 +89,7 @@ pub(crate) async fn serve() -> Result<(), AppError> { retention_previews: Arc::new(RetentionPreviewStore::default()), }; + state.oidc.validate_console_access(&state.auth)?; let listener = TcpListener::bind(bind_addr).await?; info!( %bind_addr, diff --git a/tests/config_validate.rs b/tests/config_validate.rs index 788b1ae..ad08166 100644 --- a/tests/config_validate.rs +++ b/tests/config_validate.rs @@ -92,6 +92,34 @@ fn cli_deployment_preflight_requires_postgres_and_rejects_unsafe_enterprise_tls( assert!(invalid_proxy_text.contains("MODELPORT_TRUSTED_PROXIES")); } +#[test] +fn console_login_policy_rejects_lockout_and_password_assurance_bypass() { + for value in ["0", "false"] { + let output = run_config_validate(&[("MODELPORT_PASSWORD_LOGIN_ENABLED", value)]); + assert!(!output.status.success()); + assert!(output_text(&output).contains("requires a configured OIDC provider")); + } + let invalid = run_config_validate(&[("MODELPORT_PASSWORD_LOGIN_ENABLED", "typo")]); + assert!(!invalid.status.success()); + let oidc = [ + ("MODELPORT_OIDC_ISSUER", "https://identity.example.com"), + ("MODELPORT_OIDC_CLIENT_ID", "modelport"), + ( + "MODELPORT_OIDC_REDIRECT_URI", + "https://modelport.example.com/admin/auth/oidc/callback", + ), + ("MODELPORT_ADMIN_COOKIE_SECURE", "1"), + ("MODELPORT_OIDC_REQUIRED_ACR", "urn:modelport:assurance:mfa"), + ]; + let bypass = run_config_validate(&oidc); + assert!(!bypass.status.success()); + assert!(output_text(&bypass).contains("password login cannot bypass")); + let mut protected = oidc.to_vec(); + protected.push(("MODELPORT_PASSWORD_LOGIN_ENABLED", "0")); + let output = run_config_validate(&protected); + assert!(output.status.success(), "{}", output_text(&output)); +} + #[test] fn cli_deployment_preflight_enforces_the_enterprise_security_profile() { let missing_security = run_config_validate(&[ diff --git a/tests/runtime/oidc.test.mjs b/tests/runtime/oidc.test.mjs new file mode 100644 index 0000000..8a2d37b --- /dev/null +++ b/tests/runtime/oidc.test.mjs @@ -0,0 +1,154 @@ +import assert from 'node:assert/strict' +import { createHash, generateKeyPairSync, sign } from 'node:crypto' +import { test } from 'node:test' +import { adminSession, body, createUser, databaseURL, evidence, gateway, json, secret, server } from './support.mjs' + +async function identityProvider(t) { + const { privateKey, publicKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }) + const wrongKey = generateKeyPairSync('rsa', { modulusLength: 2048 }).privateKey + const jwk = { ...publicKey.export({ format: 'jwk' }), kid: 'assurance', alg: 'RS256', use: 'sig' } + const grants = new Map() + const clientSecret = secret() + let selected = {} + let tokenCalls = 0 + const idp = await server(async (req, res) => { + const url = new URL(req.url, idp.url) + if (url.pathname === '/.well-known/openid-configuration') return json(res, { + issuer: idp.url, authorization_endpoint: `${idp.url}/authorize`, token_endpoint: `${idp.url}/token`, + jwks_uri: `${idp.url}/jwks`, response_types_supported: ['code'], subject_types_supported: ['public'], + id_token_signing_alg_values_supported: ['RS256'], token_endpoint_auth_methods_supported: ['client_secret_basic'], + }) + if (url.pathname === '/jwks') return json(res, { keys: [jwk] }) + if (url.pathname === '/authorize') { + assert.equal(url.searchParams.get('code_challenge_method'), 'S256') + assert.equal(url.searchParams.get('client_id'), 'modelport-assurance') + const code = secret() + grants.set(code, { params: url.searchParams, selected: { ...selected } }) + const redirect = new URL(url.searchParams.get('redirect_uri')) + redirect.searchParams.set('code', code) + redirect.searchParams.set('state', url.searchParams.get('state')) + res.writeHead(302, { location: redirect.href }); return res.end() + } + if (url.pathname === '/token') { + tokenCalls++ + assert.ok(req.headers.authorization === `Basic ${Buffer.from(`modelport-assurance:${clientSecret}`).toString('base64')}`, 'confidential client authenticates at the token endpoint') + const input = new URLSearchParams(await body(req)) + const grant = grants.get(input.get('code')) + grants.delete(input.get('code')) + assert.ok(grant, 'authorization codes are single use') + assert.equal(input.get('grant_type'), 'authorization_code') + assert.equal(input.get('redirect_uri'), grant.params.get('redirect_uri')) + assert.equal(createHash('sha256').update(input.get('code_verifier')).digest('base64url'), grant.params.get('code_challenge')) + const now = Math.floor(Date.now() / 1000) + const claims = { + iss: idp.url, aud: 'modelport-assurance', sub: 'assurance-subject', iat: now, exp: now + 300, + nonce: grant.params.get('nonce'), email_verified: true, + acr: 'urn:modelport:assurance:mfa', ...grant.selected.claims, + } + const accessToken = secret() + claims.at_hash = createHash('sha256').update(accessToken).digest().subarray(0, 16).toString('base64url') + if (grant.selected.badAccessHash) claims.at_hash = 'incorrect-token-hash' + const encode = value => Buffer.from(JSON.stringify(value)).toString('base64url') + const unsigned = `${encode({ alg: 'RS256', kid: 'assurance', typ: 'JWT' })}.${encode(claims)}` + const signature = sign('RSA-SHA256', Buffer.from(unsigned), grant.selected.badSignature ? wrongKey : privateKey).toString('base64url') + return json(res, { access_token: accessToken, token_type: 'Bearer', expires_in: 300, id_token: `${unsigned}.${signature}` }) + } + res.writeHead(404); res.end() + }) + t.after(() => idp.close()) + return { ...idp, clientSecret, select: value => { selected = value }, tokenCalls: () => tokenCalls } +} + +async function begin(app) { + const start = await app.request('/admin/auth/oidc/start?returnTo=/models') + assert.equal(start.status, 302) + const authorize = start.headers.get('location') + const cookie = start.headers.getSetCookie()[0].split(';')[0] + const response = await fetch(authorize, { redirect: 'manual', signal: AbortSignal.timeout(5000) }) + assert.equal(response.status, 302) + return { callback: new URL(response.headers.get('location')), cookie, authorize: new URL(authorize) } +} + +test('OIDC wire flow verifies signed claims, PKCE, browser binding, identity status and SSO policy', { skip: !databaseURL, timeout: 120_000 }, async t => { + const idp = await identityProvider(t) + const app = await gateway(t, { + MODELPORT_OIDC_ISSUER: idp.url, + MODELPORT_OIDC_CLIENT_ID: 'modelport-assurance', + MODELPORT_OIDC_CLIENT_SECRET: idp.clientSecret, + MODELPORT_OIDC_ALLOW_INSECURE_HTTP: '1', + }) + let admin = await adminSession(app) + let failures = 0 + const scenario = (name, fn) => t.test(name, async () => { + try { await fn() } catch (error) { failures++; throw error } + }) + const user = await createUser(app, admin, 'oidc') + const baseClaims = { email: user.email, preferred_username: user.username } + async function complete(grant, cookie = grant.cookie) { + return app.request(`${grant.callback.pathname}${grant.callback.search}`, { headers: { cookie } }) + } + await scenario('SSO-only startup refuses a deployment without a linked administrator', async () => { + await app.stop() + await assert.rejects(app.start({ MODELPORT_PASSWORD_LOGIN_ENABLED: '0' }), /isolated gateway must start/) + assert.match(app.logs(), /active administrator already linked/) + await app.start() + admin = await adminSession(app) + }) + await scenario('binds a verified identity, then rejects callback replay and cookie use on the data plane', async () => { + idp.select({ claims: baseClaims }) + const flow = await begin(app) + const response = await complete(flow) + assert.equal(response.headers.get('location'), '/models') + const cookie = response.headers.getSetCookie().find(value => value.startsWith('modelport_admin_session=')).split(';')[0] + assert.equal((await app.request('/admin/auth/me', { headers: { cookie } })).status, 200) + assert.equal((await app.request('/v1/models', { headers: { cookie } })).status, 401) + assert.match((await complete(flow)).headers.get('location'), /oidc_error=invalid_state/) + }) + await scenario('rejects another browser before attempting the token exchange', async () => { + const flow = await begin(app) + const calls = idp.tokenCalls() + assert.match((await complete(flow, 'modelport_oidc_flow=wrong-browser')).headers.get('location'), /invalid_state/) + assert.equal(idp.tokenCalls(), calls) + }) + for (const [label, selected] of [ + ['wrong issuer', { claims: { iss: 'https://wrong.example.test' } }], + ['wrong audience', { claims: { aud: 'another-client' } }], + ['expired token', { claims: { exp: 1 } }], + ['wrong nonce', { claims: { nonce: 'incorrect-nonce' } }], + ['forged signature', { badSignature: true }], + ['wrong access-token hash', { badAccessHash: true }], + ]) await scenario(`rejects ${label}`, async () => { + idp.select({ ...selected, claims: { ...baseClaims, ...selected.claims } }) + const response = await complete(await begin(app)) + assert.match(response.headers.get('location'), /oidc_error=token_invalid/) + assert.ok(!response.headers.getSetCookie().some(value => value.startsWith('modelport_admin_session='))) + }) + await scenario('requires signed assurance and rejects password fallback when SSO-only is configured', async () => { + const promoted = await app.request(`/admin/users/${user.id}`, { method: 'PUT', headers: { cookie: admin, 'x-modelport-csrf': '1' }, data: { role: 'admin' } }) + assert.equal(promoted.status, 200) + await app.stop() + await app.start({ MODELPORT_PASSWORD_LOGIN_ENABLED: '0', MODELPORT_OIDC_REQUIRED_ACR: 'urn:modelport:assurance:mfa' }) + assert.equal((await (await app.request('/admin/auth/methods')).json()).passwordEnabled, false) + assert.equal((await app.request('/admin/auth/login', { data: { username: 'assurance_admin', password: app.password } })).status, 403) + for (const acr of [undefined, 'urn:lower-assurance']) { + idp.select({ claims: { ...baseClaims, acr } }) + const flow = await begin(app) + assert.equal(flow.authorize.searchParams.get('acr_values'), 'urn:modelport:assurance:mfa') + assert.match((await complete(flow)).headers.get('location'), /oidc_error=token_invalid/) + } + idp.select({ claims: baseClaims }) + assert.equal((await complete(await begin(app))).headers.get('location'), '/models') + }) + await scenario('a disabled linked account cannot obtain a new console session', async () => { + await app.stop() + await app.start() + admin = await adminSession(app) + const disabled = await app.request(`/admin/users/${user.id}`, { method: 'PUT', headers: { cookie: admin, 'x-modelport-csrf': '1' }, data: { status: 'disabled' } }) + assert.equal(disabled.status, 200) + idp.select({ claims: baseClaims }) + assert.match((await complete(await begin(app))).headers.get('location'), /account_not_authorized/) + }) + if (failures === 0) await evidence('oidc', { signingAlgorithm: 'RS256', pkce: 'S256', confidentialClient: true, + issuerAudienceNonceExpirySignatureAndAccessHashChecked: true, ssoOnlyAndSignedAssuranceChecked: true, + browserBindingReplayAndDisabledIdentityChecked: true, realIdentityProvider: false }) +}) diff --git a/tests/runtime/protocol-recovery.test.mjs b/tests/runtime/protocol-recovery.test.mjs new file mode 100644 index 0000000..c8143cc --- /dev/null +++ b/tests/runtime/protocol-recovery.test.mjs @@ -0,0 +1,211 @@ +import assert from 'node:assert/strict' +import { execFile } from 'node:child_process' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' +import { promisify } from 'node:util' +import { test } from 'node:test' +import { adminSession, createUser, databaseURL, evidence, gateway } from './support.mjs' +import { upstream } from './upstream.mjs' + +const execute = promisify(execFile) +const model = 'custom:assurance-model' +const message = { model, max_tokens: 32, messages: [{ role: 'user', content: 'Synthetic acceptance: reply OK.' }] } + +test('gateway protocol, bounded concurrent load, cancellation, database outage and restore', { skip: !databaseURL, timeout: 180_000 }, async t => { + const mock = await upstream(t) + const app = await gateway(t, { CUSTOM_OPENAI_BASE_URL: `${mock.url}/v1`, MODELPORT_HTTP_STREAM_IDLE_TIMEOUT_SECS: '2', MODELPORT_DATABASE_ACQUIRE_TIMEOUT_SECS: '2' }, ` +[providers.custom] +protocol = "openai-compat" +base_url = "${mock.url}/v1" +api_key_env = "CUSTOM_OPENAI_API_KEY" +default_model = "assurance-model" +models = ["assurance-model"] +[providers.custom.model_profile_defaults] +reasoning_replay = "none" +[providers.custom.tool_use] +supported = true +response_validation = "strict" +`) + let activeDatabaseURL = databaseURL + let cookie = await adminSession(app) + const user = await createUser(app, cookie, 'protocol') + const response = await app.request('/admin/api-keys', { headers: { cookie, 'x-modelport-csrf': '1' }, + data: { userId: user.id, name: 'assurance-client', allowedProviders: ['custom'], allowedModels: ['assurance-model'] } }) + assert.equal(response.status, 200) + const key = await response.json() + const apiKey = key.key + assert.ok(apiKey, 'one-time API key must be returned') + const headers = { 'x-api-key': apiKey, 'x-modelport-traffic-class': 'synthetic' } + const writeHeaders = () => ({ cookie, 'x-modelport-csrf': '1' }) + for (const path of ['/v1/messages', '/v1/chat/completions']) { + await t.test(`${path}: text, live streaming and tool-result round trip`, async () => { + let response = await app.request(path, { headers, data: message }) + assert.equal(response.status, 200, 'synthetic text request') + const text = await response.json() + assert.equal(path === '/v1/messages' ? text.content[0].text : text.choices[0].message.content, 'OK') + response = await app.request(path, { headers, data: { ...message, stream: true } }) + assert.equal(response.status, 200) + const stream = await response.text() + assert.match(stream, /OK/) + assert.match(stream, path === '/v1/messages' ? /event: message_stop/ : /data: \[DONE\]/) + const definition = { name: 'weather', description: 'Synthetic weather', parameters: { type: 'object', properties: { city: { type: 'string' } }, required: ['city'] } } + const tools = path === '/v1/messages' + ? [{ name: definition.name, description: definition.description, input_schema: definition.parameters }] + : [{ type: 'function', function: definition }] + response = await app.request(path, { headers, data: { ...message, tools } }) + assert.equal(response.status, 200) + const completion = await response.json() + let messages + if (path === '/v1/messages') { + const call = completion.content.find(block => block.type === 'tool_use') + assert.deepEqual(call.input, { city: 'Shanghai' }) + messages = [...message.messages, { role: 'assistant', content: completion.content }, { role: 'user', content: [{ type: 'tool_result', tool_use_id: call.id, content: 'Sunny' }] }] + } else { + const assistant = completion.choices[0].message + assert.deepEqual(JSON.parse(assistant.tool_calls[0].function.arguments), { city: 'Shanghai' }) + messages = [...message.messages, assistant, { role: 'tool', tool_call_id: assistant.tool_calls[0].id, content: 'Sunny' }] + } + response = await app.request(path, { headers, data: { ...message, tools, messages } }) + assert.equal(response.status, 200, (await response.clone().json()).error?.message) + const result = await response.json() + assert.equal(path === '/v1/messages' ? result.content[0].text : result.choices[0].message.content, 'OK') + response = await app.request(path, { headers, data: { ...message, tools, stream: true } }) + assert.equal(response.status, 200) + const toolStream = await response.text() + assert.match(toolStream, /Shanghai/) + assert.match(toolStream, path === '/v1/messages' ? /event: message_stop/ : /data: \[DONE\]/) + }) + } + await t.test('rejects forged identity, data-plane keys on the console and writes without CSRF', async () => { + const calls = mock.calls() + assert.equal((await app.request('/v1/models', { headers: { 'x-api-key': 'incorrect-key' } })).status, 401) + assert.equal((await app.request('/admin/users', { headers })).status, 401) + assert.equal((await app.request(`/admin/users/${user.id}`, { method: 'PUT', headers: { cookie }, data: { email: user.email } })).status, 403) + assert.equal(mock.calls(), calls) + }) + await t.test('40 distinct users sustain synthetic streaming load with bounded outcomes', async () => { + const durationSeconds = Number(process.env.MODELPORT_ASSURANCE_LOAD_SECONDS || 10) + assert.ok(Number.isInteger(durationSeconds) && durationSeconds >= 1 && durationSeconds <= 120) + const clients = [] + for (let i = 0; i < 40; i++) { + const owner = await createUser(app, cookie, `load${i}`) + const response = await app.request('/admin/api-keys', { headers: writeHeaders(), + data: { userId: owner.id, name: `assurance-load-${i}`, allowedProviders: ['custom'], allowedModels: ['assurance-model'] } }) + assert.equal(response.status, 200) + clients.push((await response.json()).key) + } + const started = performance.now() + const results = [] + // Ten paced waves keep the request budget fixed even for a longer soak. + // Each request goes through real auth, policy, PostgreSQL and SSE handling. + for (let wave = 0; wave < 10; wave++) { + const batch = await Promise.all(clients.map(async (token, index) => { + const start = performance.now() + const response = await app.request(index % 2 ? '/v1/messages' : '/v1/chat/completions', { + headers: { ...headers, 'x-api-key': token }, data: { ...message, stream: true }, + }) + const content = await response.text() + assert.ok([200, 429].includes(response.status), 'load must succeed or reject admission explicitly') + if (response.status === 200) assert.match(content, index % 2 ? /event: message_stop/ : /data: \[DONE\]/) + else assert.ok(Number(response.headers.get('retry-after')) > 0) + return { status: response.status, ms: performance.now() - start } + })) + results.push(...batch) + const untilNext = started + (wave + 1) * durationSeconds * 100 - performance.now() + if (untilNext > 0) await delay(untilNext) + } + assert.ok(results.some(result => result.status === 200)) + const sorted = results.map(result => result.ms).sort((a, b) => a - b) + const accepted = results.filter(result => result.status === 200).map(result => result.ms).sort((a, b) => a - b) + await evidence('synthetic-load', { clients: 40, distinctUsers: 40, requests: results.length, + elapsedMs: performance.now() - started, success: results.filter(result => result.status === 200).length, + rejected: results.filter(result => result.status === 429).length, + p50Ms: sorted[Math.ceil(sorted.length * 0.50) - 1], p95Ms: sorted[Math.ceil(sorted.length * 0.95) - 1], + successP50Ms: accepted[Math.ceil(accepted.length * 0.50) - 1], successP95Ms: accepted[Math.ceil(accepted.length * 0.95) - 1], + latencyScope: 'whole-response-including-stream', localExecutionSlots: 1, + upstreamPeak: mock.peak(), realProvider: false }) + }) + await t.test('truncated SSE is an error and client cancellation releases the upstream stream', async () => { + mock.setMode('truncated') + const response = await app.request('/v1/messages', { headers, data: { ...message, stream: true } }) + assert.equal(response.status, 200) + const truncated = await response.text() + assert.match(truncated, /event: error/) + assert.doesNotMatch(truncated, /event: message_stop/) + mock.setMode('hold') + const abort = new AbortController() + const held = await app.request('/v1/messages', { headers, data: { ...message, stream: true }, signal: abort.signal }) + const reader = held.body.getReader() + await reader.read() + abort.abort() + await reader.cancel().catch(() => {}) + for (let i = 0; i < 50 && mock.active(); i++) await delay(100) + assert.equal(mock.active(), 0, 'cancelled stream releases its upstream connection') + mock.setMode('normal') + }) + await t.test('forced process restart invalidates sessions and preserves scoped keys', async () => { + await app.stop('SIGKILL') + await app.start() + assert.equal((await app.request('/admin/auth/me', { headers: { cookie } })).status, 401) + assert.equal((await app.request('/v1/models', { headers })).status, 200) + cookie = await adminSession(app) + }) + const container = process.env.MODELPORT_RUNTIME_TEST_POSTGRES_CONTAINER + await t.test('database outage fails closed, then backup/restore recovers auth and ledger state', { skip: !container, timeout: 90_000 }, async () => { + assert.match(container, /^modelport-assurance-[0-9]+-[0-9]+$/) + const inspect = await execute('docker', ['inspect', '--format', '{{index .Config.Labels "io.modelport.test"}}', container]) + assert.equal(inspect.stdout.trim(), 'assurance') + const docker = args => execute('docker', ['exec', container, ...args], { maxBuffer: 8 * 1024 * 1024 }) + const sql = (query, database = 'modelport_assurance') => docker(['psql', '-U', 'modelport', '-d', database, '-Atc', query]) + const calls = mock.calls() + await execute('docker', ['stop', '--time', '5', container]) + try { + assert.equal((await app.request('/readyz', { headers: { 'x-api-key': app.token } })).status, 503) + const failed = await app.request('/v1/messages', { headers, data: message }) + assert.ok(failed.status >= 500) + assert.equal(mock.calls(), calls, 'storage outage must prevent unrecorded model egress') + } finally { await execute('docker', ['start', container]) } + for (let i = 0; i < 100; i++) { + try { await sql('SELECT 1'); break } catch { await delay(100) } + } + assert.equal((await app.request('/readyz', { headers: { 'x-api-key': app.token } })).status, 200) + await app.stop() + const fingerprintQuery = "SELECT namespace || ':' || md5(document::text) FROM modelport_state ORDER BY namespace" + const fingerprint = (await sql(fingerprintQuery)).stdout + const ledger = (await sql('SELECT count(*) FROM modelport_gateway_requests')).stdout + const dump = await execute('docker', ['exec', container, 'pg_dump', '-U', 'modelport', '-d', 'modelport_assurance', '-Fc', '--no-owner', '--no-privileges'], { encoding: 'buffer', maxBuffer: 16 * 1024 * 1024 }) + const temporary = await mkdtemp(join(tmpdir(), 'modelport-restore-')) + try { + const archive = join(temporary, 'postgres.dump') + await writeFile(archive, dump.stdout, { mode: 0o600 }) + await docker(['createdb', '-U', 'modelport', 'modelport_assurance_restore']) + await execute('docker', ['cp', archive, `${container}:/tmp/assurance.dump`]) + await docker(['pg_restore', '-U', 'modelport', '-d', 'modelport_assurance_restore', '--exit-on-error', '--no-owner', '--no-privileges', '/tmp/assurance.dump']) + assert.equal((await sql(fingerprintQuery, 'modelport_assurance_restore')).stdout, fingerprint) + assert.equal((await sql('SELECT count(*) FROM modelport_gateway_requests', 'modelport_assurance_restore')).stdout, ledger) + const restored = new URL(databaseURL); restored.pathname = '/modelport_assurance_restore' + activeDatabaseURL = restored.href + await app.start({ MODELPORT_DATABASE_URL: restored.href }) + cookie = await adminSession(app) + assert.equal((await app.request('/v1/models', { headers })).status, 200) + if (process.env.MODELPORT_ROLLBACK_TEST_BINARY) { + await app.stop() + await app.start({ MODELPORT_DATABASE_URL: restored.href }, process.env.MODELPORT_ROLLBACK_TEST_BINARY) + assert.equal((await app.request('/readyz', { headers: { 'x-api-key': app.token } })).status, 200) + assert.equal((await app.request('/v1/models', { headers })).status, 200) + } + await evidence('recovery', { databaseOutageRejectedBeforeEgress: true, stateFingerprintMatched: true, ledgerRows: Number(ledger.trim()), restoredLoginPassed: true, + rollbackBinaryPassed: Boolean(process.env.MODELPORT_ROLLBACK_TEST_BINARY), productionRtoRpoVerified: false }) + } finally { await rm(temporary, { recursive: true, force: true }) } + }) + await t.test('revoking a client key remains effective after restart', async () => { + cookie = await adminSession(app) + const revoked = await app.request(`/admin/api-keys/${key.id}`, { method: 'DELETE', headers: writeHeaders() }) + assert.equal(revoked.status, 200) + await app.stop() + await app.start({ MODELPORT_DATABASE_URL: activeDatabaseURL }) + assert.equal((await app.request('/v1/models', { headers })).status, 401) + }) +}) diff --git a/tests/runtime/run.sh b/tests/runtime/run.sh new file mode 100755 index 0000000..eafb9a0 --- /dev/null +++ b/tests/runtime/run.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail +trap 'printf "[modelport-assurance] setup failed at line %s\n" "$LINENO" >&2' ERR + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" +cd "$ROOT_DIR" +umask 077 +command -v docker >/dev/null +command -v node >/dev/null +if [[ -z "${MODELPORT_TEST_BINARY:-}" ]]; then + cargo build --locked --bin model-port + export MODELPORT_TEST_BINARY="$ROOT_DIR/target/debug/model-port" +fi +[[ -x "$MODELPORT_TEST_BINARY" ]] || { echo 'MODELPORT_TEST_BINARY must be executable' >&2; exit 1; } + +runtime_dir="$(mktemp -d)" +postgres_container="modelport-assurance-$$-$RANDOM" +cleanup() { + docker rm -f -v "$postgres_container" >/dev/null 2>&1 || true + rm -rf -- "$runtime_dir" +} +trap cleanup EXIT +# shellcheck disable=SC2016 +node -e ' +const fs = require("node:fs") +const password = require("node:crypto").randomBytes(24).toString("hex") +fs.writeFileSync(process.argv[1], `POSTGRES_USER=modelport\nPOSTGRES_DB=modelport_assurance\nPOSTGRES_PASSWORD=${password}\n`, { mode: 0o600 }) +' "$runtime_dir/postgres.env" +printf '%s\n' '[modelport-assurance] starting disposable PostgreSQL' +postgres_port="$(node -e 'const s = require("node:net").createServer(); s.listen(0, "127.0.0.1", () => { process.stdout.write(String(s.address().port)); s.close() })')" +docker run --detach --name "$postgres_container" --label io.modelport.test=assurance \ + --env-file "$runtime_dir/postgres.env" -p "127.0.0.1:$postgres_port:5432" postgres:18.4-alpine >/dev/null +for _ in {1..100}; do + if docker exec "$postgres_container" pg_isready -h 127.0.0.1 -U modelport -d modelport_assurance >/dev/null 2>&1; then break; fi + sleep 0.2 +done +docker exec "$postgres_container" pg_isready -h 127.0.0.1 -U modelport -d modelport_assurance >/dev/null +postgres_password="$(sed -n 's/^POSTGRES_PASSWORD=//p' "$runtime_dir/postgres.env")" +export MODELPORT_RUNTIME_TEST_DATABASE_URL="postgres://modelport:$postgres_password@127.0.0.1:$postgres_port/modelport_assurance" +export MODELPORT_RUNTIME_TEST_POSTGRES_CONTAINER="$postgres_container" +printf '%s\n' '[modelport-assurance] isolated PostgreSQL, signed OIDC and synthetic loopback upstreams only' +node --test --test-concurrency=1 "$ROOT_DIR/tests/runtime/"*.test.mjs diff --git a/tests/runtime/support.mjs b/tests/runtime/support.mjs new file mode 100644 index 0000000..50cabfa --- /dev/null +++ b/tests/runtime/support.mjs @@ -0,0 +1,156 @@ +import assert from 'node:assert/strict' +import { spawn, spawnSync } from 'node:child_process' +import { randomBytes } from 'node:crypto' +import { once } from 'node:events' +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { createServer } from 'node:http' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' + +export const databaseURL = process.env.MODELPORT_RUNTIME_TEST_DATABASE_URL +export const root = resolve(import.meta.dirname, '../..') +export const secret = () => randomBytes(24).toString('base64url') + +export async function server(handler) { + const http = createServer((req, res) => { + Promise.resolve(handler(req, res)).catch(() => { + if (!res.headersSent) res.writeHead(500) + res.end() + }) + }) + http.listen(0, '127.0.0.1') + await once(http, 'listening') + return { + url: `http://127.0.0.1:${http.address().port}`, + async close() { + const closed = once(http, 'close') + http.close() + http.closeAllConnections() + await closed + }, + } +} + +export async function body(req) { + const chunks = [] + let bytes = 0 + for await (const chunk of req) { + bytes += chunk.length + assert.ok(bytes < 1024 * 1024, 'fixture request is bounded') + chunks.push(chunk) + } + return Buffer.concat(chunks).toString() +} + +export function json(res, value, status = 200) { + res.writeHead(status, { 'content-type': 'application/json' }) + res.end(JSON.stringify(value)) +} + +export async function gateway(t, overrides = {}, config) { + assert.ok(databaseURL, 'MODELPORT_RUNTIME_TEST_DATABASE_URL is required') + const database = new URL(databaseURL) + assert.ok(['127.0.0.1', 'localhost', '[::1]'].includes(database.hostname), 'runtime tests require loopback PostgreSQL') + assert.match(database.pathname, /^\/modelport_assurance(?:_[a-z0-9_]+)?$/, 'use a dedicated modelport_assurance database') + const runtime = await mkdtemp(join(tmpdir(), 'modelport-assurance-')) + const reservation = await server((_req, res) => res.end()) + const url = reservation.url + await reservation.close() + // Keep test credentials stable across restarts of this isolated database. + const password = 'Assurance_local_7kP9wR4zM2' + const token = secret() + const env = { + PATH: process.env.PATH, + HOME: runtime, + MODELPORT_ENV_FILE: join(runtime, 'absent.env'), + MODELPORT_CONFIG: join(runtime, 'absent.toml'), + MODELPORT_DATABASE_URL: databaseURL, + MODELPORT_DATABASE_TLS_MODE: 'disable', + MODELPORT_BIND: new URL(url).host, + MODELPORT_ADMIN_USERNAME: 'assurance_admin', + MODELPORT_ADMIN_PASSWORD: password, + MODELPORT_AUTH_TOKEN: token, + MODELPORT_DEFAULT_PROVIDER: 'custom', + CUSTOM_OPENAI_BASE_URL: 'http://127.0.0.1:9/v1', + CUSTOM_OPENAI_API_KEY: secret(), + CUSTOM_OPENAI_MODEL: 'assurance-model', + ...overrides, + } + if (config) { + env.MODELPORT_CONFIG = join(runtime, 'config.toml') + await writeFile(env.MODELPORT_CONFIG, config, { mode: 0o600 }) + } + if (env.MODELPORT_OIDC_ISSUER) env.MODELPORT_OIDC_REDIRECT_URI = `${url}/admin/auth/oidc/callback` + const binary = process.env.MODELPORT_TEST_BINARY || join(root, 'target/debug/model-port') + let child + let output = '' + let launchError + async function stop(signal = 'SIGTERM') { + if (!child || child.exitCode !== null || child.signalCode !== null) return + const exited = once(child, 'exit') + child.kill(signal) + const timeout = setTimeout(() => child.kill('SIGKILL'), 10_000) + try { await exited } finally { clearTimeout(timeout) } + } + async function start(extra = {}, executable = binary) { + output = '' + child = spawn(executable, [], { cwd: root, env: { ...env, ...extra }, stdio: ['ignore', 'pipe', 'pipe'] }) + child.on('error', error => { launchError = error }) + for (const stream of [child.stdout, child.stderr]) stream.on('data', chunk => { output = (output + chunk).slice(-20_000) }) + for (let i = 0; i < 160; i++) { + if (launchError) throw launchError + assert.equal(child.exitCode, null, 'isolated gateway must start; inspect its configuration') + try { + const response = await fetch(`${url}/livez`, { signal: AbortSignal.timeout(500) }) + if (response.ok) return + } catch { /* wait for the listener */ } + await delay(100) + } + throw new Error('isolated gateway did not become live') + } + t.after(async () => { await stop(); await rm(runtime, { recursive: true, force: true }) }) + await start() + return { + url, token, password, start, stop, + logs: () => output, + async request(path, { data, headers, method, ...options } = {}) { + return fetch(`${url}${path}`, { + method: method || (data ? 'POST' : 'GET'), + headers: { ...(data ? { 'content-type': 'application/json' } : {}), ...headers }, + body: data ? JSON.stringify(data) : undefined, + signal: AbortSignal.timeout(15_000), + redirect: 'manual', + ...options, + }) + }, + } +} + +export async function evidence(name, value) { + const directory = process.env.MODELPORT_ASSURANCE_OUTPUT_DIR + if (!directory) return + await mkdir(directory, { recursive: true, mode: 0o700 }) + await writeFile(join(directory, `${name}.json`), JSON.stringify({ + schemaVersion: 1, scope: 'isolated-synthetic', generatedAt: new Date().toISOString(), + commit: spawnSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).stdout.trim(), + sourceState: spawnSync('git', ['status', '--porcelain'], { cwd: root, encoding: 'utf8' }).stdout.trim() ? 'dirty' : 'clean', + ...value, + }, null, 2) + '\n', { mode: 0o600 }) +} + +export async function adminSession(app) { + const response = await app.request('/admin/auth/login', { data: { username: 'assurance_admin', password: app.password } }) + assert.equal(response.status, 200, 'fixture administrator login') + return response.headers.getSetCookie()[0].split(';')[0] +} + +export async function createUser(app, cookie, label) { + const username = `assurance_${label}_${secret().slice(0, 8).toLowerCase().replace(/[^a-z0-9]/g, 'x')}` + const response = await app.request('/admin/users', { + headers: { cookie, 'x-modelport-csrf': '1' }, + data: { username, email: `${username}@example.test`, password: secret(), role: 'user', status: 'active' }, + }) + assert.equal(response.status, 200, 'create isolated user') + return response.json() +} diff --git a/tests/runtime/upstream.mjs b/tests/runtime/upstream.mjs new file mode 100644 index 0000000..e12fb9e --- /dev/null +++ b/tests/runtime/upstream.mjs @@ -0,0 +1,48 @@ +import { setTimeout as delay } from 'node:timers/promises' +import { body, json, server } from './support.mjs' + +export async function upstream(t) { + let mode = 'normal' + let calls = 0 + let active = 0 + let peak = 0 + const mock = await server(async (req, res) => { + if (req.url === '/v1/models') return json(res, { data: [{ id: 'assurance-model' }] }) + if (req.url !== '/v1/chat/completions') { res.writeHead(404); return res.end() } + calls++ + active++ + peak = Math.max(peak, active) + let released = false + res.on('close', () => { if (!released) { active--; released = true } }) + const input = JSON.parse(await body(req)) + const toolResult = input.messages.some(message => message.role === 'tool') + const tool = input.tools?.[0]?.function + const choice = tool && !toolResult + ? { role: 'assistant', content: null, tool_calls: [{ id: 'assurance_call', type: 'function', function: { name: tool.name, arguments: '{"city":"Shanghai"}' } }] } + : { role: 'assistant', content: 'OK' } + if (mode === 'error') return json(res, { error: { message: 'synthetic unavailable' } }, 503) + if (!input.stream) { + await delay(20) + return json(res, { id: 'assurance_completion', object: 'chat.completion', model: 'assurance-model', created: 1, + choices: [{ index: 0, message: choice, finish_reason: choice.tool_calls ? 'tool_calls' : 'stop' }], + usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 } }) + } + res.writeHead(200, { 'content-type': 'text/event-stream' }) + const chunk = (delta, finish = null) => res.write(`data: ${JSON.stringify({ + id: 'assurance_completion', object: 'chat.completion.chunk', created: 1, model: 'assurance-model', + choices: [{ index: 0, delta, finish_reason: finish }], + })}\n\n`) + chunk({ role: 'assistant' }) + if (choice.tool_calls) { + chunk({ tool_calls: [{ index: 0, ...choice.tool_calls[0], function: { name: tool.name, arguments: '{"city":' } }] }) + chunk({ tool_calls: [{ index: 0, function: { arguments: '"Shanghai"}' } }] }) + } else chunk({ content: 'OK' }) + if (mode === 'hold') return + await delay(100) + if (mode === 'truncated') return res.end() + chunk({}, choice.tool_calls ? 'tool_calls' : 'stop') + res.end('data: [DONE]\n\n') + }) + t.after(() => mock.close()) + return { ...mock, setMode: value => { mode = value }, calls: () => calls, active: () => active, peak: () => peak } +}