feat: add executable production acceptance suite

.github/workflows/acceptance.yml

@@ -0,0 +1,148 @@
+name: Production Acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      suite:
+        description: Acceptance suite to run
+        required: true
+        default: all
+        type: choice
+        options:
+          - all
+          - probe
+          - regression
+          - load
+          - shadow
+      load_requests:
+        description: Total load-test requests
+        required: false
+        default: '100'
+      load_concurrency:
+        description: Load-test concurrency
+        required: false
+        default: '10'
+
+permissions:
+  contents: read
+
+env:
+  CANDIDATE_URL: ${{ vars.NIVORA_ACCEPTANCE_CANDIDATE_URL }}
+  BASELINE_URL: ${{ vars.NIVORA_ACCEPTANCE_BASELINE_URL }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    environment: nivora-staging
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.23.x'
+          cache: true
+      - run: go build -trimpath ./...
+
+  probe:
+    if: inputs.suite == 'all' || inputs.suite == 'probe'
+    needs: build
+    runs-on: ubuntu-latest
+    environment: nivora-staging
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.23.x'
+          cache: true
+      - name: Run security probes
+        env:
+          NIVORA_PROBE_SHARED_SECRET: ${{ secrets.NIVORA_ACCEPTANCE_CANDIDATE_KEY }}
+          NIVORA_PROBE_BEARER_TOKEN: ${{ secrets.NIVORA_ACCEPTANCE_CANDIDATE_BEARER }}
+        run: |
+          go run ./cmd/nivora-probe \
+            -base-url "$CANDIDATE_URL" \
+            -output probe-results.jsonl
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: security-probe-results
+          path: probe-results.jsonl
+
+  regression:
+    if: inputs.suite == 'all' || inputs.suite == 'regression'
+    needs: build
+    runs-on: ubuntu-latest
+    environment: nivora-staging
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.23.x'
+          cache: true
+      - name: Run support regression
+        env:
+          NIVORA_EVAL_SHARED_SECRET: ${{ secrets.NIVORA_ACCEPTANCE_CANDIDATE_KEY }}
+          NIVORA_EVAL_BEARER_TOKEN: ${{ secrets.NIVORA_ACCEPTANCE_CANDIDATE_BEARER }}
+        run: |
+          go run ./cmd/nivora-eval \
+            -base-url "$CANDIDATE_URL" \
+            -output regression-results.jsonl
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: support-regression-results
+          path: regression-results.jsonl
+
+  load:
+    if: inputs.suite == 'all' || inputs.suite == 'load'
+    needs: build
+    runs-on: ubuntu-latest
+    environment: nivora-staging
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.23.x'
+          cache: true
+      - name: Run bounded load test
+        env:
+          NIVORA_LOAD_SHARED_SECRET: ${{ secrets.NIVORA_ACCEPTANCE_CANDIDATE_KEY }}
+          NIVORA_LOAD_BEARER_TOKEN: ${{ secrets.NIVORA_ACCEPTANCE_CANDIDATE_BEARER }}
+        run: |
+          go run ./cmd/nivora-load \
+            -base-url "$CANDIDATE_URL" \
+            -requests "${{ inputs.load_requests }}" \
+            -concurrency "${{ inputs.load_concurrency }}" \
+            > load-results.json
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: load-results
+          path: load-results.json
+
+  shadow:
+    if: inputs.suite == 'all' || inputs.suite == 'shadow'
+    needs: build
+    runs-on: ubuntu-latest
+    environment: nivora-staging
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.23.x'
+          cache: true
+      - name: Compare baseline and candidate
+        env:
+          NIVORA_SHADOW_BASELINE_KEY: ${{ secrets.NIVORA_ACCEPTANCE_BASELINE_KEY }}
+          NIVORA_SHADOW_CANDIDATE_KEY: ${{ secrets.NIVORA_ACCEPTANCE_CANDIDATE_KEY }}
+          NIVORA_SHADOW_BASELINE_BEARER: ${{ secrets.NIVORA_ACCEPTANCE_BASELINE_BEARER }}
+          NIVORA_SHADOW_CANDIDATE_BEARER: ${{ secrets.NIVORA_ACCEPTANCE_CANDIDATE_BEARER }}
+        run: |
+          go run ./cmd/nivora-shadow \
+            -baseline-url "$BASELINE_URL" \
+            -candidate-url "$CANDIDATE_URL" \
+            -output shadow-results.jsonl
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: shadow-results
+          path: shadow-results.jsonl

Makefile

@@ -1,10 +1,14 @@
-.PHONY: build test fmt vet run eval knowledge knowledge-eval
+.PHONY: build test fmt vet run eval knowledge knowledge-eval probe load shadow test-provider
 
 build:
 	go build -trimpath -o bin/nivora ./cmd/nivora
 	go build -trimpath -o bin/nivora-eval ./cmd/nivora-eval
 	go build -trimpath -o bin/nivora-knowledge ./cmd/nivora-knowledge
 	go build -trimpath -o bin/nivora-knowledge-eval ./cmd/nivora-knowledge-eval
+	go build -trimpath -o bin/nivora-probe ./cmd/nivora-probe
+	go build -trimpath -o bin/nivora-load ./cmd/nivora-load
+	go build -trimpath -o bin/nivora-shadow ./cmd/nivora-shadow
+	go build -trimpath -o bin/nivora-test-provider ./cmd/nivora-test-provider
 
 test:
 	go test ./...
@@ -26,3 +30,15 @@ knowledge:
 
 knowledge-eval:
 	go run ./cmd/nivora-knowledge-eval
+
+probe:
+	go run ./cmd/nivora-probe
+
+load:
+	go run ./cmd/nivora-load
+
+shadow:
+	go run ./cmd/nivora-shadow
+
+test-provider:
+	go run ./cmd/nivora-test-provider

README.md

@@ -2,11 +2,11 @@
 
 English | [简体中文](README.zh-CN.md)
 
-Nivora is a reusable, tenant-aware customer-support Agent Runtime written in Go. It uses Eino for agent orchestration and keeps product data behind a versioned Provider API.
+Nivora is a reusable, tenant-aware customer-support Agent Runtime written in Go. It uses Eino for Agent orchestration and keeps product data behind a versioned Provider API.
 
 Lumio is the first planned provider integration, but Nivora itself does not know about Lumio tables, NextAuth, credits, generation pipelines, or SQLite.
 
-> Nivora is an integration-ready runtime under active production hardening. It is not considered production-accepted until a real Provider, security evaluation, load test, and shadow-traffic evaluation have passed.
+> Nivora now includes the engineering and acceptance tooling needed to prepare a production candidate. It is not production-approved until the real Provider contract, Volcengine environment, load and recovery tests, and consented-redacted shadow comparison have passed in the company's isolated staging environment.
 
 ## Current foundation
 
@@ -15,17 +15,22 @@ Lumio is the first planned provider integration, but Nivora itself does not know
 - optional CozeLoop tracing and PromptHub policy versions with strict trace redaction and bundled fallback
 - capability- and scope-driven Tool registration
 - provider-neutral Tools for knowledge, customer context, resources, diagnosis, transactions, and human-support cases
-- Provider-side approved-knowledge reference service using the official Eino VikingDB retriever
+- Provider-side approved-knowledge reference service using the official Eino VikingDB Retriever
 - tenant, approval, freshness, provenance, and score validation after semantic retrieval
 - SQLite development and PostgreSQL production storage for public transcripts, run metadata, sanitized Tool audits, and support-case references
 - deterministic replay protection and tenant-scoped transcript access
 - black-box customer-support and knowledge-retrieval JSONL evaluation tools
+- deterministic HTTP security probes for authentication, tenant, scope, request-shape, and history-injection boundaries
+- SSE load testing with first-token and completion p50/p95/p99, success rate, and error distribution
+- privacy-safe baseline/candidate shadow comparison using answer hashes instead of answer text
+- a synthetic Provider with deterministic support facts, idempotent cases, latency, and 429/5xx fault injection
+- a manual staging acceptance workflow for probes, regression, load, and shadow suites
 - bounded Provider retries for idempotent reads and idempotent support-case creation
 - stable Server-Sent Events protocol with heartbeat comments
 - private service authentication between the product BFF and Nivora
 - real Provider and storage readiness checks with short caching
 - global concurrency and queue protection
-- Prometheus-compatible runtime metrics
+- Prometheus-compatible Agent and process metrics
 - loopback-first production deployment examples
 
 ## Architecture
@@ -104,13 +109,28 @@ Tool results are not forwarded to the browser. They remain inside the Agent run.
 ## Security boundary
 
 - Bind Nivora and its reference services to loopback or private VPC addresses.
-- Do not expose ports `3100` or `3110` through a public reverse proxy.
+- Do not expose ports `3100`, `3110`, or the synthetic Provider through a public reverse proxy.
 - Use separate secrets for product-to-Nivora, Nivora-to-Provider, and Provider-to-knowledge authentication.
 - The Provider API must enforce customer ownership and redact internal fields.
 - Anonymous requests can receive only explicitly granted knowledge and case scopes.
 - Durable storage contains public messages and sanitized audit metadata only; it never stores chain of thought, bearer contexts, Tool payloads, or product recipes.
+- Shadow output stores answer hashes and byte counts, not answer text.
+- The synthetic Provider is for isolated acceptance only and must never serve production customer traffic.
 - Nivora currently performs read operations plus idempotent `case.create` only.
 
+## Production acceptance commands
+
+```bash
+make probe          # deterministic HTTP security boundaries
+make eval           # customer-support regression
+make knowledge-eval # approved knowledge retrieval benchmark
+make load           # bounded SSE load and latency test
+make shadow         # baseline/candidate privacy-safe comparison
+make test-provider  # synthetic Provider for isolated staging
+```
+
+A manually triggered GitHub Actions workflow is available at `.github/workflows/acceptance.yml`. It expects protected `nivora-staging` environment variables and secrets; it does not run against production automatically.
+
 ## Documentation
 
 - [Runtime API v1](docs/runtime-api.md)
@@ -119,6 +139,7 @@ Tool results are not forwarded to the browser. They remain inside the Agent run.
 - [Approved VikingDB knowledge](docs/approved-knowledge.md)
 - [Durable conversation storage](docs/durable-storage.md)
 - [Customer-support evaluation](docs/evaluation.md)
+- [Production acceptance gate](docs/production-acceptance.md)
 - [Volcengine production stack](docs/volcengine-production-stack.md)
 
 ## Development
@@ -128,12 +149,9 @@ make fmt
 make test
 make vet
 make build
-make eval
-make knowledge
-make knowledge-eval
 ```
 
 ## Roadmap
 
-1. Add the production security, load, and shadow-traffic acceptance suite.
-2. Add Eino interrupt/resume for human approval of future high-risk actions.
+1. Run the complete acceptance gate in the company's isolated staging environment and establish approved SLO, quality, cost, and rollback baselines.
+2. Add Eino interrupt/resume and explicit human approval before any future high-risk write Tool is introduced.

README.zh-CN.md

@@ -6,21 +6,31 @@ Nivora 是一个使用 Go 编写、可复用且支持多租户的智能客服 Ag
 
 Lumio 是 Nivora 计划接入的第一个业务系统，但 Nivora 本身并不知道 Lumio 的数据库表、NextAuth、积分系统、生成流水线或 SQLite 实现。
 
-> Nivora 当前已经具备业务接入所需的 Runtime 基础，但仍处于生产加固阶段。只有真实 Provider、安全评测、压力测试和影子流量评测全部通过后，才能认定为生产验收完成。
+> Nivora 目前已经具备用于构建生产候选版本的工程能力和验收工具，但尚不能直接认定为生产验收完成。真实 Provider 契约、火山引擎环境、压力与故障恢复测试，以及经授权脱敏的 Shadow 对比，仍需在公司的隔离预发布环境中实际通过。
 
 ## 当前能力
 
 - 基于 Eino `ChatModelAgent`，通过官方 `eino-ext` 适配器接入火山引擎方舟 Ark
 - 在开始流式输出前，按顺序进行多个方舟推理接入点故障转移
+- 可选接入 CozeLoop 链路追踪与 PromptHub 版本化策略，并实施严格脱敏和本地安全回退
 - 根据 Provider Capability 与可信 BFF 授予的 Scope 动态注册 Tool
 - 提供与具体业务无关的通用 Tool：知识检索、用户上下文、业务资源、故障诊断、账务流水和人工客服工单
+- 提供基于官方 Eino VikingDB Retriever 的 Provider 侧已审核知识参考服务
+- 对语义检索结果再次校验租户、审核状态、有效期、来源版本和最低置信度
+- 使用 SQLite 支持开发测试，并使用 PostgreSQL 支持生产环境中的公开会话、运行元数据、脱敏 Tool 审计和工单引用
+- 对请求、消息和 Tool 调用实施确定性幂等保护，并提供按租户隔离的会话转录接口
+- 提供客服回答与知识召回的 JSONL 黑盒回归评测工具
+- 提供针对鉴权、租户、Scope、请求结构和历史消息注入的确定性 HTTP 安全探针
+- 提供 SSE 压力测试，统计首 Token 与完成时延的 p50/p95/p99、成功率和错误分布
+- 提供 Baseline/Candidate Shadow 对比，结果仅保存答案哈希和字节数，不保存答案正文
+- 提供带确定性知识、失败作品、退款流水、幂等工单、延迟和 429/5xx 故障注入的 synthetic Provider
+- 提供可手动触发的预发布验收工作流，执行安全探针、客服回归、压力测试和 Shadow 对比
 - 对幂等 Provider 读取执行有界重试，并为客服工单生成稳定幂等键
 - 带心跳的稳定 Server-Sent Events（SSE）协议
 - 产品 BFF 与 Nivora 之间的私有服务鉴权
-- 带短期缓存的真实 Provider 就绪检查
+- 带短期缓存的真实 Provider 与存储就绪检查
 - 全局并发限制和排队超时保护
-- Prometheus 兼容运行指标
-- 针对回答、Tool 使用、延迟和拒答行为的 JSONL 黑盒回归评测
+- Prometheus 兼容的 Agent 与进程运行指标
 - 默认仅监听回环地址的生产部署示例
 
 ## 架构
@@ -29,11 +39,14 @@ Lumio 是 Nivora 计划接入的第一个业务系统，但 Nivora 本身并不
 浏览器
   -> 产品 BFF（会话、租户、品牌、Scope、限流）
      -> Nivora :3100（Eino Runtime，私有服务）
+        -> Nivora 会话与审计数据库
         -> 产品 Provider API（鉴权与业务事实来源）
-           -> 产品服务与数据库 / 已审核知识服务
+           -> 产品服务与业务数据库
+           -> 已审核知识服务 :3110
+              -> VikingDB
 ```
 
-Nivora 不接受聊天请求动态指定 Provider 地址，也不会直接连接产品数据库。部署时配置的 Provider 始终是业务事实的唯一来源。
+Nivora 不接受聊天请求动态指定 Provider 地址，也不会直接连接产品业务数据库或 VikingDB。部署时配置的 Provider 始终是业务事实的唯一来源。
 
 ## 本地运行
 
@@ -49,6 +62,8 @@ go run ./cmd/nivora
 curl http://127.0.0.1:3100/healthz
 curl -i http://127.0.0.1:3100/readyz
 curl http://127.0.0.1:3100/metrics
+curl -H 'X-Nivora-Key: replace-with-a-long-random-secret' \
+  http://127.0.0.1:3100/v1/conversations/conv-id/transcript
 ```
 
 聊天请求必须由可信的产品 BFF 发起。BFF 必须丢弃浏览器提交的租户与 Principal 信息，并在服务端验证会话后重新注入可信数据。
@@ -93,18 +108,38 @@ Tool 的原始结果不会直接返回浏览器，而是仅保留在本次 Agent
 
 ## 安全边界
 
-- 将 Nivora 绑定到 `127.0.0.1` 或私有 VPC 地址。
-- 不要通过公网反向代理暴露 `3100` 端口。
-- 产品到 Nivora、Nivora 到 Provider 应使用不同的服务密钥。
+- 将 Nivora 及其参考服务绑定到回环地址或私有 VPC 地址。
+- 不要通过公网反向代理暴露 `3100`、`3110` 或 synthetic Provider。
+- 产品到 Nivora、Nivora 到 Provider、Provider 到知识服务应分别使用不同的服务密钥。
 - Provider API 必须校验用户对业务资源的归属，并剥离内部敏感字段。
 - 匿名请求只能获得明确授予的知识检索与创建客服工单 Scope。
+- 持久化存储只保存公开消息和脱敏审计元数据，不保存思维链、Bearer Context、Tool 原始载荷或业务内部配方。
+- Shadow 结果只保存回答哈希和字节数，不保存回答正文。
+- synthetic Provider 只能用于隔离验收环境，严禁接收生产客户流量。
 - Nivora 当前只允许读取操作和具有幂等保护的 `case.create`，不自动退款、补偿、删除、取消或修改权限。
 
+## 生产验收命令
+
+```bash
+make probe          # 确定性 HTTP 安全边界测试
+make eval           # 客服能力回归评测
+make knowledge-eval # 已审核知识召回评测
+make load           # 有界 SSE 压力与时延测试
+make shadow         # 隐私安全的 Baseline/Candidate 对比
+make test-provider  # 隔离预发布环境使用的 synthetic Provider
+```
+
+仓库还提供 `.github/workflows/acceptance.yml` 手动验收工作流。它依赖受保护的 `nivora-staging` Environment 变量和 Secrets，不会自动对生产环境发起测试。
+
 ## 文档
 
 - [Runtime API v1](docs/runtime-api.md)
 - [Provider API v1](docs/provider-api.md)
+- [CozeLoop 接入](docs/cozeloop.md)
+- [VikingDB 已审核知识服务](docs/approved-knowledge.md)
+- [持久化会话与审计](docs/durable-storage.md)
 - [客服回归评测](docs/evaluation.md)
+- [生产验收门槛](docs/production-acceptance.md)
 - [火山引擎生产技术栈](docs/volcengine-production-stack.md)
 
 ## 开发
@@ -114,13 +149,9 @@ make fmt
 make test
 make vet
 make build
-make eval
 ```
 
 ## 路线图
 
-1. 接入 CozeLoop 链路追踪、Prompt 版本管理、Token 统计和评测器分数，并实施严格脱敏。
-2. 建设 Provider 管理的知识检索链路，可使用 VikingDB 存储已审核知识向量。
-3. 在 Nivora 自有存储中加入持久化会话、审计日志和客服工单。
-4. 加入 Shadow 和 Canary 模式，实现安全的生产灰度发布。
-5. 对未来的高风险写操作加入 Eino Interrupt/Resume 人工审批流程。
+1. 在公司的隔离预发布环境中执行完整验收门槛，确定获批的 SLO、质量、成本和回滚基线。
+2. 在未来引入任何高风险写入 Tool 前，使用 Eino Interrupt/Resume 加入明确的人工审批流程。