diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index 39a2c9a6..1bb0ed74 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -1,10 +1,10 @@ +# PAUSED: public CE releases stay disabled until the repository-history exposure +# review is closed and a maintainer deliberately removes this workflow guard. # Build and publish the HugAgentOS Tauri desktop client from the public CE tree. # # This workflow belongs to the CE overlay because the main repository excludes # `.github/**` during derivation. Windows and macOS stage the already-derived -# tracked CE checkout as their local-server payload; the private FULL checkout -# continues to generate that payload with scripts/build_ce.py before it reaches -# this tree. +# tracked CE checkout as their local-server payload. # # Required repository secrets: # TAURI_SIGNING_PRIVATE_KEY @@ -34,6 +34,7 @@ concurrency: jobs: validate-release: + if: ${{ false }} # Security hold: CE release publication is intentionally paused. runs-on: ubuntu-22.04 outputs: version: ${{ steps.desktop_version.outputs.version }} diff --git a/README.md b/README.md index 2cecc74a..865e2ba1 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,10 @@ run code, and carry real tasks through to completion.

+

+ HugAgentOS capability overview poster +

+

English · 简体中文 diff --git a/README_CN.md b/README_CN.md index 83edeee3..eb220144 100644 --- a/README_CN.md +++ b/README_CN.md @@ -15,6 +15,10 @@ 并持续完成真实任务。

+

+ HugAgentOS 功能概览海报 +

+

English · 简体中文 diff --git a/assets/poster-cn.png b/assets/poster-cn.png new file mode 100644 index 00000000..5d6303f0 Binary files /dev/null and b/assets/poster-cn.png differ diff --git a/assets/poster.png b/assets/poster.png new file mode 100644 index 00000000..f152254e Binary files /dev/null and b/assets/poster.png differ diff --git a/desktop/README.md b/desktop/README.md index 0d277f52..b15fafe1 100644 --- a/desktop/README.md +++ b/desktop/README.md @@ -113,7 +113,7 @@ HUGAGENT_SERVER_BASE=https://你的后端 npm run dev > Windows 和 macOS overlay 的 `beforeBuildCommand` 会运行 `scripts/prepare-bundle.mjs`：构建 > 桌面前端、准备 CE 服务树、构建 CE 登录前端，并删除构建期 `node_modules` 后再交给 Tauri -> 打包。FULL 主仓存在 +> 打包。源代码仓存在 > `scripts/build_ce.py` 时，脚本正常运行生成器并执行开源边界门禁；公开 CE 仓不含生成器，脚本会先 > 校验根目录 `.hugagent-edition` 为 `ce`，再只复制当前已派生 checkout 中的 Git tracked 文件。 > Linux 仍只构建桌面前端，不携带 Windows 本机服务载荷；dev 模式从仓库内 @@ -143,7 +143,7 @@ HUGAGENT_SERVER_BASE=https://你的后端 npm run dev | `resources/server-bootstrap/install-local-server.ps1` | Windows 用户目录内创建 Python 环境并安装随包 CE 服务 | | `resources/server-bootstrap/install-local-server.sh` | macOS 应用数据目录内准备独立 Python 运行时并安装随包 CE 服务 | | `scripts/prepare-bundle.mjs` | 发行构建前生成同版本 CE 服务资源和 `desktop-bundle.json` | -| `scripts/ce-payload.mjs` | 在公开 CE 仓校验版本标识并只暂存 tracked tree，FULL 仓仍走生成器 | +| `scripts/ce-payload.mjs` | 在派生 CE 仓校验版本标识并只暂存 tracked tree，源代码仓仍走生成器 | | `scripts/validate-release-version.mjs` | CI 三平台矩阵启动前校验桌面版本文件与 release tag | | `src-tauri/capabilities/default.json` | 插件权限（opener / deep-link / notification / global-shortcut / updater） | diff --git a/desktop/scripts/ce-payload.mjs b/desktop/scripts/ce-payload.mjs index 25d39365..1f38f19b 100644 --- a/desktop/scripts/ce-payload.mjs +++ b/desktop/scripts/ce-payload.mjs @@ -43,24 +43,23 @@ export function assertDerivedCeRepository(repoRoot) { } export function assertCleanRepository(repoRoot) { - // Tauri may touch Cargo.toml while inspecting dependencies. On Windows with - // CRLF conversion, `git status` can report that stat-only rewrite as modified - // even when the normalized content is identical to HEAD. Compare content - // directly so only real staged or unstaged tracked changes block a release. - const result = spawnSync("git", ["diff", "--quiet", "HEAD", "--"], { + // Release payloads require a genuinely clean checkout: staged, unstaged, and + // non-ignored untracked files all block staging. This is deliberately stricter + // than a content-only diff because release inputs must be auditable from HEAD. + const result = spawnSync("git", ["status", "--porcelain", "--untracked-files=all"], { cwd: repoRoot, encoding: "utf8", shell: false, }); if (result.error) throw result.error; - if (result.status === 1) { + if (result.status !== 0) { throw new Error( - "Desktop release payloads must be built from a clean Git checkout", + `git status --porcelain failed with exit code ${result.status}`, ); } - if (result.status !== 0) { + if (result.stdout.trim()) { throw new Error( - `git diff --quiet HEAD -- failed with exit code ${result.status}`, + "Desktop release payloads must be built from a clean Git checkout", ); } } diff --git a/desktop/scripts/ce-payload.test.mjs b/desktop/scripts/ce-payload.test.mjs index eeb6bf9f..7fe58884 100644 --- a/desktop/scripts/ce-payload.test.mjs +++ b/desktop/scripts/ce-payload.test.mjs @@ -97,6 +97,20 @@ test("rejects release staging when a tracked change is staged", () => { } }); +test("rejects release staging when a non-ignored untracked file exists", () => { + const root = createCeFixture(); + const output = join(root, "desktop", "generated", "server-ce"); + try { + writeFileSync(join(root, "stray-source.py"), 'print("untracked")\n'); + assert.throws( + () => stageTrackedCeRepository(root, output, { requireClean: true }), + /clean Git checkout/, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + test("rejects fallback staging without the derived CE marker", () => { const root = mkdtempSync(join(tmpdir(), "hugagent-not-ce-")); try { diff --git a/desktop/scripts/prepare-bundle.mjs b/desktop/scripts/prepare-bundle.mjs index 973f4c33..53b4a5ce 100644 --- a/desktop/scripts/prepare-bundle.mjs +++ b/desktop/scripts/prepare-bundle.mjs @@ -68,7 +68,7 @@ if (existsSync(ceBuilder)) { if (!requireClean) ceArgs.push("--allow-dirty"); console.log( - "[desktop] Generating the CE server payload from the FULL source tree", + "[desktop] Generating the CE server payload from the source checkout", ); rmSync(generatedRoot, { recursive: true, diff --git a/desktop/src-tauri/src/auth.rs b/desktop/src-tauri/src/auth.rs index 04d51569..c9370c1d 100644 --- a/desktop/src-tauri/src/auth.rs +++ b/desktop/src-tauri/src/auth.rs @@ -55,7 +55,10 @@ pub async fn redeem( return Err(format!("换票失败: HTTP {}", resp.status())); } - let body: serde_json::Value = resp.json().await.map_err(|e| format!("响应解析失败: {e}"))?; + let body: serde_json::Value = resp + .json() + .await + .map_err(|e| format!("响应解析失败: {e}"))?; // 后端统一信封 { code, message, data: { token, cookie_name, expires_at } } let token = body .get("data") diff --git a/desktop/src-tauri/src/notify.rs b/desktop/src-tauri/src/notify.rs index b6610bb9..8d4efaff 100644 --- a/desktop/src-tauri/src/notify.rs +++ b/desktop/src-tauri/src/notify.rs @@ -69,7 +69,10 @@ pub fn start(app: AppHandle, port: u16, token: Arc>>, http } let status = it.get("status").and_then(|v| v.as_str()).unwrap_or(""); - let name = it.get("task_name").and_then(|v| v.as_str()).unwrap_or("任务"); + let name = it + .get("task_name") + .and_then(|v| v.as_str()) + .unwrap_or("任务"); let summary = it.get("summary").and_then(|v| v.as_str()).unwrap_or(""); let title = if status == "failed" { @@ -83,7 +86,12 @@ pub fn start(app: AppHandle, port: u16, token: Arc>>, http format!("{}：{}", name, summary) }; - let _ = app.notification().builder().title(title).body(body_text).show(); + let _ = app + .notification() + .builder() + .title(title) + .body(body_text) + .show(); } if seen.len() > SEEN_CAP { diff --git a/document/en/api/error-codes.md b/document/en/api/error-codes.md index 04c8929d..9fcaf685 100644 --- a/document/en/api/error-codes.md +++ b/document/en/api/error-codes.md @@ -1,8 +1,8 @@ # Error Code Reference -> Last updated: 2026-06-11 +> Last updated: 2026-07-22 -This document reflects what is actually implemented in code: business exception classes are defined in `src/backend/core/infra/exceptions.py` (the two license-related ones live in `src/backend/core/licensing/features.py`) and are converted into the [unified response envelope](overview.md#unified-response-envelope) by the global exception handler `src/backend/api/middleware/error_handler.py`. Only **implemented** error codes are listed here; unlisted positions within each range are reserved for future use. +This document reflects what is actually implemented in code: business exception classes are defined in `src/backend/core/infra/exceptions.py` (the two EE license-related ones live in `src/backend/edition_ee/licensing/features.py`) and are converted into the [unified response envelope](overview.md#unified-response-envelope) by the global exception handler `src/backend/api/middleware/error_handler.py`. Only **implemented** error codes are listed here; unlisted positions within each range are reserved for future use. ## Error Response Shape @@ -108,7 +108,7 @@ A mismatched `require_admin` / `require_config` token is even simpler: just `{"d ## Unlicensed Features (HTTP 402) -EE routes are mounted with license feature guards per the registry in `api/routes/v1/__init__.py` (`core/licensing/deps.py` → `requires_feature`). When a feature is not licensed, `FeatureNotLicensed` is raised and rendered by `error_handler` as: +EE routes are mounted with license feature guards per the registry in `edition_ee/routes/registry.py` (`edition_ee/licensing/deps.py` → `requires_feature`). When a feature is not licensed, `FeatureNotLicensed` is raised and rendered by `error_handler` as: ```json { @@ -120,7 +120,7 @@ EE routes are mounted with license feature guards per the registry in `api/route } ``` -Design notes (`core/licensing/features.py`): +Design notes (`edition_ee/licensing/features.py`): - `FeatureNotLicensed` is the **single source** of the 402 envelope; routes/services must never hand-roll `HTTPException(402)`. - 402 was chosen over 403 deliberately: the frontend treats 403 as session expiry and forces a logout, and a missing license must not log the user out. diff --git a/document/en/api/overview.md b/document/en/api/overview.md index 2a5a6378..10d8cc84 100644 --- a/document/en/api/overview.md +++ b/document/en/api/overview.md @@ -228,11 +228,11 @@ The last column is the license feature flag declared in `EE_ROUTERS`; `—` mean | Observability & audit | `admin_logs.py` | `/v1/admin/logs` | `GET /tools`, `GET /subagents`, `GET /trace/{trace_id}` | CONFIG | `audit` | | Login & session | `auth.py` | `/v1/auth` | `POST /ticket/exchange` (SSO ticket → session), `GET /session/check`, `POST /logout` | Public (session infrastructure) | — | | Config console | `config_verify.py` | `/v1/config` | `GET /verify` (CONFIG_TOKEN validation) | CONFIG | — | -| Config console | `config_license.py` | `/v1/config/license` | `GET /` (license details), `POST /` (replace license) | CONFIG | — | -| Multi-tenancy | `config_users.py` | `/v1/config/users` | `GET /`, `PATCH /{user_id}/status`, `POST /{user_id}/reset-password` | CONFIG | `multi_tenancy` | -| Multi-tenancy | `config_teams.py` | `/v1/config/teams` | `GET/POST /`, `POST /{team_id}/members` | CONFIG | `multi_tenancy` | -| Multi-tenancy | `config_invites.py` | `/v1/config/invite-codes` | `GET/POST /`, `POST /{code}/revoke` | CONFIG | `multi_tenancy` | -| Multi-tenancy | `team_files.py` | `/v1/my-teams`, `/v1/teams`, `/v1/artifacts` | `GET /my-teams`, `POST /teams/{id}/files/upload`, `POST /artifacts/{id}/move-to-team` | User + team file permission | `multi_tenancy` | +| Config console | `edition_ee/routes/config_license.py` | `/v1/config/license` | `GET /` (license details), `POST /` (replace license) | CONFIG | — | +| Multi-tenancy | `edition_ee/routes/config_users.py` | `/v1/config/users` | `GET /`, `PATCH /{user_id}/status`, `POST /{user_id}/reset-password` | CONFIG | `multi_tenancy` | +| Multi-tenancy | `edition_ee/routes/config_teams.py` | `/v1/config/teams` | `GET/POST /`, `POST /{team_id}/members` | CONFIG | `multi_tenancy` | +| Multi-tenancy | `edition_ee/routes/config_invites.py` | `/v1/config/invite-codes` | `GET/POST /`, `POST /{code}/revoke` | CONFIG | `multi_tenancy` | +| Multi-tenancy | `edition_ee/routes/team_files.py` | `/v1/my-teams`, `/v1/teams`, `/v1/artifacts` | `GET /my-teams`, `POST /teams/{id}/files/upload`, `POST /artifacts/{id}/move-to-team` | User + team file permission | `multi_tenancy` | | System config | `config_security.py` | `/v1/config/security` | `GET /sandbox/overview`, `GET /audit-logs`, `GET /system-health` | CONFIG | `system_config` | | System config | `service_configs.py` | `/v1/service-configs` | `GET/PUT /`, `POST /test/{group_key}` (external service connectivity tests) | CONFIG | `system_config` | diff --git a/document/en/architecture/backend.md b/document/en/architecture/backend.md index 2043963c..19d0e99e 100644 --- a/document/en/architecture/backend.md +++ b/document/en/architecture/backend.md @@ -1,6 +1,6 @@ # Backend Architecture -> Last updated: 2026-07-19 +> Last updated: 2026-07-22 The backend lives in `src/backend/` and is a cleanly layered FastAPI monolith: the API layer handles only protocol and auth, the orchestration layer turns each chat into a resumable streaming Run, `core/` holds all domain logic, and MCP tools plus the script-execution sidecar run as independent processes. This page walks the stack top-down. @@ -8,7 +8,8 @@ The backend lives in `src/backend/` and is a cleanly layered FastAPI monolith: t ``` src/backend/ -├── api/ # FastAPI app, middleware, 74 v1 route files +├── api/ # FastAPI app, middleware, and CE routes +├── edition_ee/ # EE routes, License, Team/RBAC, ORM, Dify, and service implementations ├── orchestration/ # Chat orchestration: Run executor, workflow, strategy, citations, schedulers ├── core/ # Domain core: 17 submodules (auth/llm/db/ontology/services/...) ├── mcp_servers/ # 10 standalone MCP servers (streamable-http processes) @@ -133,10 +134,10 @@ Enterprise Edition (EE): `team_service` / `team_folder_service` / `sso_sync` (te |---|---| | `core/chat` | Workflow context assembly (`context.py`), SSE tool-log event construction (`tool_log.py`) | | `core/content` | Attachment parsing (`file_parser.py`), KB document chunking/keywords/vectorization (`kb_processing.py`), upload validation (`file_validation.py`), artifact reading and summaries (`artifact_reader/refs/summary.py`), content-block import/export (`content_blocks.py`), `svg_fit.py` | -| `core/kb` | Private-KB parsing and parent-child chunking (`kb_parser.py`), Milvus vector store (`kb_vector.py`), Dify external-KB client (`dify_kb.py`; external KB integration is an Enterprise Edition (EE) add-on) | +| `core/kb` / `edition_ee/kb` | Shared private-KB parsing and Milvus vector storage stay in `core/kb`; the Dify client and external retrieval adapter live only in `edition_ee/kb` (EE) | | `core/artifacts` | Artifact store `store.py`: local / OSS dual mode | | `core/infra` | Unified responses (`responses.py`), exceptions (`exceptions.py`), structured logging (`logging.py`), rate limiting (`rate_limit.py`), Redis singleton (`redis.py`), metrics (`metrics.py`), background-task registry (`runtime_state.py`), data masking (`data_masking.py`), distillation budget gates (`distillation_budget.py`, Enterprise Edition, EE) | -| `core/licensing` | License facade `manager.py` (GitLab-style offline model: signed file + in-process verification), feature enum `features.py`, FastAPI guard dependency `deps.py`, seat counting `seats.py`; the verification implementation `_ee_verify.py` (Enterprise Edition, EE — replaced by a hard-`False` stub in the CE tree) | +| `edition_ee/licensing` | Feature enums, signature verification, clock guard, seat policy, middleware, and the manager all exist only in EE; the CE tree has no License package and gets a fixed edition probe from its `api/middleware/edition.py` overlay | | `core/storage` | Storage protocol `protocol.py` + factory `factory.py`; `local.py` (CE), `s3.py` / `oss.py` (Enterprise Edition, EE) | ## orchestration/ — the orchestration layer @@ -167,9 +168,11 @@ Enterprise Edition (EE): `team_service` / `team_folder_service` / `sso_sync` (te ### The router registry (CE/EE seam C1) -`api/routes/v1/__init__.py` is the registry shared by both editions: `CE_ROUTERS` (39 entries) register unconditionally; `EE_ROUTERS` (32 entries) each carry a license feature bit enforced by `core/licensing/deps.py` as the second line of defense (the first being that the CE derived tree physically deletes those files). Three entries — `config_verify` / `config_license` / `auth` — are explicitly exempt so that an expired license can still be replaced. +The full repository composes `CE_ROUTERS` in `api/routes/v1/__init__.py` with `EE_ROUTERS` in `edition_ee/routes/registry.py` (currently 36 entries each). EE entries carry a License feature bit enforced by `edition_ee/licensing/deps.py` as the second line of defense (the first is physical removal of EE files from the CE tree); the CE overlay fixes `EE_ROUTERS` to empty. Three entries — `config_verify` / `config_license` / `auth` — are explicitly exempt so login and license replacement remain reachable after expiry. -### Route file groups (74 files under v1) +### Route file groups + +CE routes live under `api/routes/v1/`; physically split commercial routes live under `edition_ee/routes/`, while remaining historical admin routes are mounted only by the EE registry and its feature guards. | Group | Files | |---|---| @@ -197,7 +200,7 @@ Enterprise Edition (EE): `team_service` / `team_folder_service` / `sso_sync` (te 2. **orchestration does orchestration only**: it chains domain services into streaming workflows and never touches the ORM directly; 3. **core/services is the only business entrance**: routes must not bypass the service layer to query `core/db/models` (a handful of read-only fast paths excepted); 4. **Process boundaries are failure boundaries**: MCP, script-runner, and sandboxes all run as separate processes/containers and interact with the backend only through protocol layers; -5. **CE/EE seams stay concentrated**: the router registry, `edition_tables`, `permissions_iface`, and the licensing facade are the four choke points — business code carries no scattered `if edition` branches. +5. **CE/EE seams stay concentrated**: the router registry, `edition_tables`, `permissions_iface`, and edition middleware are the choke points; commercial implementations move under `edition_ee` instead of scattering `if edition` branches through business code. ## Related Source @@ -210,6 +213,6 @@ Enterprise Edition (EE): `team_service` / `team_folder_service` / `sso_sync` (te | Capability catalog | `src/backend/core/config/catalog.py` | | Sandbox protocol | `src/backend/core/sandbox/protocol.py` | | Memory pipeline | `src/backend/core/memory/pipeline.py` | -| License facade | `src/backend/core/licensing/manager.py` | +| EE License implementation / CE edition middleware | `src/backend/edition_ee/licensing/`, `src/backend/api/middleware/edition.py` | | Skill engine | `src/backend/core/agent_skills/loader.py` | | MCP port table | `src/backend/mcp_servers/_ports.py` | diff --git a/document/en/architecture/data-model.md b/document/en/architecture/data-model.md index a4ff378e..04baa173 100644 --- a/document/en/architecture/data-model.md +++ b/document/en/architecture/data-model.md @@ -141,24 +141,26 @@ Tables marked "(Enterprise Edition, EE)" belong to the `EE_ONLY_TABLES` set and - **EE main chain**: 53 migrations under `src/backend/alembic/versions/`, evolving from the initial schema (including structural moves such as MCP-to-streamable-http and the retirement of the office MCPs in favor of skills). Common commands: `alembic upgrade head`, `make migrate-new msg="..."` (autogenerate is driven by `core/db/models` metadata); - **Startup fallback**: the lifespan hook `_startup_ensure_tables` in `api/app.py` calls `core/db/engine.py::init_db`, which idempotently fills in missing tables for the SQLite dev database; -- **Independent CE chain**: the CE derived tree excludes the entire main chain; the overlay supplies a single baseline, `ce/overlay/src/backend/alembic/versions/ce_0001_initial.py` — `create_all` from SQLAlchemy metadata filtered by `EE_ONLY_TABLES`, dialect-aware (works on both SQLite and PostgreSQL). Subsequent CE schema evolution appends regular migrations on that chain. +- **Independent CE chain**: the CE derived tree excludes the entire main chain; the overlay supplies a single baseline, `ce/overlay/src/backend/alembic/versions/ce_0001_initial.py` — `create_all` directly from CE-only SQLAlchemy metadata, dialect-aware on SQLite and PostgreSQL. Subsequent CE schema evolution appends regular migrations on that chain. ## The CE/EE Table Boundary (core/db/edition_tables.py) -The `core.db.models` package is shared by both editions (EE model class definitions are harmless), but CE must not create empty EE-only tables. `EE_ONLY_TABLES` is the single source of truth for this boundary — 18 tables: +EE ORM classes are concentrated under `edition_ee/db/models/`, a package physically absent from the CE derived tree. Full-source validation uses `edition_ee/db/edition_tables.py::EE_ONLY_TABLES`, while the release gate independently forbids the same 20 tables: ``` teams · team_members · team_folders · invite_codes # multi-tenancy / SSO / invites roles · role_assignments # organization role model +chat_session_user_states # per-member state for shared team chats kb_grants # per-user / per-team KB grants -audit_logs · memory_audit # audit (the CE memory audit is a stub — no table) +marketplace_visibility_grants # scoped marketplace visibility +audit_logs · memory_audit # audit (CE contains no implementation or table) model_pricing # billing data_sources · ds_table_meta · ds_column_meta · ds_golden_sql # data sources / metadata governance gateway_virtual_keys # external model-gateway virtual key mirror sandbox_rebuilds · admin_skill_drafts · distillation_runs # sandbox rebuilds / skill distillation ``` -`ce_create_all(bind)` creates every non-EE table on a **cloned MetaData**: cross-boundary foreign keys from CE tables into EE tables (e.g. `projects/artifacts → teams/team_folders`, scheme D3 "keep the column, always NULL") would make PostgreSQL fail because the referenced tables don't exist — so those constraints are stripped on the clone (columns kept, the original metadata untouched, ORM mappings unaffected). Both table-creation entry points filter identically from the same source: the CE branch of `init_db` (filtering only when `JX_EDITION=ce`) and the CE migration baseline `ce_0001`. Maintenance rule: any new EE-only model must be added to `EE_ONLY_TABLES`; set membership is asserted against the real metadata table names at startup, so a renamed model cannot silently degrade the boundary into create-everything. +In the full source checkout, `ce_create_all(bind)` filters those names and cross-boundary foreign keys on a cloned MetaData for local boundary validation. The actual CE tree never imports EE ORM at all; its overlay clones only the registered CE metadata and defensively strips any foreign key whose target table is absent. Adding an EE model therefore requires coordinated updates to `EE_ONLY_TABLES`, the forbidden-table contract in `ce/manifest.yaml`, and any affected overlay. A few tables that *look* EE but are required by CE are deliberately excluded from the set: `admin_prompt_parts` (read by the prompt runtime), `memory_sanitizer_rules` (queried unconditionally by the scrubbing gate), `admin_skills` / `admin_mcp_servers` (personal self-service capabilities, owner-isolated), and `marketplace_submissions` (CE keeps the submission endpoint). @@ -166,7 +168,7 @@ A few tables that *look* EE but are required by CE are deliberately excluded fro | Topic | Path | |---|---| -| ORM model package | `src/backend/core/db/models/` | +| Shared ORM / EE ORM | `src/backend/core/db/models/`, `src/backend/edition_ee/db/models/` | | Ontology repository | `src/backend/core/db/repository/ontology.py` | | Engine and startup table creation | `src/backend/core/db/engine.py` | | Repository layer | `src/backend/core/db/repository/` | diff --git a/document/en/architecture/overview.md b/document/en/architecture/overview.md index 5212bfb3..f5700a0b 100644 --- a/document/en/architecture/overview.md +++ b/document/en/architecture/overview.md @@ -225,7 +225,7 @@ Streaming chats are not tied to the HTTP connection lifecycle: `chat_run_executo ### One codebase, two editions -The commercial main repository carries the full code; the Community Edition is derived by `scripts/build_ce.py` according to `ce/manifest.yaml` (exclude EE files + brand-neutralizing text transforms + overlay). Three runtime seams: the router registry `api/routes/v1/__init__.py` (EE routers carry license feature bits), the table-creation boundary `core/db/edition_tables.py` (CE skips 18 EE-only tables), and the license facade `core/licensing/manager.py` (all feature bits are hard-`False` under CE). See [Editions](../editions/overview.md). +The commercial main repository carries the full code; the Community Edition is derived by `scripts/build_ce.py` according to `ce/manifest.yaml` (exclude `edition_ee` implementations + text transforms + CE overlays). Runtime boundaries include the router registry `api/routes/v1/__init__.py`, the table-creation boundary `core/db/edition_tables.py` (CE never registers the 20 EE-only tables), EE's `edition_ee/licensing/` package versus CE's fixed edition middleware, and policy seams such as site visibility. See [Editions](../editions/overview.md). ## Related Source @@ -240,5 +240,5 @@ The commercial main repository carries the full code; the Community Edition is d | Capability catalog | `src/backend/core/config/catalog.json`, `catalog.py` | | Response envelope | `src/backend/core/infra/responses.py` | | Prompt assembly | `src/backend/prompts/prompt_runtime.py` | -| CE/EE seams | `src/backend/api/routes/v1/__init__.py`, `src/backend/core/db/edition_tables.py`, `src/backend/core/licensing/` | +| CE/EE seams | `src/backend/api/routes/v1/__init__.py`, `src/backend/core/db/edition_tables.py`, `src/backend/api/middleware/edition.py`, `src/backend/edition_ee/licensing/` | | Container orchestration | `docker-compose.yml` | diff --git a/document/en/deployment/docker-compose.md b/document/en/deployment/docker-compose.md index 33a79a87..4c3fbdc8 100644 --- a/document/en/deployment/docker-compose.md +++ b/document/en/deployment/docker-compose.md @@ -1,6 +1,6 @@ # Docker Compose Deployment -> Last updated: 2026-07-19 ｜ [简体中文](../../zh-CN/deployment/docker-compose.md) ｜ Back to [Deployment Guide](README.md) +> Last updated: July 23, 2026 ｜ [简体中文](../../zh-CN/deployment/docker-compose.md) ｜ Back to [Deployment Guide](README.md) > **When to use**: the **standard deployment form** for teams / production — multi-user, full features. For a personal single-machine trial, the lighter [No-Docker Quick Install](quick-install.md) is available. @@ -15,11 +15,18 @@ All HugAgentOS services are orchestrated by a single `docker-compose.yml` at the | `postgres` | hugagent-postgres | `postgres:15-alpine` | `${POSTGRES_HOST_PORT:-5432}:5432` | Primary relational DB (business data, content_blocks, usage logs) | | `redis` | hugagent-redis | `redis:7-alpine` | `${REDIS_HOST_PORT:-6380}:6379` | Session store, streaming follower (Redis Streams), rate limiting | | `backend` | hugagent-backend | `docker/Dockerfile` (target `production`) | `${BACKEND_HOST_PORT:-3001}:${BACKEND_PORT:-3001}` | FastAPI app; runs alembic migrations automatically at startup | -| `mcp` | hugagent-mcp | `docker/Dockerfile.mcp` | none exposed | 10 MCP servers listening on streamable-http ports `9100–9108` and `9112`; the backend calls them at `http://mcp:91XX/mcp/` | +| `mcp` | hugagent-mcp | `docker/Dockerfile.mcp` | none exposed | CE starts 9 general MCP servers; EE also starts commercial MCP servers such as database query. The backend calls them at `http://mcp:91XX/mcp/` | | `frontend` | hugagent-frontend | `src/frontend/Dockerfile` | `${FRONTEND_PORT:-3002}:80` | nginx serving the frontend static bundle + `/api` reverse proxy to the backend | `BACKEND_PORT` is the container-internal listen port used by nginx, MCP, and the health check, and should normally remain `3001`. If host ports are occupied, adjust only `BACKEND_HOST_PORT`, `POSTGRES_HOST_PORT`, or `REDIS_HOST_PORT`; do not change the container-internal ports. For example, publish the backend on `13003` while keeping its internal port at `3001`. +On the first startup with an empty database, CE globally installs and enables +the `automation`, `skill-manager`, and `sites` plugins. Every user can use them +without installing them separately from the plugin marketplace. The CE +capability page, runtime catalog, MCP port registry, and MCP image don't contain +the database-query tool. That capability exists only in deployments marked as +Enterprise Edition (EE). + ### Sandbox sidecars (pick one profile; mutually exclusive) | Service | Profile | Container | Image / build | Role | diff --git a/document/en/deployment/quick-install.md b/document/en/deployment/quick-install.md index 005ac65e..3816da24 100644 --- a/document/en/deployment/quick-install.md +++ b/document/en/deployment/quick-install.md @@ -1,6 +1,6 @@ # No-Docker Quick Install (Single Machine) -> Last updated: July 21, 2026 ｜ [简体中文](../../zh-CN/deployment/quick-install.md) ｜ Back to [Deployment Guide](README.md) +> Last updated: July 23, 2026 ｜ [简体中文](../../zh-CN/deployment/quick-install.md) ｜ Back to [Deployment Guide](README.md) The simplest way to deploy, aimed at **personal single-machine trials** and **development experience**: one command installs everything, a terminal wizard sets the admin account and configures the model, then a single process starts the server and opens the browser. Zero **Docker, PostgreSQL, and Redis**. @@ -62,7 +62,13 @@ The chat model is assigned to all 7 chat roles at once. Two more model types can > HugAgentOS has 9 model roles: 7 chat roles (main agent / summarizer / follow-up / memory / chart / planning / code execution — all share the chat model above) + embedding + reranker. Onboard covers all three types; after logging in you can also assign a different model per role under Settings → System → Model Services. -**Step 3 · Plugins** — pick which built-in plugins to install (comma-separated indices / `all` / `none`; Enter installs the ★ recommended set). Recommended: `automation` (scheduled tasks), `skill-manager` (skill authoring), `sites` (conversational site-building). Installing `sites` also provisions the React site-building template. You can add or remove plugins from the plugin market later. +**Step 3 · Plugin initialization** — a fresh installation automatically +installs and enables `automation` (scheduled tasks), `skill-manager` (skill +management), and `sites` (conversational site-building). Users don't need to +install them from the plugin marketplace after signing in. Installing `sites` +also provisions the React site-building template. After the first bootstrap, +users can still disable or uninstall these plugins; later restarts don't +restore a plugin that a user deliberately removed. **Step 4 (optional) · File parser** — parsing uploaded PDFs / scanned documents needs an external parser service (MinerU-compatible); enter its API URL to enable it (written to `file_parser.api_url`), or press Enter to skip. Excel / CSV / PPTX / text parse in-process and need none of this. @@ -132,7 +138,11 @@ The no-Docker single-machine mode is built to be lightweight. Here is how it dif - **Self-built vector knowledge base**: backed by embedded **Milvus Lite** (a single file, no server), **dense-only** retrieval; requires configuring an embedding model during onboarding. For stronger hybrid retrieval, point `MILVUS_URL` at a real Milvus server (switches back automatically). - **L2 vector memory**: the installer includes mem0 and Milvus Lite and enables the memory runtime by default. Persistent memory and automatic writes default to on after an active embedding provider is assigned; without one, both the frontend and backend prevent enabling memory. -- **Automation / skill-authoring / site-building plugin capabilities**: `automation` / `skill-manager` / `sites` are **plugins** — install them with one keystroke in onboard Step 3 (or add/remove them later from the plugin market); once installed their MCP is auto-reachable locally (`http://mcp:*` hostnames are rewritten to `127.0.0.1`). +- **Automation / skill-authoring / site-building plugin capabilities**: + `automation`, `skill-manager`, and `sites` are installed and enabled by + default on a fresh installation. Users don't need to visit the plugin + marketplace. Startup verifies their MCP tool lists, and local MCP addresses + use `127.0.0.1`. **Needs extra conditions** - **React project build for conversational site-building**: supported once the `sites` plugin is installed — onboard provisions the React template into `~/.hugagent/site-template/` and runs `npm install` on first build. **Requires host Node.js ≥ 20 + npm**; without it only hand-written static sites are possible. The build chain's `/workspace` paths are parameterized to the local workspace (static sites match the Docker form). diff --git a/document/en/development/backend.md b/document/en/development/backend.md index 568bbef0..627d69bc 100644 --- a/document/en/development/backend.md +++ b/document/en/development/backend.md @@ -100,7 +100,7 @@ raise ResourceNotFoundError("chat_session", chat_id) raise BadRequestError("parameter 'name' must not be empty") ``` -The same applies to license 402s: the single source is `FeatureNotLicensed` (40201) / `SeatLimitExceeded` (40202) in `core/licensing/features.py`; routes and services must not hand-craft 402s. Error-code ranges are documented in [Error Codes](../api/error-codes.md). +The same applies to license 402s: the single source is `FeatureNotLicensed` (40201) / `SeatLimitExceeded` (40202) in `edition_ee/licensing/features.py`; routes and services must not hand-craft 402s. Error-code ranges are documented in [Error Codes](../api/error-codes.md). ### Dependency injection @@ -138,7 +138,7 @@ Steps for a new route: 1. Create the route file under `api/routes/v1/` with `router = APIRouter(prefix="/v1/xxx", tags=["Xxx"])`; 2. Decide the edition: - **CE capability** (self-contained for an individual) → append `("module_name", "router")` to `CE_ROUTERS`; - - **EE capability** (organization-scale) → append `("module_name", "router", "")` to `EE_ROUTERS`, with the feature taken from `core/licensing/features.py::Feature`; use `None` only when the endpoint must remain reachable with an invalid license (login / license-swap infrastructure), and document why; + - **EE capability** (organization-scale) → append `("module_name", "router", "")` to `edition_ee/routes/registry.py::EE_ROUTERS`, with the feature taken from `edition_ee/licensing/features.py::Feature`; use `None` only when the endpoint must remain reachable with an invalid license (login / license-swap infrastructure), and document why; 3. For EE routes, also exclude the file in `ce/manifest.yaml` (`admin_*.py` / `config_*.py` are already covered by wildcard patterns); 4. Table order is registration order; relative order within a prefix family is invariant (e.g. the public-read `config` must precede the `config_*` console routes). diff --git a/document/en/editions/build-ce.md b/document/en/editions/build-ce.md index 6696b38b..d6021565 100644 --- a/document/en/editions/build-ce.md +++ b/document/en/editions/build-ce.md @@ -1,5 +1,5 @@ # CE Build Pipeline -> Last updated: 2026-07-02 +> Last updated: 2026-07-22 The Community Edition (CE) is not a separate branch. It is a subset tree **deterministically derived** from the main repo (EE, the single development source of truth) by `scripts/build_ce.py`, written to `dist/ce/`. The core constraint is the **whitelist iron rule: EE-only code is physically absent from the CE tree** — not commented out, not flag-disabled, but removed at the file level. The pipeline's only input is `ce/manifest.yaml`. @@ -25,7 +25,7 @@ The manifest has the following sections, in processing order: Glob patterns (relative to the repo root); a match means the file is never copied. Covers: -- **Backend EE modules**: SSO / team permissions (`core/auth/sso.py`, `team_permissions.py`, …), cloud storage (`core/storage/s3.py`, `oss.py`), persistent sandbox providers (the whole opensandbox / cube set), memory audit, skill distillation, the license verification implementation `core/licensing/_ee_verify.py`, EE services (team / sso_sync / distillation / sandbox_rebuild / security / cube_template_builder); +- **Backend EE modules**: the complete `edition_ee/**` implementation root (Team/RBAC, SSO, license verification and gate, EE ORM, Dify integration), cloud storage (`core/storage/s3.py`, `oss.py`), persistent sandbox providers, memory audit, skill distillation, and other EE services; - **EE routes**: `api/routes/v1/admin_*.py`, `config_*.py`, `audit.py`, `auth.py`, `team_files.py`, `service_configs.py`, `data_sources.py`, `db_metadata.py`, `gateway_*.py`; - **Industry MCP servers**: `mcp_servers/query_database_mcp/**`, `ai_chain_information_mcp/**`; - **The entire main-repo alembic chain** (`alembic/versions/**` — CE uses an independent chain from the overlay, see below); @@ -60,7 +60,7 @@ Content edits that plain text substitution cannot express, implemented in `build ### 4. `split` — assertion for files mixing user + admin endpoints -The three route files `content.py` / `models.py` / `projects.py` contain both user and admin endpoints; CE takes user-subset versions from the overlay. **build_ce.py asserts before the overlay step that these files exist in the overlay** — the main repo's full versions must never leak into CE; a missing file fails the build. +`manifest.split` explicitly lists every edition seam that CE must replace as a whole file. **build_ce.py asserts each replacement exists before applying the overlay** — a full source-tree implementation must never leak into CE; a missing replacement fails the build. ### 5. `overlay` — whole-file CE replacements / additions @@ -71,20 +71,29 @@ The three route files `content.py` / `models.py` / `projects.py` contain both us | `README.md` / `README_CN.md` / `LICENSE` / `NOTICE` / `CONTRIBUTING.md` / `SECURITY.md` | CE open-source repo front matter; English is the default README and Chinese remains available as a language alternative | | `install.sh` | Public one-command installer for the personal no-Docker profile | | `.env.example` | CE environment template (`JX_EDITION=ce`, no intranet IPs / brand defaults) | -| `.hugagent-edition` | Machine-readable `ce` marker used only after derivation; it lets release tooling distinguish a public CE checkout from a FULL checkout whose generator is unexpectedly missing | +| `.hugagent-edition` | Machine-readable `ce` marker used only after derivation; it lets release tooling distinguish a derived CE checkout from a source checkout whose generator is unexpectedly missing | | `.github/workflows/desktop-release.yml` | Public CE desktop release workflow, including the release-tag/version preflight gate | -| `src/backend/core/licensing/manager.py` | **CE stub**: `mode()` always `"ce"`, `has()` always False, unlimited seats, no verification logic at all | -| `src/backend/core/auth/permissions_iface.py` | Single-tenant permission-interface stub (seam C3): your own resources are always full-permission; team permission is always `none` (legacy team data migrated from EE must not become world-readable through a permissive stub) | +| `src/backend/api/routes/v1/__init__.py` | CE route registry; `EE_ROUTERS` is always empty | +| `src/backend/core/auth/permissions_iface.py` | Owner-only single-tenant authorization interface with no team permission exports | +| `src/backend/core/services/artifact_edition.py` | Personal artifact-scope interface with no team fields, permissions, or repository methods | +| `src/backend/core/llm/tools/edition_{myspace,myspace_vfs,artifact_recovery}.py` | Personal MySpace tool, VFS, and recovery interfaces; organization implementations do not enter CE | +| `src/backend/core/config/edition_display_names.py` | CE tool display names with no team-tool names | | `src/backend/core/memory/audit.py` | Memory-audit no-op stub (same interface, writes nothing) | | `src/backend/alembic/versions/ce_0001_initial.py` | CE independent migration-chain baseline (next section) | -| `src/backend/api/routes/v1/{content,models,projects}.py` | User-subset versions of the split files | +| `src/backend/api/routes/v1/{agents,content,kb_models,projects}.py` | CE API contracts with administration endpoints and organization fields removed | | `src/backend/mcp_servers/_ports.py` | Port table for the 8 general tools (EE industry-tool ports marked reserved) | | `src/frontend/default.conf.template` | CE frontend Nginx template with `/gateway/**` proxying and the litellm upstream removed | | `src/frontend/src/main.tsx` | CE entry: mounts only the main app / API docs / share preview — no /admin, no /config | | `src/frontend/src/updates.ts` | CE release-notes data | | `.claude/skills/hugagent-{backend,frontend}-dev/…` | CE versions of the project dev skills' SKILL.md and references (admin-console / EE router-registration sections stripped) | -> The router registry `api/routes/v1/__init__.py` needs **no** overlay copy: `iter_edition_routers` silently skips EE modules that are physically absent, so the same file is shared by both trees. +> License, Team/RBAC, EE ORM, and Dify implementations all live under `edition_ee/**`. CE supplies no same-name implementation stubs; module discovery for `edition_ee` must report that it is absent. + +The team-file repository, organization MySpace tools, VFS, and artifact recovery +implementations live in `edition_ee/db/artifact_repository.py` and +`edition_ee/services/{myspace_tools,myspace_vfs,artifact_recovery}.py`. +Shared modules expose only edition-neutral call interfaces. CE overlays implement the +personal profile without retaining commercial fields or tool names. ### 6. `brand_scan` — the brand-gate regex file @@ -93,7 +102,7 @@ The three route files `content.py` / `models.py` / `projects.py` contain both us ## build_ce.py step pipeline ``` -[1/7] Copy git ls-files (cached + untracked-unignored) as the whitelist, minus exclude +[1/7] Copy git ls-files --cached as the whitelist, minus exclude and default ignores [1/7] Rename apply optional path migrations from manifest.renames (currently empty) [2/7] Transform manifest.transforms tree-wide text rewrites (binaries skipped; source-code @@ -101,9 +110,11 @@ The three route files `content.py` / `models.py` / `projects.py` contain both us [3/7] Prune the five pruners in manifest.prunes [4/7] Overlay first assert the split files exist in the overlay, then layer the whole tree (skipping __pycache__/pyc) +[4/7] Forbidden assert zero EE paths, table names, foreign keys, and commercial runtime-source + symbols; test directories may retain only the negative contract assertions +[4/7] Binary gate every PNG/PDF/DOCX must match a manually/OCR-reviewed path + SHA-256 allowlist [5/7] Brand gate line-by-line text regex must hit zero + a full file-PATH scan (covers binary - asset filenames; an extra path-only pattern blocks commercial font files); - the count of unscannable binaries is reported with the result + asset filenames; an extra path-only pattern blocks commercial font files) [6/7] LICENSE gate refuse to generate while the overlay LICENSE is still placeholder text (contains the NOTE TO MAINTAINERS marker) [7/7] Self-checks --import-check / --pytest-check / --frontend-check (optional) @@ -113,21 +124,20 @@ The three route files `content.py` / `models.py` / `projects.py` contain both us Using `git ls-files` as the copy list means `.env`, local databases, and other untracked/ignored files **can never enter the CE tree**. -The Windows desktop payload follows the same boundary in both repositories. In FULL, `desktop/scripts/prepare-bundle.mjs` finds and runs `scripts/build_ce.py` as before. In the public CE repository, where the generator is intentionally absent, it requires `.hugagent-edition` to contain `ce` and stages only the current checkout's tracked files. Release builds reject a dirty checkout. This fallback cannot silently turn an arbitrary repository into a CE payload. +The Windows desktop payload follows the same boundary in both checkout types. In the source checkout, `desktop/scripts/prepare-bundle.mjs` finds and runs `scripts/build_ce.py`. In a derived CE checkout, where the generator is intentionally absent, it requires `.hugagent-edition` to contain `ce` and stages only the current checkout's tracked files. Release builds reject a dirty checkout. This fallback cannot silently turn an arbitrary repository into a CE payload. ## CE database differences -CE does not create EE-only tables; the single source of truth is `src/backend/core/db/edition_tables.py`: +CE does not register or create EE-only tables. Enterprise ORM classes live under `src/backend/edition_ee/db/models/`, and that package is physically absent from the derived tree. The CE model export contains only CE mappings; compatibility attributes come from CE model extensions and do not register commercial columns or tables. -- `EE_ONLY_TABLES` (18 tables): `teams`, `team_members`, `team_folders`, `invite_codes`, `roles`, `role_assignments`, `kb_grants`, `audit_logs`, `memory_audit`, `model_pricing`, `data_sources`, `ds_table_meta`, `ds_column_meta`, `ds_golden_sql`, `gateway_virtual_keys`, `sandbox_rebuilds`, `admin_skill_drafts`, `distillation_runs`. -- `ce_create_all(bind)`: creates tables on a **cloned MetaData** after filtering — CE tables carry cross-boundary FK constraints into EE tables (e.g. `projects → teams`; design decision D3 "keep the column, always NULL"); shipped as-is they would fail on PostgreSQL because the referenced tables do not exist, so the constraints are stripped on the clone (columns kept) while the original metadata and ORM mappings stay untouched. Every name in the set must actually exist in metadata (asserted in the function) so a renamed model cannot silently degrade the filter into a full create_all. +The release contract checks 20 forbidden table names: `chat_session_user_states`, `teams`, `team_members`, `team_folders`, `invite_codes`, `roles`, `role_assignments`, `kb_grants`, `marketplace_visibility_grants`, `audit_logs`, `memory_audit`, `model_pricing`, `data_sources`, `ds_table_meta`, `ds_column_meta`, `ds_golden_sql`, `gateway_virtual_keys`, `sandbox_rebuilds`, `admin_skill_drafts`, and `distillation_runs`. The import gate fails if any is registered in CE metadata or referenced by a CE foreign key. It also rejects the corresponding commercial-scope columns on `projects`, `artifacts`, `user_agents`, `chat_sessions`, `marketplace_listing_states`, and `sites`. -Two table-creation entry points share this filter: +Two table-creation entry points use the CE-only metadata: 1. The CE branch of `core/db/engine.py::init_db` (when `JX_EDITION=ce`, startup fallback uses `ce_create_all`); -2. The CE overlay migration baseline `ce_0001_initial.py` — CE runs an **independent alembic chain** (it does not replay the main repo's 50+ historical migrations); the baseline is "create_all filtered by `EE_ONLY_TABLES`", dialect-aware (SQLite and PostgreSQL), same source and same filter as init_db, both idempotent and non-conflicting. Subsequent CE schema evolution adds regular migrations on the `ce_0001` chain. +2. The CE overlay migration baseline `ce_0001_initial.py` — CE runs an **independent alembic chain** and creates directly from CE-only metadata, dialect-aware (SQLite and PostgreSQL), idempotent with `init_db`. Subsequent CE schema evolution adds regular migrations on the `ce_0001` chain. -EE (including internal / licensed and every other license state) always creates the full schema, identical to historical behavior. Maintenance rule: **when adding an EE-only model, add its table name to `EE_ONLY_TABLES`**. +EE always creates the full schema. Maintenance rule: **new EE mappings belong under `edition_ee/db/models`, and their table names must be added to the CE release contract**. ## Release acceptance @@ -135,20 +145,21 @@ A qualifying release build must pass all of: | Gate | Criterion | Enforced by | |---|---|---| -| Zero EE route leakage | the CE tree physically lacks EE routes/modules; `import api.app` succeeds under `--import-check` (missing modules skipped by the registry); `--pytest-check` collection reports no EE import errors | exclude + `iter_edition_routers` | -| Brand gate | zero text hits; full path scan passes; new binary assets (content-scan blind spot) require manual review | `brand_scan()` | +| Zero EE route/schema leakage | the CE tree physically lacks `edition_ee`; `EE_ROUTERS` is empty; organization routes, fields, and wording are absent from OpenAPI; forbidden tables, foreign keys, and commercial-scope columns are absent from metadata | `--import-check` | +| Zero commercial runtime symbols | backend and frontend runtime sources contain no Team/RBAC model, scope-field, permission, or tool symbols; tests retain only negative assertions | `find_forbidden_artifacts()` + CE runtime contract | +| Brand / binary gate | zero text hits and a clean full-path scan; every PNG/PDF/DOCX matches a manually/OCR-reviewed path + SHA-256 allowlist | `brand_scan()` + `binary_allowlist_check()` | | LICENSE gate | the overlay LICENSE is not placeholder text | `license_placeholder_check()` | -| Split assertion | the CE subsets of the three split files exist in the overlay | pre-overlay check in `main()` | +| Split assertion | every declared CE split replacement exists in the overlay | pre-overlay check in `main()` | | Frontend buildability | `--frontend-check`: npm install + vite build succeed | `frontend_check()` | | Delivery hygiene | all self-check residue removed | `cleanup_gate_artifacts()` | ## Day-to-day maintenance - **New EE route**: register in `EE_ROUTERS` (see [Backend Development Guide](../development/backend.md)) + add the file glob to `manifest.exclude` (`admin_*.py` / `config_*.py` are already covered by wildcards). -- **New EE table**: add the name to `EE_ONLY_TABLES`. +- **New EE table**: define it under `edition_ee/db/models` and add the name to `contracts.forbidden_tables`. - **New EE dependency / compose service**: add a drop entry to the relevant prune section. -- **New branded asset**: confirm brand_scan can intercept it at the path or text level; binaries are a content-scan blind spot and rely on path patterns + manual review. -- After changes, run `python scripts/build_ce.py --allow-dirty --import-check --pytest-check` to validate. +- **New PNG/PDF/DOCX**: manually inspect or OCR-review its content, then add its relative path and SHA-256 to `ce/binary_allowlist.sha256`; any hash change requires a fresh review. +- After changes, run `python scripts/build_ce.py --allow-dirty --import-check --pytest-check --frontend-check`. Release builds must not use `--allow-dirty`, and untracked files are never copy inputs. ## Related source diff --git a/document/en/editions/license.md b/document/en/editions/license.md index 4040e680..1551175a 100644 --- a/document/en/editions/license.md +++ b/document/en/editions/license.md @@ -1,11 +1,11 @@ # License Mechanism (Enterprise Edition) -> Last updated: 2026-06-11 +> Last updated: 2026-07-22 -The Enterprise Edition (EE) uses a **GitLab-style offline license model**: a single Ed25519-signed authorization file (`.lic`) verified in-process — **fully offline, no license server** — designed for air-gapped environments such as government intranets. This page documents the state machine, enforcement, issuance flow, and management UI; everything here is verified against the code in `src/backend/core/licensing/`. +The Enterprise Edition (EE) uses a **GitLab-style offline license model**: a single Ed25519-signed authorization file (`.lic`) verified in-process — **fully offline, no license server** — designed for air-gapped environments such as government intranets. Feature bits, verification, state machine, seat handling, and enforcement all live under `src/backend/edition_ee/licensing/`; the CE derived tree contains no license runtime code. ## License file format -A `.lic` file is a JSON envelope (`src/backend/core/licensing/_ee_verify.py`, format version `jx-license/1`): +A `.lic` file is a JSON envelope (`src/backend/edition_ee/licensing/verify.py`, format version `jx-license/1`): ```json { @@ -29,11 +29,11 @@ Payload fields: } ``` -The verification public key is built into `_ee_verify.py` (`_BUILTIN_PUBKEY`) and can be overridden via the `LICENSE_PUBLIC_KEY` environment variable (for key rotation). `_ee_verify.py` is an **EE-only module that never enters the CE derived tree** (explicitly excluded in `ce/manifest.yaml`; the CE tree's `manager.py` is replaced by an always-False stub via overlay). +The verification public key is built into `edition_ee/licensing/verify.py` (`_BUILTIN_PUBKEY`) and can be overridden via the `LICENSE_PUBLIC_KEY` environment variable (for key rotation). The entire `edition_ee/licensing` implementation is physically absent from the CE derived tree; CE supplies a fixed edition probe through its middleware overlay and contains neither a license manager nor a same-name verification stub. ## State machine -`core/licensing/manager.py::LicenseManager.mode()` returns one of seven states: +`edition_ee/licensing/manager.py::LicenseManager.mode()` returns one of seven states: | mode | Trigger | EE feature bits | |---|---|---| @@ -65,14 +65,14 @@ Key design points (all verifiable in `manager.py`): ### The Feature enum -`core/licensing/features.py::Feature` lists only **organization-level** commercial bits (automation / batch / personal canvas / L2–L3 memory belong to CE and are deliberately absent): +`edition_ee/licensing/features.py::Feature` lists only **organization-level** commercial bits (automation / batch / personal canvas / L2–L3 memory belong to CE and are deliberately absent): `sso`, `multi_tenancy`, `audit`, `memory_audit`, `billing`, `quota`, `persistent_sandbox`, `cloud_storage`, `industry_tools`, `content_admin`, `system_config`, `canvas_collab`, `whitelabel`. ### Two lines of defense 1. **First line: the router registry** — the CE tree physically lacks EE route files. See [CE Build Pipeline](build-ce.md). -2. **Second line: the `requires_feature` guard** (`core/licensing/deps.py`) — protects against "EE code is fully deployed but the license does not include a given capability pack". +2. **Second line: the `requires_feature` guard** (`edition_ee/licensing/deps.py`) — protects against "EE code is fully deployed but the license does not include a given capability pack". The mapping between EE routes and feature bits is declared in the registry `src/backend/api/routes/v1/__init__.py::EE_ROUTERS` (the third tuple element is the feature bit); `api/app.py` attaches guards from the table at registration time: @@ -85,7 +85,7 @@ The mapping between EE routes and feature bits is declared in the registry `src/ | `config_security`, `service_configs` | `system_config` | | `config_verify`, `config_license`, `auth` | **None (explicit exemption)** | -The three exemptions are deliberate: `config_verify` is the console login check, `config_license` is the entry point for swapping licenses, and `auth` is login/session infrastructure — all of these must remain reachable when the license is invalid, otherwise users are trapped in a "402 → logout → login → 402" loop with no way to replace the license. The SSO bit is not blanket-exempted at the router level; it guards itself: the authorize-url endpoint carries `requires_feature(Feature.SSO)` (`api/routes/v1/auth.py`), and remote ticket exchange checks inside `core/auth/sso.py::exchange_ticket`. +The three exemptions are deliberate: `config_verify` is the console login check, `config_license` is the entry point for swapping licenses, and `auth` is login/session infrastructure — all of these must remain reachable when the license is invalid, otherwise users are trapped in a "402 → logout → login → 402" loop with no way to replace the license. The SSO bit is not blanket-exempted at the router level; it guards itself: the authorize-url endpoint carries `requires_feature(Feature.SSO)` (`edition_ee/routes/auth.py`), and remote ticket exchange checks inside `edition_ee/auth/sso.py::exchange_ticket`. > Note: `quota` / `persistent_sandbox` / `cloud_storage` / `industry_tools` / `canvas_collab` / `whitelabel` / `memory_audit` are currently expressed in license entitlements and the probe but have **no router-level guard attached** — those boundaries are enforced mainly by physical exclusion from the CE tree and by deployment configuration. @@ -101,7 +101,7 @@ An unauthorized access raises `FeatureNotLicensed` (`features.py`), rendered by ## Seat limits -Seat counting has a single source of truth in `core/licensing/seats.py`: +Seat counting has a single source of truth in `edition_ee/licensing/seats.py`: - `seats_used(db)`: seats in use = the full row count of `users_shadow` (including SSO shadow accounts); - `seat_available(db)`: the check run before creating any user (shared by local sign-up and SSO auto-provisioning). Always allowed in CE / internal / unlimited (`seats=0`); under `licensed` / `grace` it requires `active_users < seats`; always denied under `expired` / `invalid` / `missing`; @@ -115,7 +115,7 @@ Seat counting has a single source of truth in `core/licensing/seats.py`: ### `GET/POST /v1/config/license` (CONFIG_TOKEN auth) -`api/routes/v1/config_license.py`: +`edition_ee/routes/config_license.py`: - `GET`: full status (`license_manager.status()` + `seats_used`), including live per-feature evaluation, grace days, and license metadata; - `POST`: upload the full `.lic` text (≤64 KB) to hot-swap. Flow: **verify before writing** (an invalid file never overwrites the current license) → reject activation of licenses past the grace window (within grace it is allowed, so a lost file / rebuilt host can re-attach the same license during the window) → atomic write to `LICENSE_KEY_PATH` (tmp file + `os.replace`) → `license_manager.reload()` takes effect immediately, **no restart**. Returns 400 if `LICENSE_KEY_PATH` is not configured. @@ -142,7 +142,7 @@ Usage examples: `components/settings/SettingsModal.tsx` (hides the Teams section # 1. Generate an Ed25519 keypair (one-time; keep the private key offline — # a leak allows arbitrary issuance) python scripts/license_tool.py keygen --out-dir ~/jx-license-keys -# The printed public key goes into core/licensing/_ee_verify.py::_BUILTIN_PUBKEY +# The printed public key goes into edition_ee/licensing/verify.py::_BUILTIN_PUBKEY # (or LICENSE_PUBLIC_KEY on the customer side) # 2. Issue @@ -159,7 +159,7 @@ python scripts/license_tool.py inspect customer.lic python scripts/license_tool.py verify customer.lic --pub ~/jx-license-keys/license_signing.pub ``` -`issue` auto-generates `license_id` (`lic_` + 16 hex chars) and validates date formats; `--seats 0` means unlimited; `--features` is a comma-separated bit list, `"*"` for everything. The envelope format and verification logic have a single source of truth in the backend's `_ee_verify.py`, which the tool reuses directly (passing the public key explicitly to avoid pulling in the backend settings chain). +`issue` auto-generates `license_id` (`lic_` + 16 hex chars) and validates date formats; `--seats 0` means unlimited; `--features` is a comma-separated bit list, `"*"` for everything. The envelope format and verification logic have a single source of truth in `edition_ee/licensing/verify.py`, which the tool reuses directly (passing the public key explicitly to avoid pulling in the backend settings chain). ## Private-delivery checklist @@ -172,14 +172,14 @@ python scripts/license_tool.py verify customer.lic --pub ~/jx-license-keys/licen | Topic | Path | |---|---| -| State machine / facade | `src/backend/core/licensing/manager.py` | -| Ed25519 verification (EE-only) | `src/backend/core/licensing/_ee_verify.py` | -| Feature enum + 402 exceptions | `src/backend/core/licensing/features.py` | -| `requires_feature` guard | `src/backend/core/licensing/deps.py` | -| Seat counting | `src/backend/core/licensing/seats.py` | +| EE state machine | `src/backend/edition_ee/licensing/manager.py` | +| Ed25519 verification (EE-only) | `src/backend/edition_ee/licensing/verify.py` | +| Feature enum + 402 exceptions | `src/backend/edition_ee/licensing/features.py` | +| `requires_feature` guard | `src/backend/edition_ee/licensing/deps.py` | +| Seat counting | `src/backend/edition_ee/licensing/seats.py` | | EE route ↔ feature registry | `src/backend/api/routes/v1/__init__.py` | | Guard attachment | `src/backend/api/app.py` (edition registration loops) | -| Status query / hot-swap | `src/backend/api/routes/v1/config_license.py` | +| Status query / hot-swap | `src/backend/edition_ee/routes/config_license.py` | | Probe | `src/backend/api/routes/v1/meta.py` | | Issuance tool | `scripts/license_tool.py` | | License panel | `src/frontend/src/components/config/LicensePanel.tsx` | diff --git a/document/en/editions/overview.md b/document/en/editions/overview.md index 0d03b930..89270677 100644 --- a/document/en/editions/overview.md +++ b/document/en/editions/overview.md @@ -1,5 +1,5 @@ # Community vs. Enterprise Edition -> Last updated: 2026-07-02 +> Last updated: 2026-07-22 HugAgentOS is distributed under an **open-core** model, following the established practice of open-source agent platforms such as Dify and FastGPT: @@ -59,7 +59,7 @@ Two environment variables define the deployment shape (`EditionSettings` / `Lice | Shape | Configuration | Behavior | |---|---|---| -| Community | `JX_EDITION=ce` (default in the CE tree's `.env.example`) | All EE feature bits are constantly `False`; unlimited seats; `core/licensing/manager.py` is replaced in the CE tree by a verification-free stub via overlay | +| Community | `JX_EDITION=ce` (default in the CE tree's `.env.example`) | The CE tree physically contains no license implementation; its edition probe reports the CE shape and an empty EE capability set, with no signature verification or seat limit | | Enterprise · internal | `JX_EDITION=ee`, no license file configured, and `JX_LICENSE_REQUIRED=false` (main-repo default) | **Internal / fully-managed mode: everything enabled** — identical to historical behavior, so existing deployments need zero config changes after upgrading to a license-aware release | | Enterprise · licensed | `JX_EDITION=ee` + a valid license file | Gated by the license entitlement (feature list + seats + validity window) | @@ -86,7 +86,7 @@ COMPOSE_PROFILES=mem0 docker compose up -d ## Upgrade paths -- **CE → EE**: the Enterprise Edition is delivered as an **image bundle + license file** (suitable for air-gapped government networks). At the database level, the CE table set is a strict subset of EE's, and cross-boundary foreign-key columns (`team_id`, etc.) are kept in CE as always-NULL (design decision D3), so data carries over. +- **CE → EE**: the Enterprise Edition is delivered as an **image bundle + license file** (suitable for air-gapped government networks). The CE table set is a strict subset of EE's: 20 EE tables, their foreign keys, and commercial-scope columns on shared resources are not registered; delivery migration adds organization structures during an upgrade. > ⚠️ Caveat: CE runs an independent migration chain (baseline `ce_0001`, see [CE Build Pipeline](build-ce.md#ce-database-differences)) which differs from the EE chain; reconciling the alembic version of an existing CE database when switching to the EE image is handled during delivery — there is no automated conversion tool yet (planned). - **EE internal → EE licensed**: for private delivery, set `LICENSE_KEY_PATH` and `JX_LICENSE_REQUIRED=true`, then upload the `.lic` file in the License panel of the `/config` console — activation is immediate, no restart needed. - **Renewal / expansion**: upload a new license file to hot-swap (same flow); after expiry there is a grace window (14 days by default). @@ -96,8 +96,8 @@ COMPOSE_PROFILES=mem0 docker compose up -d | Topic | Path | |---|---| | Edition / license settings | `src/backend/core/config/settings.py` (`EditionSettings` / `LicenseSettings`) | -| License facade & state machine | `src/backend/core/licensing/manager.py` | -| EE feature enum | `src/backend/core/licensing/features.py` | +| License state machine | `src/backend/edition_ee/licensing/manager.py` | +| EE feature enum | `src/backend/edition_ee/licensing/features.py` | | Router registry (CE/EE tables) | `src/backend/api/routes/v1/__init__.py` | | Edition probe | `src/backend/api/routes/v1/meta.py` | | Frontend edition gating | `src/frontend/src/stores/editionStore.ts` | diff --git a/document/en/modules/admin-console.md b/document/en/modules/admin-console.md index 82a58e28..40902559 100644 --- a/document/en/modules/admin-console.md +++ b/document/en/modules/admin-console.md @@ -11,7 +11,7 @@ HugAgentOS ships **two independent management consoles**, aimed at content opera Entry routing happens in `src/frontend/src/main.tsx`: based on the `window.location.pathname` prefix it renders `AdminApp` (`/admin`), `ConfigApp` (`/config`), `ApiDocApp` (`/api-docs`), or the main app. The two consoles cross-link (the `/admin` header's "系统配置" button → `/config`; the `/config` header's "内容管理" button → `/admin`). For how the two tokens authenticate, see [Authentication & Permissions](auth.md). -The CE/EE assignment of backend admin routes has a single source of truth — the route registry `src/backend/api/routes/v1/__init__.py`: the CE-derived tree **physically omits** EE route files (first line of defense), and EE deployments are additionally guarded by license features (`content_admin` / `billing` / `audit` / `multi_tenancy` / `system_config`) via `core/licensing/deps.py::requires_feature` (second line of defense). Each group below is labeled accordingly. +The CE/EE assignment of backend admin routes has one composed source of truth — `src/backend/api/routes/v1/__init__.py` plus `src/backend/edition_ee/routes/registry.py`: the CE-derived tree **physically omits** EE route files (first line of defense), and EE deployments are additionally guarded by license features (`content_admin` / `billing` / `audit` / `multi_tenancy` / `system_config`) via `edition_ee/licensing/deps.py::requires_feature` (second line of defense). Each group below is labeled accordingly. ## /admin operations console @@ -148,13 +148,14 @@ Aligned with chapter 4 of the productization plan and the route registry: | System console | `src/frontend/src/ConfigApp.tsx`, `src/frontend/src/components/config/` | | Route registry (CE/EE single source of truth) | `src/backend/api/routes/v1/__init__.py` | | Administrative credential dependencies | `src/backend/api/deps.py` | -| License features | `src/backend/core/licensing/features.py`, `src/backend/core/licensing/deps.py` | +| License features | `src/backend/edition_ee/licensing/features.py`, `src/backend/edition_ee/licensing/deps.py` | | Skills / drafts / marketplace | `src/backend/api/routes/v1/admin_skills.py`, `admin_skill_drafts.py`, `admin_marketplace.py` | | Prompts / MCP / sub-agents | `src/backend/api/routes/v1/admin_prompts.py`, `admin_mcp_servers.py`, `admin_agents.py` | -| Knowledge base / sandbox | `src/backend/api/routes/v1/admin_kb.py`, `admin_sandbox.py` | +| Knowledge base / sandbox | `src/backend/edition_ee/routes/admin_kb.py`, `src/backend/api/routes/v1/admin_sandbox.py` | | Billing / usage / logs / chat review | `src/backend/api/routes/v1/admin_billing.py`, `admin_usage_logs.py`, `admin_logs.py`, `admin_chat_history.py` | | Content management | `src/backend/api/routes/v1/content.py` | -| Users / teams / invites / security / license | `src/backend/api/routes/v1/config_users.py`, `config_teams.py`, `config_invites.py`, `config_security.py`, `config_license.py` | +| Users / teams / invites / license (EE) | `src/backend/edition_ee/routes/config_users.py`, `config_teams.py`, `config_invites.py`, `config_license.py` | +| Security | `src/backend/api/routes/v1/config_security.py` | | Service configs | `src/backend/api/routes/v1/service_configs.py` | Further reading: [Authentication & Permissions](auth.md) · [Prompt System](prompts.md) · [Editions & Licensing](../editions/overview.md) diff --git a/document/en/modules/auth.md b/document/en/modules/auth.md index 40662d13..f607eaae 100644 --- a/document/en/modules/auth.md +++ b/document/en/modules/auth.md @@ -111,8 +111,8 @@ User-facing endpoints: | `POST/PUT/DELETE /v1/me/avatar` | `api/routes/v1/users.py` | Avatar upload (≤2 MB) / set / clear | | `GET/PUT /v1/users/{id}/preferences` | `api/routes/v1/users.py` | User preferences | | `POST /v1/me/onboarding/complete` | `api/routes/v1/users.py` | Validate the primary model and complete CE first-run setup | -| `GET /v1/me/teams` etc. | `api/routes/v1/me.py` | User-side team viewing, member invitation, removal / leaving (Enterprise Edition; degrades to 404 in CE trees lacking the team module) | -| `GET /v1/me/users/search` | `api/routes/v1/me.py` | User search for invitations | +| `GET /v1/me/teams` etc. | `edition_ee/routes/me_teams.py` | User-side team viewing, member invitation, removal / leaving (Enterprise Edition; the module is physically absent from CE) | +| `GET /v1/me/users/search` | `edition_ee/routes/me_teams.py` | User search for invitations | ## Permission system @@ -122,15 +122,15 @@ User-facing endpoints: | Implementation file | Responsibility | |---|---| -| `core/auth/team_permissions.py` | Team folder permission resolution (Enterprise Edition) | -| `core/auth/project_permissions.py` | Project access (team projects are Enterprise Edition) | -| `core/auth/chat_share_permissions.py` | Chat access / deletion / share-scope permissions | +| `edition_ee/auth/team_permissions.py` | Team folder permission resolution (Enterprise Edition) | +| `edition_ee/auth/project_permissions.py` | Project access for team projects (Enterprise Edition) | +| `edition_ee/auth/chat_share_permissions.py` | Team-chat access / deletion / share-scope permissions (Enterprise Edition) | `resolve_artifact_access(db, user_id, owner_id, team_id)` is the unified owner ∪ team access-level resolver: owner is always `admin` → team members follow team permission → everyone else gets `none`. File download (`api/routes/files.py`), knowledge base, My Space, and all other artifact access points share it. ### Team roles and file permissions (Enterprise Edition) -`core/auth/roles.py` defines three team roles: `owner` > `admin` > `member`. The team file permission mapping (`team_permissions.py`): +`edition_ee/auth/roles.py` defines three team roles: `owner` > `admin` > `member`. The team file permission mapping lives in `edition_ee/auth/team_permissions.py`: | Team role | File permission | Allowed actions | |---|---|---| @@ -143,7 +143,7 @@ Routes consume them through the dependency factories in `api/deps.py`: `require_ ### Per-user permission flags -User-granular feature switches are stored in the `users_shadow.metadata` JSON column (ORM attribute `extra_data`) and set from the user management module of the Config console (`api/routes/v1/config_users.py`). **All default to off** (turning a flag off removes the key from metadata): +User-granular feature switches are stored in the `users_shadow.metadata` JSON column (ORM attribute `extra_data`) and set by the EE user-management module in the Config console (`edition_ee/routes/config_users.py`). **All default to off** (turning a flag off removes the key from metadata): | Flag | Default | Control endpoint | Gates | |---|---|---|---| @@ -188,9 +188,9 @@ Keys look like `sk-jx-...` and can be used as a Bearer token to call business AP Teams and registration codes are multi-tenant capabilities (license feature `multi_tenancy`), administered from the Config system console: -- **Team management** (`api/routes/v1/config_teams.py`): team CRUD, member add/remove, role assignment (owner/admin/member). -- **Invite code management** (`api/routes/v1/config_invites.py`): batch generation, listing, revocation, deletion. Codes look like `JX-ABCD-2345` (`core/auth/invite.py`, with an alphabet that drops confusable characters like O/0 and I/1); default validity is `INVITE_CODE_DEFAULT_TTL_HOURS` (168 hours). Consumption uses a conditional UPDATE for concurrency safety and can pre-bind a team and role. -- **User side** (`api/routes/v1/me.py`): team owners/admins can invite and remove members directly; members can leave. +- **Team management** (`edition_ee/routes/config_teams.py`): team CRUD, member add/remove, role assignment (owner/admin/member). +- **Invite code management** (`edition_ee/routes/config_invites.py`): batch generation, listing, revocation, deletion. Codes look like `JX-ABCD-2345` (`edition_ee/auth/invite.py`, with an alphabet that drops confusable characters like O/0 and I/1); default validity is `INVITE_CODE_DEFAULT_TTL_HOURS` (168 hours). Consumption uses a conditional UPDATE for concurrency safety and can pre-bind a team and role. +- **User side** (`edition_ee/routes/me_teams.py`): team owners/admins can invite and remove members directly; members can leave. ## Auditing @@ -206,15 +206,15 @@ Key authentication events all land in the audit table (`audit_logs`): login succ | Mock SSO / local login & registration page | `src/backend/api/routes/v1/mock_sso.py`, `src/backend/core/auth/mock_ticket_store.py` | | Password hashing | `src/backend/core/auth/password.py` | | Permission interface layer (CE/EE seam) | `src/backend/core/auth/permissions_iface.py` | -| Team roles / file permissions | `src/backend/core/auth/roles.py`, `src/backend/core/auth/team_permissions.py` | -| Project / chat-share permissions | `src/backend/core/auth/project_permissions.py`, `src/backend/core/auth/chat_share_permissions.py` | +| Team roles / file permissions (EE) | `src/backend/edition_ee/auth/roles.py`, `src/backend/edition_ee/auth/team_permissions.py` | +| Project / chat-share permissions (EE) | `src/backend/edition_ee/auth/project_permissions.py`, `src/backend/edition_ee/auth/chat_share_permissions.py` | | Administrative credential dependencies | `src/backend/api/deps.py` | -| Profile / preferences | `src/backend/api/routes/v1/users.py`, `src/backend/api/routes/v1/me.py` | -| Per-user permission flags | `src/backend/api/routes/v1/config_users.py` | +| Profile / preferences | `src/backend/api/routes/v1/users.py` | +| Per-user permission flags | `src/backend/edition_ee/routes/config_users.py` | | Personal API keys | `src/backend/api/routes/v1/api_keys.py`, `src/backend/core/services/api_key_service.py` | | Capability-center self-service (owner isolation) | `src/backend/api/routes/v1/me_capabilities.py` | -| Invite codes | `src/backend/core/auth/invite.py`, `src/backend/api/routes/v1/config_invites.py` | -| Team management | `src/backend/api/routes/v1/config_teams.py` | -| License feature guards | `src/backend/core/licensing/features.py`, `src/backend/core/licensing/deps.py` | +| Invite codes | `src/backend/edition_ee/auth/invite.py`, `src/backend/edition_ee/routes/config_invites.py` | +| Team management | `src/backend/edition_ee/routes/config_teams.py`, `src/backend/edition_ee/routes/me_teams.py` | +| License feature guards | `src/backend/edition_ee/licensing/features.py`, `src/backend/edition_ee/licensing/deps.py` | Further reading: [Admin Consoles](admin-console.md) · [Editions & Licensing](../editions/overview.md) · [Environment Variables](../deployment/environment-variables.md) diff --git a/document/en/modules/canvas-artifacts.md b/document/en/modules/canvas-artifacts.md index 25cd2348..26a9a8b0 100644 --- a/document/en/modules/canvas-artifacts.md +++ b/document/en/modules/canvas-artifacts.md @@ -30,7 +30,7 @@ Panel state is managed by `stores/canvasStore.ts` (`openCanvas` / `closeCanvas` 2. At runtime it dynamically `import('@univerjs/presets')` plus `@univerjs/preset-sheets-core` (with the zh-CN locale) to render the spreadsheet — **only the free core preset is actually loaded**. 3. After editing, `exportXlsx()` produces a new xlsx File which `CanvasPanel` writes back to the same `file_id` via `api.ts::overwriteFile`; a dirty flag drives the Save button. -> Real-time collaborative editing is an Enterprise Edition capability (`Feature.CANVAS_COLLAB`, `core/licensing/features.py`). Note that `src/frontend/package.json` still declares the `@univerjs/preset-sheets-advanced` dependency (Univer's commercially licensed preset) — runtime code never imports it, and per the open-sourcing plan the CE-derived tree must not ship it. +> Real-time collaborative editing is an Enterprise Edition capability (`Feature.CANVAS_COLLAB`, `edition_ee/licensing/features.py`). Note that `src/frontend/package.json` still declares the `@univerjs/preset-sheets-advanced` dependency (Univer's commercially licensed preset) — runtime code never imports it, and per the open-sourcing plan the CE-derived tree must not ship it. ## Artifact center (My Space) diff --git a/document/en/modules/catalog.md b/document/en/modules/catalog.md index 77047d39..b1edf788 100644 --- a/document/en/modules/catalog.md +++ b/document/en/modules/catalog.md @@ -72,7 +72,7 @@ On every chat, `core/chat/context.py::resolve_enabled_capabilities()` writes the | Source | Condition | Marking | |---|---|---| -| Dify external KB | `KNOWLEDGE_BASE=dify` with valid credentials (`core/kb/dify_kb.py::is_dify_enabled`); dataset list cached 60 s in-process | `visibility: public` (**Enterprise Edition (EE)**: external Dify KB integration) | +| Dify external KB | `KNOWLEDGE_BASE=dify` with valid credentials (`edition_ee/kb/dify.py::is_dify_enabled`); dataset list cached in-process | `visibility: public` (**Enterprise Edition (EE)**: the adapter is absent from CE) | | Public self-hosted KB | created in the admin "KB management" console (local Milvus); visible to all users, read-only on the frontend | `visibility: public` | | Private user KB | the current user's local KB spaces | `visibility: private` | @@ -123,6 +123,6 @@ State is centralized in `src/frontend/src/stores/catalogStore.ts`; local default | User self-service capabilities | `src/backend/api/routes/v1/me_capabilities.py` | | MCP server config (DB) | `src/backend/core/services/mcp_service.py`, `api/routes/v1/admin_mcp_servers.py` | | Skill management | `src/backend/api/routes/v1/admin_skills.py`, `core/agent_skills/` | -| Dify KB injection | `src/backend/core/kb/dify_kb.py` | +| Dify KB injection (EE) | `src/backend/edition_ee/kb/dify.py`, via the shared `core/kb/external_provider.py` seam | | Frontend Capability Center | `src/frontend/src/components/catalog/`, `stores/catalogStore.ts` | | Factory consumption | `src/backend/core/llm/agent_factory.py::_effective_mcp_server_keys` | diff --git a/document/en/modules/knowledge-base.md b/document/en/modules/knowledge-base.md index e6f20922..fc4c4612 100644 --- a/document/en/modules/knowledge-base.md +++ b/document/en/modules/knowledge-base.md @@ -106,7 +106,7 @@ The `/v1/admin/kb/*` admin routes live in `src/backend/api/routes/v1/admin_kb.py ## External Dify knowledge bases (Enterprise Edition, EE) -The client wrapper is `src/backend/core/kb/dify_kb.py`. The `is_dify_enabled()` decision has three priority levels: +The EE-only client is `src/backend/edition_ee/kb/dify.py`; shared routes call it through `core/kb/external_provider.py`. The derived CE tree replaces that seam with a disabled implementation and contains no Dify client. The `is_dify_enabled()` decision has three priority levels: 1. DB system config `knowledge_base.provider == "dify"` (editable in the Config console); 2. Environment variable `KNOWLEDGE_BASE=dify`; @@ -138,7 +138,8 @@ The general-purpose parser `core/content/file_parser.py::parse_file()` (shared b |---|---| | `src/backend/core/kb/kb_parser.py` | Document parsing + parent-child chunking (5 chunk methods) | | `src/backend/core/kb/kb_vector.py` | Milvus collection, embedding, hybrid search, reranking | -| `src/backend/core/kb/dify_kb.py` | Dify datasets client and enablement logic | +| `src/backend/edition_ee/kb/dify.py` | Dify datasets client and enablement logic (EE only) | +| `src/backend/core/kb/external_provider.py` | Edition-neutral external-provider seam; disabled by the CE overlay | | `src/backend/core/content/kb_processing.py` | Background vectorization, LLM keyword / question enrichment | | `src/backend/core/content/file_validation.py` | Upload validation (extension + magic bytes) | | `src/backend/core/content/file_parser.py` | General-purpose file parser | diff --git a/document/en/modules/memory.md b/document/en/modules/memory.md index c1956b94..53878a6e 100644 --- a/document/en/modules/memory.md +++ b/document/en/modules/memory.md @@ -84,7 +84,7 @@ Rules are runtime-extensible: the DB table `memory_sanitizer_rules` (ORM: `core/ - Failures never propagate (auditing never blocks the hot path); - Toggle: `MEMORY_AUDIT_ENABLED` (default `true`). -Per the [edition comparison](../editions/overview.md), memory auditing is a commercial feature flag (`core/licensing/features.py::Feature.MEMORY_AUDIT`). The audit query endpoint is `GET /v1/memories/audit` (filterable by action / layer). +Per the [edition comparison](../editions/overview.md), memory auditing is a commercial feature flag (`edition_ee/licensing/features.py::Feature.MEMORY_AUDIT`). The audit query endpoint is `GET /v1/memories/audit` (filterable by action / layer). ## Memory management API @@ -196,7 +196,8 @@ See the [environment variable reference](../deployment/environment-variables.md) | `src/backend/orchestration/memory_integration.py` | Retrieval launch, frozen-block assembly and injection, save delegation | | `src/backend/orchestration/workflow.py` | Main orchestration: memory hook wiring | | `src/backend/api/routes/v1/memories.py` | `/v1/memories` management API | -| `src/backend/core/db/models/memory.py` | `MemoryAudit` / `MemorySanitizerRule` ORM | +| `src/backend/core/db/models/memory.py` | Shared `MemorySanitizerRule` ORM | +| `src/backend/edition_ee/db/models/memory.py` | `MemoryAudit` ORM (EE only) | | `src/frontend/src/components/settings/SettingsModal.tsx` | Memory settings + layered memory modal | | `src/frontend/src/components/memory/FactsList.tsx` | L2 fact list component | | `docker-compose.yml` (`mem0` profile) | Milvus / etcd / MinIO / Neo4j | diff --git a/document/en/modules/projects-myspace.md b/document/en/modules/projects-myspace.md index 14940485..5e157bac 100644 --- a/document/en/modules/projects-myspace.md +++ b/document/en/modules/projects-myspace.md @@ -97,7 +97,7 @@ Project frontend lives in `src/frontend/src/components/projects/`: `ProjectsPane ## Team folders and team files (Enterprise Edition, EE) -User-facing routes are in `src/backend/api/routes/v1/team_files.py`, gated by the `multi_tenancy` feature flag (EE router table); the admin counterpart is `/v1/config/teams/*` (`config_teams.py`). +User-facing routes are in `src/backend/edition_ee/routes/team_files.py`, gated by the `multi_tenancy` feature flag (EE router table); the admin counterpart is `/v1/config/teams/*` (`edition_ee/routes/config_teams.py`). | Method | Path | Description | |---|---|---| @@ -109,7 +109,7 @@ User-facing routes are in `src/backend/api/routes/v1/team_files.py`, gated by th | POST | `/v1/artifacts/{artifact_id}/move-to-team` | Convert a personal file into a team file | | GET / PUT | `/v1/teams/{team_id}/members/permissions`, `.../{user_id}/permission` | View / adjust member file permissions | -Permission model: `TeamMember.role` (owner/admin/member) + `file_permission` (viewer/editor, effective only for members), encapsulated in `core/auth/team_permissions.py`. Team files have a dedicated shared sandbox cache, `team_cache_dir(team_id)`, reused across members of the same team. +Permission model: `TeamMember.role` (owner/admin/member) + `file_permission` (viewer/editor, effective only for members), encapsulated in the EE-only `edition_ee/auth/team_permissions.py`. Team files have a dedicated shared sandbox cache, `team_cache_dir(team_id)`, reused across members of the same team. ## How files enter conversation context @@ -128,10 +128,11 @@ Three complementary paths: | `src/backend/core/services/project_scope.py` | `ProjectScope` (sandbox path scoping) | | `src/backend/api/routes/v1/myspace_folders.py` | Personal folders API | | `src/backend/api/routes/v1/artifacts.py` | Asset list / chat favorites / add-to-KB | -| `src/backend/api/routes/v1/team_files.py` | Team folders & files API (Enterprise Edition, EE) | +| `src/backend/edition_ee/routes/team_files.py` | Team folders & files API (Enterprise Edition, EE) | | `src/backend/api/routes/v1/file_upload.py` | File upload (folder targeting) | | `src/backend/core/db/models/project.py` | `Project` / `ProjectFavorite` ORM | -| `src/backend/core/db/models/identity.py` | `Team` / `TeamMember` / `TeamFolder` / `UserFolder` ORM | +| `src/backend/core/db/models/identity.py` | Shared identity ORM such as `UserFolder` | +| `src/backend/edition_ee/db/models/identity.py` | `Team` / `TeamMember` / `TeamFolder` ORM (EE only) | | `src/backend/core/db/models/artifact.py` | `Artifact` ORM | | `src/backend/core/llm/hooks.py` | Attachment context injection (`_build_file_context`, etc.) | | `src/backend/core/llm/agent_factory.py` | Project section injection into the system prompt | diff --git a/document/zh-CN/api/error-codes.md b/document/zh-CN/api/error-codes.md index 1f128a60..a7166f33 100644 --- a/document/zh-CN/api/error-codes.md +++ b/document/zh-CN/api/error-codes.md @@ -1,8 +1,8 @@ # 错误码参考 -> 最后更新：2026-06-11 +> 最后更新：2026-07-22 -本文以代码实际实装为准：业务异常类定义在 `src/backend/core/infra/exceptions.py`（license 相关两个在 `src/backend/core/licensing/features.py`），由全局异常处理器 `src/backend/api/middleware/error_handler.py` 统一转换为[统一响应信封](overview.md#统一响应信封)。本文只列**已实装**的错误码；各分类号段内未列出的码位为预留空间。 +本文以代码实际实装为准：业务异常类定义在 `src/backend/core/infra/exceptions.py`（商业版 License 相关两个在 `src/backend/edition_ee/licensing/features.py`），由全局异常处理器 `src/backend/api/middleware/error_handler.py` 统一转换为[统一响应信封](overview.md#统一响应信封)。本文只列**已实装**的错误码；各分类号段内未列出的码位为预留空间。 ## 错误响应结构 @@ -108,7 +108,7 @@ ## License 未授权（HTTP 402） -EE 路由按 `api/routes/v1/__init__.py` 注册表挂载 license 能力位守卫（`core/licensing/deps.py` → `requires_feature`）。未授权时抛 `FeatureNotLicensed`，由 `error_handler` 兑现为： +EE 路由按 `edition_ee/routes/registry.py` 注册表挂载 License 能力位守卫（`edition_ee/licensing/deps.py` → `requires_feature`）。未授权时抛 `FeatureNotLicensed`，由 `error_handler` 兑现为： ```json { @@ -120,7 +120,7 @@ EE 路由按 `api/routes/v1/__init__.py` 注册表挂载 license 能力位守卫 } ``` -设计要点（`core/licensing/features.py`）： +设计要点（`edition_ee/licensing/features.py`）： - `FeatureNotLicensed` 是 402 信封的**唯一来源**，路由/服务层不允许再手搓 `HTTPException(402)`。 - 选 402 而非 403：403 会被前端当作会话失效触发强制登出，而 license 缺失不应把用户登出。 diff --git a/document/zh-CN/api/overview.md b/document/zh-CN/api/overview.md index ab37eec8..90ae5cc2 100644 --- a/document/zh-CN/api/overview.md +++ b/document/zh-CN/api/overview.md @@ -228,11 +228,11 @@ curl -X POST http://localhost:3000/api/v1/chat-runs/run_9f8e7d/cancel \ | 观测与审计 | `admin_logs.py` | `/v1/admin/logs` | `GET /tools`、`GET /subagents`、`GET /trace/{trace_id}` | CONFIG | `audit` | | 登录与会话 | `auth.py` | `/v1/auth` | `POST /ticket/exchange`（SSO 票据换会话）、`GET /session/check`、`POST /logout` | 公开（会话基础设施） | — | | 配置台 | `config_verify.py` | `/v1/config` | `GET /verify`（CONFIG_TOKEN 校验） | CONFIG | — | -| 配置台 | `config_license.py` | `/v1/config/license` | `GET /`（license 详情）、`POST /`（更换 license） | CONFIG | — | -| 多租户 | `config_users.py` | `/v1/config/users` | `GET /`、`PATCH /{user_id}/status`、`POST /{user_id}/reset-password` | CONFIG | `multi_tenancy` | -| 多租户 | `config_teams.py` | `/v1/config/teams` | `GET/POST /`、`POST /{team_id}/members` | CONFIG | `multi_tenancy` | -| 多租户 | `config_invites.py` | `/v1/config/invite-codes` | `GET/POST /`、`POST /{code}/revoke` | CONFIG | `multi_tenancy` | -| 多租户 | `team_files.py` | `/v1/my-teams`、`/v1/teams`、`/v1/artifacts` | `GET /my-teams`、`POST /teams/{id}/files/upload`、`POST /artifacts/{id}/move-to-team` | 用户 + 团队文件权限 | `multi_tenancy` | +| 配置台 | `edition_ee/routes/config_license.py` | `/v1/config/license` | `GET /`（license 详情）、`POST /`（更换 license） | CONFIG | — | +| 多租户 | `edition_ee/routes/config_users.py` | `/v1/config/users` | `GET /`、`PATCH /{user_id}/status`、`POST /{user_id}/reset-password` | CONFIG | `multi_tenancy` | +| 多租户 | `edition_ee/routes/config_teams.py` | `/v1/config/teams` | `GET/POST /`、`POST /{team_id}/members` | CONFIG | `multi_tenancy` | +| 多租户 | `edition_ee/routes/config_invites.py` | `/v1/config/invite-codes` | `GET/POST /`、`POST /{code}/revoke` | CONFIG | `multi_tenancy` | +| 多租户 | `edition_ee/routes/team_files.py` | `/v1/my-teams`、`/v1/teams`、`/v1/artifacts` | `GET /my-teams`、`POST /teams/{id}/files/upload`、`POST /artifacts/{id}/move-to-team` | 用户 + 团队文件权限 | `multi_tenancy` | | 系统配置 | `config_security.py` | `/v1/config/security` | `GET /sandbox/overview`、`GET /audit-logs`、`GET /system-health` | CONFIG | `system_config` | | 系统配置 | `service_configs.py` | `/v1/service-configs` | `GET/PUT /`、`POST /test/{group_key}`（外部服务连通性测试） | CONFIG | `system_config` | diff --git a/document/zh-CN/architecture/backend.md b/document/zh-CN/architecture/backend.md index 016b36fc..1cc1a310 100644 --- a/document/zh-CN/architecture/backend.md +++ b/document/zh-CN/architecture/backend.md @@ -1,6 +1,6 @@ # 后端架构详解 -> 最后更新：2026-07-19 +> 最后更新：2026-07-22 后端位于 `src/backend/`，是一个分层清晰的 FastAPI 单体：API 层只做协议与鉴权，编排层负责把一次对话变成可断线续播的流式 Run，`core/` 承载全部领域逻辑，MCP 工具与脚本执行 sidecar 则以独立进程运行。本文自顶向下逐层拆解。 @@ -8,7 +8,8 @@ ``` src/backend/ -├── api/ # FastAPI 应用、中间件、74 个 v1 路由文件 +├── api/ # FastAPI 应用、中间件与 CE 路由 +├── edition_ee/ # 商业版路由、License、Team/RBAC、ORM、Dify 与服务实现 ├── orchestration/ # 对话编排：Run 执行器、工作流、策略、引用、调度器 ├── core/ # 领域核心：17 个子模块（auth/llm/db/ontology/services/...） ├── mcp_servers/ # 10 个独立 MCP 服务器（streamable-http 进程） @@ -133,10 +134,10 @@ src/backend/ |---|---| | `core/chat` | workflow 上下文组装（`context.py`）、SSE 工具日志事件构造（`tool_log.py`） | | `core/content` | 附件解析（`file_parser.py`）、KB 文档分块/关键词/向量化（`kb_processing.py`）、上传校验（`file_validation.py`）、产物读取与摘要（`artifact_reader/refs/summary.py`）、内容块导入导出（`content_blocks.py`）、`svg_fit.py` | -| `core/kb` | 自建知识库解析与父子分块（`kb_parser.py`）、Milvus 向量库（`kb_vector.py`）、Dify 外部知识库客户端（`dify_kb.py`，对接外部 KB 为商业版 EE 增项） | +| `core/kb` / `edition_ee/kb` | 自建知识库解析与 Milvus 向量库保留在 `core/kb`；Dify 客户端与外部检索适配器只存在于 `edition_ee/kb`（商业版 EE） | | `core/artifacts` | 产物存储 `store.py`：本地 / OSS 双模式 | | `core/infra` | 统一响应（`responses.py`）、异常（`exceptions.py`）、结构化日志（`logging.py`）、限流（`rate_limit.py`）、Redis 单例（`redis.py`）、指标（`metrics.py`）、后台任务注册表（`runtime_state.py`）、脱敏（`data_masking.py`）、蒸馏预算闸门（`distillation_budget.py`，商业版 EE） | -| `core/licensing` | license 门面 `manager.py`（GitLab 式离线模型：签名文件 + 进程内验签）、能力位枚举 `features.py`、FastAPI 守卫依赖 `deps.py`、席位计数 `seats.py`；验签实现 `_ee_verify.py`（商业版 EE，CE 树用恒 False stub 替换） | +| `edition_ee/licensing` | 能力枚举、验签、时钟回拨防护、席位策略、中间件和 manager 全部只存在于商业版 EE；CE 树物理不含 License 包，由 `api/middleware/edition.py` 的 CE overlay 提供固定版本探针 | | `core/storage` | 存储协议 `protocol.py` + 工厂 `factory.py`；`local.py`（CE）、`s3.py` / `oss.py`（商业版 EE） | ## orchestration/ — 编排层 @@ -167,9 +168,11 @@ src/backend/ ### 路由注册表（CE/EE 接缝 C1） -`api/routes/v1/__init__.py` 是两版共用的注册表：`CE_ROUTERS`（39 个）无条件注册；`EE_ROUTERS`（32 个）每项携带 license 能力位，由 `core/licensing/deps.py` 做第二道防线（第一道是 CE 派生树物理删除这些文件）；`config_verify` / `config_license` / `auth` 三项显式豁免，保证 license 失效时仍能换证。 +全量仓库由 `api/routes/v1/__init__.py` 的 `CE_ROUTERS` 与 `edition_ee/routes/registry.py` 的 `EE_ROUTERS` 组合注册（当前各 36 项）。EE 表项携带 License 能力位，由 `edition_ee/licensing/deps.py` 做第二道防线（第一道是 CE 派生树物理删除 EE 文件）；CE overlay 把 `EE_ROUTERS` 固定为空。`config_verify` / `config_license` / `auth` 三项显式豁免，保证 License 失效时仍能登录和换证。 -### 路由文件分组（v1 共 74 个文件） +### 路由文件分组 + +CE 路由位于 `api/routes/v1/`；已物理拆分的商业路由位于 `edition_ee/routes/`，其余历史管理路由由 EE 注册表按能力位挂载。 | 分组 | 文件 | |---|---| @@ -197,7 +200,7 @@ src/backend/ 2. **orchestration 只做编排**：把领域服务串成流式工作流，不直接操作 ORM； 3. **core/services 是唯一业务入口**：路由不得绕过服务层直查 `core/db/models`（少量只读快路径除外）； 4. **进程边界即故障边界**：MCP、script-runner、沙箱均独立进程 / 容器，与后端只过协议层； -5. **CE/EE 接缝集中**：路由注册表、`edition_tables`、`permissions_iface`、licensing 门面四处收口，业务代码不散落 `if edition` 判断。 +5. **CE/EE 接缝集中**：路由注册表、`edition_tables`、`permissions_iface` 与版本中间件集中收口，商业实现进入 `edition_ee`，业务代码不散落 `if edition` 判断。 ## 相关源码 @@ -210,6 +213,6 @@ src/backend/ | 能力目录 | `src/backend/core/config/catalog.py` | | 沙箱协议 | `src/backend/core/sandbox/protocol.py` | | 记忆流水线 | `src/backend/core/memory/pipeline.py` | -| license 门面 | `src/backend/core/licensing/manager.py` | +| EE License 实现 / CE 版本中间件 | `src/backend/edition_ee/licensing/`、`src/backend/api/middleware/edition.py` | | 技能引擎 | `src/backend/core/agent_skills/loader.py` | | MCP 端口表 | `src/backend/mcp_servers/_ports.py` | diff --git a/document/zh-CN/architecture/data-model.md b/document/zh-CN/architecture/data-model.md index 7cf6b32b..018cb00f 100644 --- a/document/zh-CN/architecture/data-model.md +++ b/document/zh-CN/architecture/data-model.md @@ -141,24 +141,26 @@ core/db/ - **商业版主链**：`src/backend/alembic/versions/` 下 53 个迁移，从初始建表一路演进（含 MCP 迁往 streamable-http、办公 MCP 下线改技能等结构性变更）。常用命令：`alembic upgrade head`、`make migrate-new msg="..."`（autogenerate 基于 `core/db/models` 元数据）； - **启动兜底**：`api/app.py` lifespan 的 `_startup_ensure_tables` 调 `core/db/engine.py::init_db`，对 SQLite 开发库幂等补建缺表； -- **社区版独立链**：CE 派生树整体排除主链迁移，overlay 提供单一基线 `ce/overlay/src/backend/alembic/versions/ce_0001_initial.py`——以 SQLAlchemy 元数据为源、按 `EE_ONLY_TABLES` 过滤后 `create_all`，方言感知（SQLite / PostgreSQL 通吃）；后续 CE schema 演进在该链上追加常规迁移。 +- **社区版独立链**：CE 派生树整体排除主链迁移，overlay 提供单一基线 `ce/overlay/src/backend/alembic/versions/ce_0001_initial.py`——直接以 CE-only SQLAlchemy 元数据 `create_all`，方言感知（SQLite / PostgreSQL 通吃）；后续 CE schema 演进在该链上追加常规迁移。 ## CE/EE 建表边界（core/db/edition_tables.py） -`core.db.models` 包在两版共用（EE 表类定义本身无害），但 CE 不应建出 EE 专属空表。`EE_ONLY_TABLES` 是这一边界的单一真源，共 18 张表： +EE ORM 类定义集中在 `edition_ee/db/models/`，CE 派生树物理不包含该包。全量源码校验使用 `edition_ee/db/edition_tables.py::EE_ONLY_TABLES`，发布门禁同步禁止 20 张表： ``` teams · team_members · team_folders · invite_codes # 多租户 / SSO / 邀请 roles · role_assignments # 组织角色权限体系 +chat_session_user_states # 团队共享会话的成员态 kb_grants # 知识库逐用户 / 团队授权 -audit_logs · memory_audit # 审计（CE 的 memory audit 为 stub，不落表） +marketplace_visibility_grants # 市场条目指定范围可见 +audit_logs · memory_audit # 审计（CE 不含实现、不落表） model_pricing # 计费 data_sources · ds_table_meta · ds_column_meta · ds_golden_sql # 数据源 / 元数据治理 gateway_virtual_keys # 对外模型网关虚拟密钥镜像 sandbox_rebuilds · admin_skill_drafts · distillation_runs # 持久沙箱重建 / 技能蒸馏 ``` -`ce_create_all(bind)` 在 **克隆的 MetaData** 上建出全部非 EE 表：CE 表里指向 EE 表的跨边界外键（如 `projects/artifacts → teams/team_folders`，方案 D3「列保留、恒 NULL」）若原样下发，PostgreSQL 会因引用表不存在而失败——因此在克隆上摘除这些约束（列保留、原 metadata 不动、ORM 映射不受影响）。两个建表入口同源同滤：`init_db` 的 CE 分支（`JX_EDITION=ce` 时过滤）与 CE 迁移基线 `ce_0001`。维护规则：新增 EE 专属模型必须同步加进 `EE_ONLY_TABLES`，集合名与 metadata 实表名做启动断言，防止改名漏更新悄悄退化为全量建表。 +全量源码下的 `ce_create_all(bind)` 会在克隆 MetaData 上过滤上述表与跨界外键，用于本地边界校验。真正的 CE 树则根本不导入 EE ORM；CE overlay 仅克隆已注册的 CE 元数据，并防御性摘除所有指向缺失表的外键。新增 EE 模型时必须同步更新 `EE_ONLY_TABLES`、`ce/manifest.yaml` 的禁止表契约与对应 overlay。 几个「看着像 EE 实际 CE 必需」的表刻意不在集合内：`admin_prompt_parts`（提示词运行时读）、`memory_sanitizer_rules`（脱敏闸门无条件查询）、`admin_skills` / `admin_mcp_servers`（个人自助能力，owner 隔离）、`marketplace_submissions`（CE 保留提交端点）。 @@ -166,7 +168,7 @@ sandbox_rebuilds · admin_skill_drafts · distillation_runs # 持久沙箱重建 | 主题 | 路径 | |---|---| -| ORM 模型包 | `src/backend/core/db/models/` | +| 共享 ORM / EE ORM | `src/backend/core/db/models/`、`src/backend/edition_ee/db/models/` | | 本体仓储 | `src/backend/core/db/repository/ontology.py` | | 引擎与启动建表 | `src/backend/core/db/engine.py` | | 仓储层 | `src/backend/core/db/repository/` | diff --git a/document/zh-CN/architecture/overview.md b/document/zh-CN/architecture/overview.md index be403bf4..859a82f9 100644 --- a/document/zh-CN/architecture/overview.md +++ b/document/zh-CN/architecture/overview.md @@ -214,7 +214,7 @@ RAG 则提供支撑决策所需的文档与证据。 ### CE/EE 同源双形态 -商业版主仓即全量代码；社区版由 `scripts/build_ce.py` 按 `ce/manifest.yaml` 派生（排除 EE 文件 + 文本品牌中性化 + overlay 覆盖）。运行时三道接缝：路由注册表 `api/routes/v1/__init__.py`（EE 路由带 license 能力位）、建表边界 `core/db/edition_tables.py`（CE 不建 18 张 EE 专属表）、license 门面 `core/licensing/manager.py`（CE 下全部能力位恒 False）。详见 [版本与授权](../editions/overview.md)。 +商业版主仓即全量代码；社区版由 `scripts/build_ce.py` 按 `ce/manifest.yaml` 派生（排除 `edition_ee` 实现 + 文本变换 + CE overlay）。运行时边界包括路由注册表 `api/routes/v1/__init__.py`、建表边界 `core/db/edition_tables.py`（CE 不注册 20 张 EE 专属表）、EE 的 `edition_ee/licensing/` 与 CE 固定版本中间件，以及站点可见性等版本策略缝隙。详见 [版本与授权](../editions/overview.md)。 ## 相关源码 @@ -229,5 +229,5 @@ RAG 则提供支撑决策所需的文档与证据。 | 能力目录 | `src/backend/core/config/catalog.json`、`catalog.py` | | 响应信封 | `src/backend/core/infra/responses.py` | | 提示词装配 | `src/backend/prompts/prompt_runtime.py` | -| CE/EE 接缝 | `src/backend/api/routes/v1/__init__.py`、`src/backend/core/db/edition_tables.py`、`src/backend/core/licensing/` | +| CE/EE 接缝 | `src/backend/api/routes/v1/__init__.py`、`src/backend/core/db/edition_tables.py`、`src/backend/api/middleware/edition.py`、`src/backend/edition_ee/licensing/` | | 容器编排 | `docker-compose.yml` | diff --git a/document/zh-CN/deployment/docker-compose.md b/document/zh-CN/deployment/docker-compose.md index 72967ac4..293f5dce 100644 --- a/document/zh-CN/deployment/docker-compose.md +++ b/document/zh-CN/deployment/docker-compose.md @@ -1,6 +1,6 @@ # Docker Compose 部署 -> 最后更新：2026-07-19 ｜ [English](../../en/deployment/docker-compose.md) ｜ 返回 [部署指南](README.md) +> 最后更新：2026-07-23 ｜ [English](../../en/deployment/docker-compose.md) ｜ 返回 [部署指南](README.md) > **适用场景**：团队 / 生产的**标准部署形态**，多用户、全功能。个人单机尝鲜可用更轻的 [无 Docker 一键安装](quick-install.md)。 @@ -15,11 +15,16 @@ HugAgentOS 的全部服务由根目录 `docker-compose.yml` 一个文件编排 | `postgres` | hugagent-postgres | `postgres:15-alpine` | `${POSTGRES_HOST_PORT:-5432}:5432` | 主关系库（业务数据、content_blocks、用量日志） | | `redis` | hugagent-redis | `redis:7-alpine` | `${REDIS_HOST_PORT:-6380}:6379` | 会话存储、流式 follower（Redis Streams）、限流 | | `backend` | hugagent-backend | `docker/Dockerfile`（target `production`） | `${BACKEND_HOST_PORT:-3001}:${BACKEND_PORT:-3001}` | FastAPI 应用；启动时自动跑 alembic 迁移 | -| `mcp` | hugagent-mcp | `docker/Dockerfile.mcp` | 无对外端口 | 10 个 MCP server，以 streamable-http 监听 `9100–9108`、`9112`，backend 经 `http://mcp:91XX/mcp/` 调用 | +| `mcp` | hugagent-mcp | `docker/Dockerfile.mcp` | 无对外端口 | CE 启动 9 个通用 MCP；EE 另启动数据库查询等商业 MCP。backend 经 `http://mcp:91XX/mcp/` 调用 | | `frontend` | hugagent-frontend | `src/frontend/Dockerfile` | `${FRONTEND_PORT:-3002}:80` | nginx 托管前端静态资源 + `/api` 反代到 backend | `BACKEND_PORT` 是容器内的监听端口，nginx、MCP 和健康检查都依赖它，通常保持 `3001`。如果宿主机端口已被占用，只调整 `BACKEND_HOST_PORT`、`POSTGRES_HOST_PORT` 或 `REDIS_HOST_PORT`，不要改容器内端口。例如可将后端公开到 `13003`，同时保持容器内的 `3001`。 +CE 在空数据库首次启动时会全局安装并启用 `automation`、 +`skill-manager` 和 `sites` 三个插件，所有用户可直接使用，无需分别前往 +插件市场安装。CE 的能力页、运行时 catalog、MCP 端口表和 MCP 镜像均不包含 +数据库查询工具；该能力仅存在于标注为 EE 的商业版部署。 + ### 沙箱 sidecar（profiles 二选一，互斥） | 服务 | profile | 容器名 | 镜像 / 构建 | 作用 | diff --git a/document/zh-CN/deployment/quick-install.md b/document/zh-CN/deployment/quick-install.md index 683c04bd..83bdabe9 100644 --- a/document/zh-CN/deployment/quick-install.md +++ b/document/zh-CN/deployment/quick-install.md @@ -1,6 +1,6 @@ # 无 Docker 一键安装（本地单机） -> 最后更新：2026-07-21 ｜ [English](../../en/deployment/quick-install.md) ｜ 返回 [部署指南](README.md) +> 最后更新：2026-07-23 ｜ [English](../../en/deployment/quick-install.md) ｜ 返回 [部署指南](README.md) 面向**个人单机尝鲜**与**二次开发体验**的极简部署方式：一条命令装好，终端引导设管理员、配模型，随后单进程起服务并打开浏览器。全程**零 Docker、零 PostgreSQL、零 Redis**。 @@ -62,7 +62,11 @@ curl -fsSL https://raw.githubusercontent.com/ZJU-REAL/HugAgentOS/main/install.sh > HugAgentOS 共有 9 个模型角色：7 个对话角色（主智能体 / 摘要 / 追问 / 记忆 / 图表 / 计划 / 代码执行，共用上面的对话模型）+ 向量（embedding）+ 重排（reranker）。onboard 覆盖全部三类；登录后还可在网页「设置 → 系统管理 → 模型服务」为单个角色指派不同模型。 -**第 3 步 · 选择插件**——从内置插件列表勾选要安装的能力（序号逗号分隔 / `all` 全装 / `none` 跳过；直接回车装 ★ 推荐项）。默认推荐：`automation`（定时任务）、`skill-manager`（技能管理）、`sites`（对话建站）。装 `sites` 时会自动铺入 React 建站工程模板。插件随后可在插件市场随时增减。 +**第 3 步 · 插件初始化**——全新安装会自动安装并启用 +`automation`（定时任务）、`skill-manager`（技能管理）和 `sites` +（对话建站）。用户无需登录后再去插件市场安装。安装 `sites` 时还会 +铺入 React 建站工程模板。首次初始化完成后，用户仍可在插件库停用或卸载； +后续重启不会把用户主动卸载的插件重新装回。 **第 4 步（可选）· 配置文件解析服务**——上传 PDF / 扫描件解析需要一个外部解析服务（MinerU 兼容），填入其 API URL 即可（写入 `file_parser.api_url`）；直接回车跳过。Excel / CSV / PPTX / 文本为进程内解析，无需此项。 @@ -139,7 +143,9 @@ hugagent doctor # 环境自检（Python 版本、端口占用、数据目录 - **自建向量知识库**：用嵌入式 **Milvus Lite**（单文件，无需服务端），**纯向量（dense）检索**；需在 onboard 配置 embedding 模型。要更强的混合检索，把 `MILVUS_URL` 指向真正的 Milvus 服务即可自动切回。 - **L2 向量记忆**：安装器会装好 mem0 与 Milvus Lite，并默认启用记忆运行时。配置并指派可用的 embedding 模型后，用户的永久记忆与自动写入默认开启；缺少 embedding 时，前后端都会阻止打开记忆开关。 -- **自动化 / 技能创作 / 建站等插件能力**：`automation` / `skill-manager` / `sites` 是**插件**，可在 onboard 第 3 步一键勾选安装（或事后到插件市场增减）；安装后其 MCP 在本地已自动连通（`http://mcp:*` 主机名会被重写到 `127.0.0.1`）。 +- **自动化 / 技能创作 / 建站等插件能力**：`automation` / `skill-manager` / + `sites` 在全新安装中默认安装并启用，无需用户前往插件市场操作。其 MCP + 启动时会经过工具列表校验，本地地址统一使用 `127.0.0.1`。 **需额外条件** - **对话建站的 React 工程构建**：装 `sites` 插件后即支持——onboard 把 React 工程模板铺入 `~/.hugagent/site-template/`，首次建站时按需 `npm install`。**需宿主装有 Node.js ≥ 20 + npm**；缺则只能手写静态站点。建站链路的 `/workspace` 路径已参数化到本地工作区（静态站与 Docker 版一致）。 diff --git a/document/zh-CN/development/backend.md b/document/zh-CN/development/backend.md index d34263d4..f9e7318a 100644 --- a/document/zh-CN/development/backend.md +++ b/document/zh-CN/development/backend.md @@ -100,7 +100,7 @@ raise ResourceNotFoundError("chat_session", chat_id) raise BadRequestError("参数 name 不能为空") ``` -license 相关 402 同理：唯一来源是 `core/licensing/features.py` 的 `FeatureNotLicensed`（40201）/ `SeatLimitExceeded`（40202），路由 / 服务层不要再手搓 402。错误码分段见 [错误码参考](../api/error-codes.md)。 +License 相关 402 同理：唯一来源是 `edition_ee/licensing/features.py` 的 `FeatureNotLicensed`（40201）/ `SeatLimitExceeded`（40202），路由 / 服务层不要再手搓 402。错误码分段见 [错误码参考](../api/error-codes.md)。 ### 依赖注入 @@ -138,7 +138,7 @@ EE_ROUTERS: tuple[tuple[str, str, str | None], ...] = ( 1. 在 `api/routes/v1/` 新建路由文件，`router = APIRouter(prefix="/v1/xxx", tags=["Xxx"])`； 2. 判断归属： - **CE 能力**（个人自洽）→ 在 `CE_ROUTERS` 追加 `("模块名", "router")`； - - **EE 能力**（组织规模化）→ 在 `EE_ROUTERS` 追加 `("模块名", "router", "")`，feature 取 `core/licensing/features.py::Feature` 的值；仅当端点在 license 失效时也必须可达（登录、换 license 类基础设施）才用 `None` 豁免，并写明理由； + - **EE 能力**（组织规模化）→ 在 `edition_ee/routes/registry.py::EE_ROUTERS` 追加 `("模块名", "router", "")`，feature 取 `edition_ee/licensing/features.py::Feature` 的值；仅当端点在 License 失效时也必须可达（登录、换 License 类基础设施）才用 `None` 豁免，并写明理由； 3. EE 路由还要在 `ce/manifest.yaml` 的 `exclude` 中排除该文件（`admin_*.py` / `config_*.py` 已有通配模式覆盖）； 4. 注意表内顺序即注册顺序，同前缀族的先后关系不可变（例：`config` 公开读必须先于 `config_*` 管理台）。 diff --git a/document/zh-CN/editions/build-ce.md b/document/zh-CN/editions/build-ce.md index a5cc387e..b41fa811 100644 --- a/document/zh-CN/editions/build-ce.md +++ b/document/zh-CN/editions/build-ce.md @@ -1,5 +1,5 @@ # CE 构建管线 -> 最后更新：2026-07-02 +> 最后更新：2026-07-22 社区版（CE）不是独立分支，而是由主仓（EE，唯一开发真源）经 `scripts/build_ce.py` **确定性派生**的子集树，输出到 `dist/ce/`。核心约束是**白名单铁律：EE 专属代码在 CE 树里物理不存在**——不是注释掉、不是 if 关掉，而是文件层面删除。整条管线的唯一输入是 `ce/manifest.yaml`。 @@ -25,7 +25,7 @@ manifest 按处理顺序分为以下几段： glob 模式（相对仓库根），命中即不拷贝。覆盖： -- **后端 EE 模块**：SSO / 团队权限（`core/auth/sso.py`、`team_permissions.py` 等）、云存储（`core/storage/s3.py`、`oss.py`）、持久沙箱 provider（opensandbox / cube 全套）、记忆审计、技能蒸馏、license 验签实现 `core/licensing/_ee_verify.py`、EE service（team / sso_sync / distillation / sandbox_rebuild / security / cube_template_builder）； +- **后端 EE 模块**：完整的 `edition_ee/**` 实现根（团队/RBAC、SSO、License 验签与闸门、EE ORM、Dify 集成），以及云存储（`core/storage/s3.py`、`oss.py`）、持久沙箱 provider、记忆审计、技能蒸馏等 EE 服务； - **EE 路由**：`api/routes/v1/admin_*.py`、`config_*.py`、`audit.py`、`auth.py`、`team_files.py`、`service_configs.py`、`data_sources.py`、`db_metadata.py`、`gateway_*.py`； - **行业 MCP**：`mcp_servers/query_database_mcp/**`、`ai_chain_information_mcp/**`； - **主仓 alembic 链整体**（`alembic/versions/**`，CE 用 overlay 的独立链，见下文）； @@ -60,7 +60,7 @@ transforms 只改文件内容不改路径，因此本步骤为确有需要的路 ### 4. `split` — 文件内 user/admin 混合端点的 CE 子集断言 -`content.py` / `models.py` / `projects.py` 三个路由文件内同时含用户端点与管理端点，CE 取 overlay 中的 user 子集版本。**build_ce.py 在 overlay 步骤前断言这些文件在 overlay 中存在**——主仓全量版禁止漏进 CE，缺失即 fail。 +`manifest.split` 明确列出所有必须由 CE overlay 整文件替换的版本接缝。**build_ce.py 在 overlay 步骤前逐项断言替代文件存在**——源树全量实现禁止漏进 CE，缺失即 fail。 ### 5. `overlay` — CE 专属整文件替换 / 新增 @@ -71,20 +71,28 @@ transforms 只改文件内容不改路径，因此本步骤为确有需要的路 | `README.md` / `README_CN.md` / `LICENSE` / `NOTICE` / `CONTRIBUTING.md` / `SECURITY.md` | CE 开源仓门面文件；默认 README 为英文，中文作为语言切换入口保留 | | `install.sh` | 面向个人无 Docker 模式的公开一键安装脚本 | | `.env.example` | CE 环境模板（`JX_EDITION=ce`，无内网 IP / 无品牌默认） | -| `.hugagent-edition` | 仅在派生后出现的机器可读 `ce` 标识；让发布工具区分公开 CE checkout 与生成器异常缺失的 FULL checkout | +| `.hugagent-edition` | 仅在派生后出现的机器可读 `ce` 标识；让发布工具区分派生 CE checkout 与生成器异常缺失的源代码 checkout | | `.github/workflows/desktop-release.yml` | 公开 CE 桌面发版 workflow，包含 release tag / 版本前置门禁 | -| `src/backend/core/licensing/manager.py` | **CE stub**：`mode()` 恒 `"ce"`、`has()` 恒 False、不限席位，无任何验签实现体 | -| `src/backend/core/auth/permissions_iface.py` | 权限接口层单租户 stub（接缝 C3）：自己的资源恒最高权限，团队权限恒 `none`（存量团队数据不因 stub 放行而对全员可读） | +| `src/backend/api/routes/v1/__init__.py` | CE 路由注册表；`EE_ROUTERS` 恒为空 | +| `src/backend/core/auth/permissions_iface.py` | 单租户 owner-only 权限接口；不导出团队权限函数 | +| `src/backend/core/services/artifact_edition.py` | 个人空间 artifact 作用域接口；不暴露团队字段、权限或仓储方法 | +| `src/backend/core/llm/tools/edition_{myspace,myspace_vfs,artifact_recovery}.py` | 个人空间工具、VFS 与恢复接口；组织空间实现不进入 CE | +| `src/backend/core/config/edition_display_names.py` | CE 工具展示名；不包含团队工具名称 | | `src/backend/core/memory/audit.py` | 记忆审计 no-op stub（同名接口、不落数据） | | `src/backend/alembic/versions/ce_0001_initial.py` | CE 独立迁移链基线（见下节） | -| `src/backend/api/routes/v1/{content,models,projects}.py` | split 文件的 user 子集版本 | +| `src/backend/api/routes/v1/{agents,content,kb_models,projects}.py` | 去除管理端点与组织字段后的 CE API 契约 | | `src/backend/mcp_servers/_ports.py` | 8 个通用工具的端口表（EE 行业工具端口标注 reserved） | | `src/frontend/default.conf.template` | CE 前端 Nginx 模板，移除 `/gateway/**` 反代与 litellm upstream | | `src/frontend/src/main.tsx` | CE 入口：只挂主应用 / API 文档 / 分享预览，不挂 /admin、/config | | `src/frontend/src/updates.ts` | CE 版本说明数据 | | `.claude/skills/hugagent-{backend,frontend}-dev/…` | 项目开发 skill 的 CE 版 SKILL.md 与 references（剔除 admin 面板 / EE 路由注册等商业版段落） | -> 路由注册表 `api/routes/v1/__init__.py` **不需要** overlay 副本：`iter_edition_routers` 对物理缺失的 EE 模块静默跳过，同一份文件两树共用。 +> License、Team/RBAC、EE ORM 与 Dify 的实现根均在 `edition_ee/**`，CE 不提供同名实现 stub；派生树内对 `edition_ee` 的 import 探测必须返回不存在。 + +团队文件仓储、组织 MySpace 工具、VFS 与 artifact 恢复实现分别位于 +`edition_ee/db/artifact_repository.py` 和 +`edition_ee/services/{myspace_tools,myspace_vfs,artifact_recovery}.py`。 +共享模块只保留版本中性的调用接口，CE overlay 提供个人版实现，不保留商业字段或工具名。 ### 6. `brand_scan` — 品牌门禁正则文件 @@ -93,13 +101,16 @@ transforms 只改文件内容不改路径，因此本步骤为确有需要的路 ## build_ce.py 步骤流水 ``` -[1/7] 拷贝 git ls-files（cached + 未跟踪未忽略）为白名单，减 exclude 与默认忽略 +[1/7] 拷贝 git ls-files --cached 为白名单，减 exclude 与默认忽略 [1/7] 改名 按 manifest.renames 执行可选路径迁移（当前配置为空） [2/7] 变换 manifest.transforms 整树文本替换（二进制免扫；其他产品线品牌字面量告警） [3/7] 裁剪 manifest.prunes 五个 pruner [4/7] overlay 先断言 split 文件在 overlay 中存在，再整树叠加（跳过 __pycache__/pyc） +[4/7] 禁止产物 断言 EE 路径、表名、外键和运行时源码商业符号均为 0 命中； + 测试目录只允许保存“不得出现”的负向契约断言 +[4/7] 二进制门禁 PNG/PDF/DOCX 必须匹配经人工/OCR 审阅后的 path + SHA-256 白名单 [5/7] 品牌门禁 文本逐行正则 0 命中 + 全量文件「路径」扫描（含二进制资产文件名； - 另有路径专用模式拦商业字体文件本体）；免扫的二进制数随结果上报 + 另有路径专用模式拦商业字体文件本体） [6/7] LICENSE 闸门 overlay LICENSE 仍是占位文本（含 NOTE TO MAINTAINERS 标记）时拒绝生成 [7/7] 自检 --import-check / --pytest-check / --frontend-check（可选） [8/8] 清残留 自检留下的 __pycache__ / .pytest_cache / node_modules / dist / 再生 lock @@ -107,24 +118,23 @@ transforms 只改文件内容不改路径，因此本步骤为确有需要的路 以 `git ls-files` 为拷贝清单意味着 `.env`、本地数据库等未跟踪 / 已忽略文件**天然不会进入 CE 树**。 -Windows 桌面服务载荷在两种仓库里遵守同一边界：FULL 仓中，`desktop/scripts/prepare-bundle.mjs` +Windows 桌面服务载荷在两类 checkout 中遵守同一边界：源代码 checkout 中，`desktop/scripts/prepare-bundle.mjs` 仍会找到并运行 `scripts/build_ce.py`；在有意移除生成器的公开 CE 仓中，它要求 `.hugagent-edition` 内容为 `ce`，然后只暂存当前 checkout 的 Git tracked 文件。正式发布会拒绝脏 checkout，因此该 fallback 不能把一个生成器异常缺失的任意仓库静默当成 CE 载荷。 ## CE 数据库差异 -CE 不建 EE 专属表，由 `src/backend/core/db/edition_tables.py` 给出**单一真源**： +CE 不注册、也不创建 EE 专属表。商业 ORM 类集中在 `src/backend/edition_ee/db/models/`，该目录在派生树中物理不存在；CE 的模型出口只导出 CE 映射，兼容属性由 CE model extension 提供，不注册商业列或表。 -- `EE_ONLY_TABLES`（18 张）：`teams`、`team_members`、`team_folders`、`invite_codes`、`roles`、`role_assignments`、`kb_grants`、`audit_logs`、`memory_audit`、`model_pricing`、`data_sources`、`ds_table_meta`、`ds_column_meta`、`ds_golden_sql`、`gateway_virtual_keys`、`sandbox_rebuilds`、`admin_skill_drafts`、`distillation_runs`。 -- `ce_create_all(bind)`：在**克隆 MetaData** 上过滤后建表——CE 表里指向 EE 表的跨边界 FK 约束（如 `projects → teams`，方案 D3「列保留、恒 NULL」）若原样下发，PostgreSQL 会因引用表不存在而失败，故在克隆上摘除约束（列保留），原 metadata 与 ORM 映射不受影响。集合里的表名必须真实存在于 metadata（函数内断言），防模型改名后漏更新、悄悄退化成全量建表。 +发布契约禁止 20 张表：`chat_session_user_states`、`teams`、`team_members`、`team_folders`、`invite_codes`、`roles`、`role_assignments`、`kb_grants`、`marketplace_visibility_grants`、`audit_logs`、`memory_audit`、`model_pricing`、`data_sources`、`ds_table_meta`、`ds_column_meta`、`ds_golden_sql`、`gateway_virtual_keys`、`sandbox_rebuilds`、`admin_skill_drafts`、`distillation_runs`。import 门禁会同时检查 CE metadata 中没有这些表、没有跨界外键；`projects`、`artifacts`、`user_agents`、`chat_sessions`、`marketplace_listing_states` 与 `sites` 也不得注册相应商业作用域列。 -两个建表入口共用该过滤： +两个建表入口共用 CE-only metadata： 1. `core/db/engine.py::init_db` 的 CE 分支（`JX_EDITION=ce` 时走 `ce_create_all`，SQLite 启动兜底）； -2. CE overlay 迁移基线 `ce_0001_initial.py`——CE 走**独立 alembic 链**（不复用主仓 50+ 个历史迁移），基线即「按 `EE_ONLY_TABLES` 过滤后的 create_all」，方言感知（SQLite / PostgreSQL 通吃），与 init_db 同源同滤、两者幂等不冲突。后续 CE schema 演进在 `ce_0001` 链上追加常规迁移。 +2. CE overlay 迁移基线 `ce_0001_initial.py`——CE 走**独立 alembic 链**，直接从 CE-only metadata 建表，方言感知（SQLite / PostgreSQL 通吃），与 init_db 幂等不冲突。后续 CE schema 演进在 `ce_0001` 链上追加常规迁移。 -EE（含 internal / licensed 等全部 license 状态）始终全量建表，行为与历史一致。维护规则：**新增 EE 专属模型时同步把表名加进 `EE_ONLY_TABLES`**。 +EE 始终全量建表。维护规则：**新增 EE 专属模型必须放进 `edition_ee/db/models`，并把表名加入 CE 发布契约**。 ## 产出验收 @@ -132,20 +142,21 @@ EE（含 internal / licensed 等全部 license 状态）始终全量建表，行 | 闸门 | 标准 | 兑现处 | |---|---|---| -| 路由零 EE 泄漏 | CE 树物理不含 EE 路由 / 模块；`--import-check` 下 `import api.app` 成功（缺失模块由注册表静默跳过）；`--pytest-check` 收集不报 EE import 错误 | exclude + `iter_edition_routers` | -| 品牌门禁 | 文本 0 命中；全量路径扫描通过；新增二进制资产（免内容扫描）须人工复核 | `brand_scan()` | +| 路由 / Schema 零 EE 泄漏 | CE 树物理不含 `edition_ee`；`EE_ROUTERS` 为空；OpenAPI 无组织路由、字段或文案；metadata 无禁止表、跨界外键与商业作用域列 | `--import-check` | +| 运行时源码零商业符号 | 后端与前端运行时源码不得出现 Team/RBAC 的模型、作用域字段、权限或工具符号；测试目录仅保留负向断言 | `find_forbidden_artifacts()` + CE runtime contract | +| 品牌 / 二进制门禁 | 文本 0 命中、全量路径扫描通过；PNG/PDF/DOCX 必须命中经人工/OCR 审阅的 path + SHA-256 白名单 | `brand_scan()` + `binary_allowlist_check()` | | LICENSE 闸门 | overlay LICENSE 非占位文本 | `license_placeholder_check()` | -| split 断言 | 三个 split 文件的 CE 子集在 overlay 中存在 | `main()` overlay 前置检查 | +| split 断言 | 每个声明的 CE split 替代文件都在 overlay 中存在 | `main()` overlay 前置检查 | | 前端可构建 | `--frontend-check`：npm install + vite build 通过 | `frontend_check()` | | 交付卫生 | 自检残留全部清除 | `cleanup_gate_artifacts()` | ## 日常维护要点 - **新增 EE 路由**：在 `EE_ROUTERS` 注册（见 [后端开发指南](../development/backend.md)）+ `manifest.exclude` 加对应文件 glob（`admin_*.py` / `config_*.py` 已有通配）。 -- **新增 EE 表**：`EE_ONLY_TABLES` 加表名。 +- **新增 EE 表**：模型放进 `edition_ee/db/models`，表名加入 `contracts.forbidden_tables`。 - **新增 EE 依赖 / compose 服务**：对应 prune 段补 drop 项。 -- **新增品牌资产**：确认 brand_scan 能在路径或文本层面拦住；二进制资产是内容扫描盲区，依赖路径模式 + 人工复核。 -- 改完跑一次 `python scripts/build_ce.py --allow-dirty --import-check --pytest-check` 验证。 +- **新增 PNG/PDF/DOCX**：先人工检查或 OCR 提取复核内容，再把相对路径与 SHA-256 加入 `ce/binary_allowlist.sha256`；哈希变化必须重新审阅。 +- 改完跑一次 `python scripts/build_ce.py --allow-dirty --import-check --pytest-check --frontend-check` 验证；正式发布不得使用 `--allow-dirty`，且未跟踪文件永不作为复制输入。 ## 相关源码 diff --git a/document/zh-CN/editions/license.md b/document/zh-CN/editions/license.md index 79387a74..3b88191f 100644 --- a/document/zh-CN/editions/license.md +++ b/document/zh-CN/editions/license.md @@ -1,11 +1,11 @@ # License 机制（商业版） -> 最后更新：2026-06-11 +> 最后更新：2026-07-22 -商业版（EE）采用 **GitLab 式离线 License 模型**：一份经 Ed25519 签名的授权文件（`.lic`）+ 进程内验签，**全程离线、无 license 服务器**，适配政务内网等隔离环境。本文逐项说明状态机、执法机制、签发流程与管理界面，全部内容以 `src/backend/core/licensing/` 代码为准。 +商业版（EE）采用 **GitLab 式离线 License 模型**：一份经 Ed25519 签名的授权文件（`.lic`）+ 进程内验签，**全程离线、无 license 服务器**，适配政务内网等隔离环境。能力位、验签、状态机、席位与闸门实现全部集中在 `src/backend/edition_ee/licensing/`，CE 派生树中不存在 License 运行时代码。 ## License 文件格式 -`.lic` 文件是 JSON 信封（`src/backend/core/licensing/_ee_verify.py`，格式版本 `jx-license/1`）： +`.lic` 文件是 JSON 信封（`src/backend/edition_ee/licensing/verify.py`，格式版本 `jx-license/1`）： ```json { @@ -29,11 +29,11 @@ } ``` -验签公钥内置于 `_ee_verify.py`（`_BUILTIN_PUBKEY`），可经环境变量 `LICENSE_PUBLIC_KEY` 覆盖（密钥轮换用）。`_ee_verify.py` 是**商业版专属模块，不进社区版派生树**（`ce/manifest.yaml` 显式排除；CE 树的 `manager.py` 被 overlay 替换为恒 False stub）。 +验签公钥内置于 `edition_ee/licensing/verify.py`（`_BUILTIN_PUBKEY`），可经环境变量 `LICENSE_PUBLIC_KEY` 覆盖（密钥轮换用）。整个 `edition_ee/licensing` 实现在 CE 派生树中物理不存在；CE 的版本中间件由 overlay 提供固定探针，不提供 License manager 或同名验签 stub。 ## 状态机 -`core/licensing/manager.py::LicenseManager.mode()` 返回 7 种状态之一： +`edition_ee/licensing/manager.py::LicenseManager.mode()` 返回 7 种状态之一： | mode | 触发条件 | EE 能力位 | |---|---|---| @@ -65,14 +65,14 @@ ### Feature 枚举 -`core/licensing/features.py::Feature` 只列「组织级」商业能力位（自动化 / 批量 / 个人画布 / L2-L3 记忆属社区版，不在枚举内）： +`edition_ee/licensing/features.py::Feature` 只列「组织级」商业能力位（自动化 / 批量 / 个人画布 / L2-L3 记忆属社区版，不在枚举内）： `sso`、`multi_tenancy`、`audit`、`memory_audit`、`billing`、`quota`、`persistent_sandbox`、`cloud_storage`、`industry_tools`、`content_admin`、`system_config`、`canvas_collab`、`whitelabel`。 ### 两道防线 1. **第一道：路由注册表**（CE 树物理不含 EE 路由文件）——见 [CE 构建管线](build-ce.md)。 -2. **第二道：`requires_feature` 守卫**（`core/licensing/deps.py`）——防「商业版部署了全量代码、但 license 未购买某能力包」。 +2. **第二道：`requires_feature` 守卫**（`edition_ee/licensing/deps.py`）——防「商业版部署了全量代码、但 license 未购买某能力包」。 EE 路由与能力位的对应关系在注册表 `src/backend/api/routes/v1/__init__.py::EE_ROUTERS` 中声明（表项第三列即能力位），`api/app.py` 注册时按表挂守卫： @@ -85,7 +85,7 @@ EE 路由与能力位的对应关系在注册表 `src/backend/api/routes/v1/__in | `config_security`、`service_configs` | `system_config` | | `config_verify`、`config_license`、`auth` | **None（显式豁免）** | -三个豁免项是刻意设计：`config_verify` 是控制台登录校验、`config_license` 是换 license 的入口、`auth` 是登录 / 会话基础设施——license 失效时也必须可达，否则用户陷入「402 → 登出 → 登录 → 402」死循环且无从换 license。SSO 能力位不在路由级整体豁免，而是自行守卫：authorize-url 端点挂 `requires_feature(Feature.SSO)`（`api/routes/v1/auth.py`），remote ticket 交换在 `core/auth/sso.py::exchange_ticket` 内检查。 +三个豁免项是刻意设计：`config_verify` 是控制台登录校验、`config_license` 是换 license 的入口、`auth` 是登录 / 会话基础设施——license 失效时也必须可达，否则用户陷入「402 → 登出 → 登录 → 402」死循环且无从换 license。SSO 能力位不在路由级整体豁免，而是自行守卫：authorize-url 端点挂 `requires_feature(Feature.SSO)`（`edition_ee/routes/auth.py`），remote ticket 交换在 `edition_ee/auth/sso.py::exchange_ticket` 内检查。 > 注：`quota` / `persistent_sandbox` / `cloud_storage` / `industry_tools` / `canvas_collab` / `whitelabel` / `memory_audit` 当前在 license entitlement 与探针中表达，但**未挂路由级守卫**——这些能力的边界主要由 CE 树物理排除与部署配置兑现。 @@ -101,7 +101,7 @@ EE 路由与能力位的对应关系在注册表 `src/backend/api/routes/v1/__in ## 席位上限 -席位计数单一真源在 `core/licensing/seats.py`： +席位计数单一真源在 `edition_ee/licensing/seats.py`： - `seats_used(db)`：已占用席位 = `users_shadow` 全量行数（含 SSO 影子账号）； - `seat_available(db)`：新增用户前的校验（本地注册、SSO 自动建号共用）。CE / internal / 不限席位（`seats=0`）恒放行；`licensed` / `grace` 下要求 `active_users < seats`；`expired` / `invalid` / `missing` 恒拒绝； @@ -115,7 +115,7 @@ EE 路由与能力位的对应关系在注册表 `src/backend/api/routes/v1/__in ### `GET/POST /v1/config/license`（CONFIG_TOKEN 鉴权） -`api/routes/v1/config_license.py`： +`edition_ee/routes/config_license.py`： - `GET`：完整状态（`license_manager.status()` + `seats_used`），含各能力位实时判定、宽限天数、license 元信息； - `POST`：上传 `.lic` 全文（≤64KB）热换。流程：**先验签再落盘**（无效文件不覆盖现有 license）→ 拒绝激活已过宽限期的 license（grace 期内放行，保证文件丢失 / 主机重建时宽限窗口内能重新挂回）→ 原子写入 `LICENSE_KEY_PATH`（tmp + `os.replace`）→ `license_manager.reload()` 即时生效，**无需重启**。未配置 `LICENSE_KEY_PATH` 时返回 400 提示先配置。 @@ -141,7 +141,7 @@ const multiTenancy = useEditionStore((s) => (s.loaded ? !!s.features.multi_tenan ```bash # 1. 生成 Ed25519 密钥对（一次性；私钥务必离线保管，泄露即可被任意签发） python scripts/license_tool.py keygen --out-dir ~/jx-license-keys -# 输出公钥值 → 更新到 core/licensing/_ee_verify.py::_BUILTIN_PUBKEY（或客户侧 LICENSE_PUBLIC_KEY） +# 输出公钥值 → 更新到 edition_ee/licensing/verify.py::_BUILTIN_PUBKEY（或客户侧 LICENSE_PUBLIC_KEY） # 2. 签发 python scripts/license_tool.py issue \ @@ -157,7 +157,7 @@ python scripts/license_tool.py inspect customer.lic python scripts/license_tool.py verify customer.lic --pub ~/jx-license-keys/license_signing.pub ``` -`issue` 自动生成 `license_id`（`lic_` + 16 位 hex）、校验日期格式；`--seats 0` 表示不限席位；`--features` 逗号分隔能力位，`"*"` 为全功能。信封格式与验签逻辑的唯一真源在后端 `_ee_verify.py`，签发工具直接复用（显式传公钥，不拖入后端配置链）。 +`issue` 自动生成 `license_id`（`lic_` + 16 位 hex）、校验日期格式；`--seats 0` 表示不限席位；`--features` 逗号分隔能力位，`"*"` 为全功能。信封格式与验签逻辑的唯一真源在 `edition_ee/licensing/verify.py`，签发工具直接复用（显式传公钥，不拖入后端配置链）。 ## 私有化交付清单 @@ -170,14 +170,14 @@ python scripts/license_tool.py verify customer.lic --pub ~/jx-license-keys/licen | 主题 | 路径 | |---|---| -| 状态机 / 门面 | `src/backend/core/licensing/manager.py` | -| Ed25519 验签（EE 专属） | `src/backend/core/licensing/_ee_verify.py` | -| 能力位枚举 + 402 异常 | `src/backend/core/licensing/features.py` | -| `requires_feature` 守卫 | `src/backend/core/licensing/deps.py` | -| 席位计数 | `src/backend/core/licensing/seats.py` | +| EE 状态机 | `src/backend/edition_ee/licensing/manager.py` | +| Ed25519 验签（EE 专属） | `src/backend/edition_ee/licensing/verify.py` | +| 能力位枚举 + 402 异常 | `src/backend/edition_ee/licensing/features.py` | +| `requires_feature` 守卫 | `src/backend/edition_ee/licensing/deps.py` | +| 席位计数 | `src/backend/edition_ee/licensing/seats.py` | | EE 路由 ↔ 能力位注册表 | `src/backend/api/routes/v1/__init__.py` | | 守卫挂接 | `src/backend/api/app.py`（edition 注册循环） | -| 状态查询 / 热换 | `src/backend/api/routes/v1/config_license.py` | +| 状态查询 / 热换 | `src/backend/edition_ee/routes/config_license.py` | | 探针 | `src/backend/api/routes/v1/meta.py` | | 签发工具 | `scripts/license_tool.py` | | License 面板 | `src/frontend/src/components/config/LicensePanel.tsx` | diff --git a/document/zh-CN/editions/overview.md b/document/zh-CN/editions/overview.md index 277c9ff6..ce4885b2 100644 --- a/document/zh-CN/editions/overview.md +++ b/document/zh-CN/editions/overview.md @@ -1,5 +1,5 @@ # 社区版与商业版总览 -> 最后更新：2026-07-02 +> 最后更新：2026-07-22 HugAgentOS 采用 **open-core（开放内核）** 模式发行，对标 Dify / FastGPT 等开源 Agent 平台的通行做法： @@ -59,7 +59,7 @@ python scripts/build_ce.py # 生成 dist/ce/（CE 派生树） | 形态 | 配置 | 行为 | |---|---|---| -| 社区版 | `JX_EDITION=ce`（CE 派生树 `.env.example` 默认值） | 全部 EE 能力位恒 `False`，不限席位；`core/licensing/manager.py` 在 CE 树被 overlay 替换为无验签 stub | +| 社区版 | `JX_EDITION=ce`（CE 派生树 `.env.example` 默认值） | CE 树物理不含 License 实现；版本探针固定返回 CE 形态与空的 EE 能力集，不执行验签、不限制席位 | | 商业版 · internal | `JX_EDITION=ee` 且未配置 license 文件且 `JX_LICENSE_REQUIRED=false`（主仓默认） | **内部 / 全托管部署模式：全功能放行**，与历史部署行为完全一致——存量部署升级到含 license 机制的版本后无需任何配置变更 | | 商业版 · licensed | `JX_EDITION=ee` + 有效 license 文件 | 按 license 中的 entitlement（能力位清单 + 席位 + 有效期）放行 | @@ -85,7 +85,7 @@ COMPOSE_PROFILES=mem0 docker compose up -d ## 升级路径 -- **CE → EE**：商业版以**镜像包 + License 文件**交付（适配政务内网离线环境）。数据库层面 CE 表集合是 EE 的真子集，且跨边界外键列（`team_id` 等）在 CE 中保留为恒 NULL（方案 D3），数据可保留迁移。 +- **CE → EE**：商业版以**镜像包 + License 文件**交付（适配政务内网离线环境）。CE 表集合是 EE 的真子集：20 张 EE 表、其外键及共享资源上的商业作用域列均不注册；升级时由交付迁移补齐组织结构。 > ⚠️ 注意：CE 走独立迁移链（基线 `ce_0001`，见 [CE 构建管线](build-ce.md#ce-数据库差异)），与 EE 的主仓迁移链不同；CE 存量库切换 EE 镜像时的 alembic 版本对接由交付实施完成（当前无自动转换工具，属规划中）。 - **EE internal → EE licensed**：私有化交付时配置 `LICENSE_KEY_PATH` 与 `JX_LICENSE_REQUIRED=true`，在 `/config` 管理台 License 面板上传 `.lic` 文件即时激活，无需重启。 - **续期 / 扩容**：上传新 license 文件热替换（同上），到期后有宽限期缓冲（默认 14 天）。 @@ -95,8 +95,8 @@ COMPOSE_PROFILES=mem0 docker compose up -d | 主题 | 路径 | |---|---| | 版本 / License 配置 | `src/backend/core/config/settings.py`（`EditionSettings` / `LicenseSettings`） | -| License 门面与状态机 | `src/backend/core/licensing/manager.py` | -| EE 能力位枚举 | `src/backend/core/licensing/features.py` | +| License 状态机 | `src/backend/edition_ee/licensing/manager.py` | +| EE 能力位枚举 | `src/backend/edition_ee/licensing/features.py` | | 路由注册表（CE/EE 两表） | `src/backend/api/routes/v1/__init__.py` | | 版本探针 | `src/backend/api/routes/v1/meta.py` | | 前端 edition 门控 | `src/frontend/src/stores/editionStore.ts` | diff --git a/document/zh-CN/modules/admin-console.md b/document/zh-CN/modules/admin-console.md index 5ff39412..d6adaa00 100644 --- a/document/zh-CN/modules/admin-console.md +++ b/document/zh-CN/modules/admin-console.md @@ -11,7 +11,7 @@ HugAgentOS 提供**两个相互独立的管理入口**，分别面向内容运 入口分流在 `src/frontend/src/main.tsx`：按 `window.location.pathname` 前缀渲染 `AdminApp`（`/admin`）、`ConfigApp`（`/config`）、`ApiDocApp`（`/api-docs`）或主应用。两台之间有互跳按钮（`/admin` 顶栏「系统配置」→ `/config`；`/config` 顶栏「内容管理」→ `/admin`）。两类令牌的鉴权机制见 [认证与权限](auth.md)。 -后端管理路由的 CE/EE 归属以路由注册表 `src/backend/api/routes/v1/__init__.py` 为唯一真源：CE 派生树**物理不包含** EE 路由文件（第一道防线），EE 部署再按 License 能力位（`content_admin` / `billing` / `audit` / `multi_tenancy` / `system_config`）做二次守卫（第二道防线，`core/licensing/deps.py::requires_feature`）。下文逐组标注。 +后端管理路由的 CE/EE 归属以 `src/backend/api/routes/v1/__init__.py` 与 `src/backend/edition_ee/routes/registry.py` 组成的注册表为唯一真源：CE 派生树**物理不包含** EE 路由文件（第一道防线），EE 部署再按 License 能力位（`content_admin` / `billing` / `audit` / `multi_tenancy` / `system_config`）做二次守卫（第二道防线，`edition_ee/licensing/deps.py::requires_feature`）。下文逐组标注。 ## /admin 运营管理台 @@ -148,13 +148,14 @@ HugAgentOS 提供**两个相互独立的管理入口**，分别面向内容运 | 系统管理台 | `src/frontend/src/ConfigApp.tsx`、`src/frontend/src/components/config/` | | 路由注册表（CE/EE 单一真源） | `src/backend/api/routes/v1/__init__.py` | | 管理凭证依赖 | `src/backend/api/deps.py` | -| License 能力位 | `src/backend/core/licensing/features.py`、`src/backend/core/licensing/deps.py` | +| License 能力位 | `src/backend/edition_ee/licensing/features.py`、`src/backend/edition_ee/licensing/deps.py` | | 技能 / 草稿 / 市场 | `src/backend/api/routes/v1/admin_skills.py`、`admin_skill_drafts.py`、`admin_marketplace.py` | | 提示词 / MCP / 子智能体 | `src/backend/api/routes/v1/admin_prompts.py`、`admin_mcp_servers.py`、`admin_agents.py` | -| 知识库 / 沙盒 | `src/backend/api/routes/v1/admin_kb.py`、`admin_sandbox.py` | +| 知识库 / 沙盒 | `src/backend/edition_ee/routes/admin_kb.py`、`src/backend/api/routes/v1/admin_sandbox.py` | | 计费 / 用量 / 日志 / 会话审查 | `src/backend/api/routes/v1/admin_billing.py`、`admin_usage_logs.py`、`admin_logs.py`、`admin_chat_history.py` | | 内容管理 | `src/backend/api/routes/v1/content.py` | -| 用户 / 团队 / 邀请 / 安全 / License | `src/backend/api/routes/v1/config_users.py`、`config_teams.py`、`config_invites.py`、`config_security.py`、`config_license.py` | +| 用户 / 团队 / 邀请 / License（EE） | `src/backend/edition_ee/routes/config_users.py`、`config_teams.py`、`config_invites.py`、`config_license.py` | +| 安全 | `src/backend/api/routes/v1/config_security.py` | | 服务配置 | `src/backend/api/routes/v1/service_configs.py` | 延伸阅读：[认证与权限](auth.md) · [提示词系统](prompts.md) · [版本与授权](../editions/overview.md) diff --git a/document/zh-CN/modules/auth.md b/document/zh-CN/modules/auth.md index 92b03adb..b4e5064a 100644 --- a/document/zh-CN/modules/auth.md +++ b/document/zh-CN/modules/auth.md @@ -102,8 +102,8 @@ POST /mock-sso/ticket/exchange → 校验 ticket，返回用户信息 | `POST/PUT/DELETE /v1/me/avatar` | `api/routes/v1/users.py` | 头像上传（≤2MB）/ 设置 / 清除 | | `GET/PUT /v1/users/{id}/preferences` | `api/routes/v1/users.py` | 用户偏好设置 | | `POST /v1/me/onboarding/complete` | `api/routes/v1/users.py` | 校验主模型并完成 CE 首次初始化 | -| `GET /v1/me/teams` 等 | `api/routes/v1/me.py` | 用户侧团队查看、邀请成员、移除 / 退出（商业版 EE；CE 树缺团队模块时降级 404） | -| `GET /v1/me/users/search` | `api/routes/v1/me.py` | 邀请成员时的用户搜索 | +| `GET /v1/me/teams` 等 | `edition_ee/routes/me_teams.py` | 用户侧团队查看、邀请成员、移除 / 退出（商业版 EE；CE 树物理不包含该模块） | +| `GET /v1/me/users/search` | `edition_ee/routes/me_teams.py` | 邀请成员时的用户搜索 | ## 权限体系 @@ -113,15 +113,15 @@ POST /mock-sso/ticket/exchange → 校验 ticket，返回用户信息 | 实现文件 | 职责 | |---|---| -| `core/auth/team_permissions.py` | 团队文件夹权限解析（商业版 EE） | -| `core/auth/project_permissions.py` | 项目访问权限（团队项目属商业版 EE） | -| `core/auth/chat_share_permissions.py` | 会话访问 / 删除 / 分享范围权限 | +| `edition_ee/auth/team_permissions.py` | 团队文件夹权限解析（商业版 EE） | +| `edition_ee/auth/project_permissions.py` | 团队项目访问权限（商业版 EE） | +| `edition_ee/auth/chat_share_permissions.py` | 团队会话访问 / 删除 / 分享范围权限（商业版 EE） | `resolve_artifact_access(db, user_id, owner_id, team_id)` 是 owner ∪ team 合成的统一访问级判定：owner 恒为 `admin` → 团队成员按团队权限 → 其余 `none`。文件下载（`api/routes/files.py`）、知识库、我的空间等所有 artifact 访问点共用。 ### 团队角色与文件权限（商业版 EE） -`core/auth/roles.py` 定义三级团队角色：`owner`（所有者）> `admin`（管理员）> `member`（成员）。团队文件权限映射（`team_permissions.py`）： +`edition_ee/auth/roles.py` 定义三级团队角色：`owner`（所有者）> `admin`（管理员）> `member`（成员）。团队文件权限映射在 `edition_ee/auth/team_permissions.py`： | 团队角色 | 文件权限 | 能做什么 | |---|---|---| @@ -134,7 +134,7 @@ POST /mock-sso/ticket/exchange → 校验 ticket，返回用户信息 ### 用户级权限位 -按用户粒度的功能开关存储在 `users_shadow.metadata`（ORM 字段 `extra_data`）JSON 列中，由 Config 管理台的用户管理模块（`api/routes/v1/config_users.py`）设置。**默认全部关闭**（关闭即从 metadata 移除键）： +按用户粒度的功能开关存储在 `users_shadow.metadata`（ORM 字段 `extra_data`）JSON 列中，由 Config 管理台的 EE 用户管理模块（`edition_ee/routes/config_users.py`）设置。**默认全部关闭**（关闭即从 metadata 移除键）： | 权限位 | 默认 | 控制接口 | 门控内容 | |---|---|---|---| @@ -179,9 +179,9 @@ Key 形如 `sk-jx-...`，在**所有 AUTH_MODE 下**都可作为 Bearer 调用 团队与注册码属多租户能力（License 能力位 `multi_tenancy`），管理端在 Config 系统管理台： -- **团队管理**（`api/routes/v1/config_teams.py`）：团队 CRUD、成员增删、角色设置（owner/admin/member）。 -- **注册码管理**（`api/routes/v1/config_invites.py`）：批量生成、列表、吊销、删除。注册码形如 `JX-ABCD-2345`（`core/auth/invite.py`，字符表去除易混淆的 O/0、I/1 等），默认有效期 `INVITE_CODE_DEFAULT_TTL_HOURS`（168 小时）；消费用条件 UPDATE 保证并发安全，可预绑定团队与角色。 -- **用户侧**（`api/routes/v1/me.py`）：团队 owner/admin 可直接邀请成员、移除成员；成员可退出。 +- **团队管理**（`edition_ee/routes/config_teams.py`）：团队 CRUD、成员增删、角色设置（owner/admin/member）。 +- **注册码管理**（`edition_ee/routes/config_invites.py`）：批量生成、列表、吊销、删除。注册码形如 `JX-ABCD-2345`（`edition_ee/auth/invite.py`，字符表去除易混淆的 O/0、I/1 等），默认有效期 `INVITE_CODE_DEFAULT_TTL_HOURS`（168 小时）；消费用条件 UPDATE 保证并发安全，可预绑定团队与角色。 +- **用户侧**（`edition_ee/routes/me_teams.py`）：团队 owner/admin 可直接邀请成员、移除成员；成员可退出。 ## 审计 @@ -197,15 +197,15 @@ Key 形如 `sk-jx-...`，在**所有 AUTH_MODE 下**都可作为 Bearer 调用 | Mock SSO / 本地登录注册页 | `src/backend/api/routes/v1/mock_sso.py`、`src/backend/core/auth/mock_ticket_store.py` | | 密码哈希 | `src/backend/core/auth/password.py` | | 权限接口层（CE/EE 接缝） | `src/backend/core/auth/permissions_iface.py` | -| 团队角色 / 文件权限 | `src/backend/core/auth/roles.py`、`src/backend/core/auth/team_permissions.py` | -| 项目 / 会话分享权限 | `src/backend/core/auth/project_permissions.py`、`src/backend/core/auth/chat_share_permissions.py` | +| 团队角色 / 文件权限（EE） | `src/backend/edition_ee/auth/roles.py`、`src/backend/edition_ee/auth/team_permissions.py` | +| 项目 / 会话分享权限（EE） | `src/backend/edition_ee/auth/project_permissions.py`、`src/backend/edition_ee/auth/chat_share_permissions.py` | | 管理凭证依赖 | `src/backend/api/deps.py` | -| 用户资料 / 偏好 | `src/backend/api/routes/v1/users.py`、`src/backend/api/routes/v1/me.py` | -| 用户权限位管理 | `src/backend/api/routes/v1/config_users.py` | +| 用户资料 / 偏好 | `src/backend/api/routes/v1/users.py` | +| 用户权限位管理 | `src/backend/edition_ee/routes/config_users.py` | | 个人 API-Key | `src/backend/api/routes/v1/api_keys.py`、`src/backend/core/services/api_key_service.py` | | 能力中心自助（owner 隔离） | `src/backend/api/routes/v1/me_capabilities.py` | -| 邀请码 | `src/backend/core/auth/invite.py`、`src/backend/api/routes/v1/config_invites.py` | -| 团队管理 | `src/backend/api/routes/v1/config_teams.py` | -| License 能力位守卫 | `src/backend/core/licensing/features.py`、`src/backend/core/licensing/deps.py` | +| 邀请码 | `src/backend/edition_ee/auth/invite.py`、`src/backend/edition_ee/routes/config_invites.py` | +| 团队管理 | `src/backend/edition_ee/routes/config_teams.py`、`src/backend/edition_ee/routes/me_teams.py` | +| License 能力位守卫 | `src/backend/edition_ee/licensing/features.py`、`src/backend/edition_ee/licensing/deps.py` | 延伸阅读：[管理台](admin-console.md) · [版本与授权](../editions/overview.md) · [环境变量](../deployment/environment-variables.md) diff --git a/document/zh-CN/modules/canvas-artifacts.md b/document/zh-CN/modules/canvas-artifacts.md index 2659953f..86fefe34 100644 --- a/document/zh-CN/modules/canvas-artifacts.md +++ b/document/zh-CN/modules/canvas-artifacts.md @@ -30,7 +30,7 @@ 2. 运行时动态 `import('@univerjs/presets')` + `@univerjs/preset-sheets-core`（中文语言包）渲染电子表格——**实际只加载免费的 core 预设**。 3. 编辑后通过 `exportXlsx()` 导出为新的 xlsx File，由 `CanvasPanel` 调 `api.ts::overwriteFile` 回写到同一个 `file_id`，dirty 状态驱动「保存」按钮。 -> 多人实时协同编辑为商业版 EE 能力（`Feature.CANVAS_COLLAB`，`core/licensing/features.py`）。注意 `src/frontend/package.json` 目前仍声明了 `@univerjs/preset-sheets-advanced` 依赖（Univer 商业 License 预设）——运行时代码并未导入它；按开源方案，CE 派生树不应携带该依赖。 +> 多人实时协同编辑为商业版 EE 能力（`Feature.CANVAS_COLLAB`，`edition_ee/licensing/features.py`）。注意 `src/frontend/package.json` 目前仍声明了 `@univerjs/preset-sheets-advanced` 依赖（Univer 商业 License 预设）——运行时代码并未导入它；按开源方案，CE 派生树不应携带该依赖。 ## Artifact 中心（我的空间） diff --git a/document/zh-CN/modules/catalog.md b/document/zh-CN/modules/catalog.md index 89f316ee..75f80d30 100644 --- a/document/zh-CN/modules/catalog.md +++ b/document/zh-CN/modules/catalog.md @@ -72,7 +72,7 @@ get_enabled_ids("mcp") # 某类全部启用 id | 来源 | 条件 | 标记 | |---|---|---| -| Dify 外部知识库 | `KNOWLEDGE_BASE=dify` 且凭据可用（`core/kb/dify_kb.py::is_dify_enabled`），数据集列表带 60s 进程缓存 | `visibility: public`（**商业版 EE**：对接外部 Dify 知识库） | +| Dify 外部知识库 | `KNOWLEDGE_BASE=dify` 且凭据可用（`edition_ee/kb/dify.py::is_dify_enabled`），数据集列表带进程缓存 | `visibility: public`（**商业版 EE**：适配器不进入 CE） | | 公共自建知识库 | 管理台「知识库管理」创建（本地 Milvus），所有用户可见、前台只读 | `visibility: public` | | 用户私有知识库 | 当前用户的本地 KB 空间 | `visibility: private` | @@ -123,6 +123,6 @@ get_enabled_ids("mcp") # 某类全部启用 id | 用户自助能力 | `src/backend/api/routes/v1/me_capabilities.py` | | MCP 服务配置（DB） | `src/backend/core/services/mcp_service.py`，`api/routes/v1/admin_mcp_servers.py` | | 技能管理 | `src/backend/api/routes/v1/admin_skills.py`，`core/agent_skills/` | -| Dify KB 注入 | `src/backend/core/kb/dify_kb.py` | +| Dify KB 注入（EE） | `src/backend/edition_ee/kb/dify.py`，经共享接缝 `core/kb/external_provider.py` 调用 | | 前端能力中心 | `src/frontend/src/components/catalog/`，`stores/catalogStore.ts` | | 工厂消费侧 | `src/backend/core/llm/agent_factory.py::_effective_mcp_server_keys` | diff --git a/document/zh-CN/modules/knowledge-base.md b/document/zh-CN/modules/knowledge-base.md index 20e33ff3..270cb03f 100644 --- a/document/zh-CN/modules/knowledge-base.md +++ b/document/zh-CN/modules/knowledge-base.md @@ -105,7 +105,7 @@ ORM 定义在 `src/backend/core/db/models/knowledge.py`： ## Dify 外接知识库（商业版 EE） -客户端封装在 `src/backend/core/kb/dify_kb.py`。启用判定 `is_dify_enabled()` 的优先级： +商业版客户端位于 `src/backend/edition_ee/kb/dify.py`，共享路由通过 `core/kb/external_provider.py` 接缝调用。CE 派生树把该接缝替换为禁用实现，且不包含 Dify 客户端。启用判定 `is_dify_enabled()` 的优先级： 1. DB 系统配置 `knowledge_base.provider == "dify"`（Config 管理台可改）； 2. 环境变量 `KNOWLEDGE_BASE=dify`； @@ -137,7 +137,8 @@ DIFY_API_KEY=dataset-... # 兼容别名 DIFY_AUTH_TOKEN |---|---| | `src/backend/core/kb/kb_parser.py` | 文档解析 + 父子分块（5 种 chunk_method） | | `src/backend/core/kb/kb_vector.py` | Milvus collection、embedding、混合检索、重排 | -| `src/backend/core/kb/dify_kb.py` | Dify datasets 客户端与启用判定 | +| `src/backend/edition_ee/kb/dify.py` | Dify datasets 客户端与启用判定（仅 EE） | +| `src/backend/core/kb/external_provider.py` | 版本中立的外部知识库接缝；CE overlay 将其禁用 | | `src/backend/core/content/kb_processing.py` | 后台向量化任务、LLM 关键词 / 问题增强 | | `src/backend/core/content/file_validation.py` | 上传文件校验（扩展名 + magic bytes） | | `src/backend/core/content/file_parser.py` | 通用文件解析器 | diff --git a/document/zh-CN/modules/memory.md b/document/zh-CN/modules/memory.md index 68a3a664..992c2edd 100644 --- a/document/zh-CN/modules/memory.md +++ b/document/zh-CN/modules/memory.md @@ -83,7 +83,7 @@ save_memories_background() - 失败不冒泡（审计不阻塞主流程）； - 开关：`MEMORY_AUDIT_ENABLED`（默认 `true`）。 -按 [版本说明](../editions/overview.md)，记忆审计是商业版能力位（`core/licensing/features.py::Feature.MEMORY_AUDIT`）。审计查询接口为 `GET /v1/memories/audit`（支持按 action / layer 过滤）。 +按 [版本说明](../editions/overview.md)，记忆审计是商业版能力位（`edition_ee/licensing/features.py::Feature.MEMORY_AUDIT`）。审计查询接口为 `GET /v1/memories/audit`（支持按 action / layer 过滤）。 ## 记忆管理 API @@ -195,7 +195,8 @@ RERANKER_API_KEY=... | `src/backend/orchestration/memory_integration.py` | 检索启动、冻结块组装与注入、保存转调 | | `src/backend/orchestration/workflow.py` | 主编排：记忆 hook 接线点 | | `src/backend/api/routes/v1/memories.py` | `/v1/memories` 管理 API | -| `src/backend/core/db/models/memory.py` | `MemoryAudit` / `MemorySanitizerRule` ORM | +| `src/backend/core/db/models/memory.py` | 共享的 `MemorySanitizerRule` ORM | +| `src/backend/edition_ee/db/models/memory.py` | `MemoryAudit` ORM（仅 EE） | | `src/frontend/src/components/settings/SettingsModal.tsx` | 记忆设置 + 分层记忆弹窗 | | `src/frontend/src/components/memory/FactsList.tsx` | L2 事实列表组件 | | `docker-compose.yml`（`mem0` profile） | Milvus / etcd / MinIO / Neo4j | diff --git a/document/zh-CN/modules/projects-myspace.md b/document/zh-CN/modules/projects-myspace.md index 8c8e14f2..355170a1 100644 --- a/document/zh-CN/modules/projects-myspace.md +++ b/document/zh-CN/modules/projects-myspace.md @@ -97,7 +97,7 @@ teams ─────────┬── team_members（role: owner/admin/memb ## 团队文件夹与团队文件（商业版 EE） -用户侧路由 `src/backend/api/routes/v1/team_files.py`，挂 `multi_tenancy` 能力位（EE 路由表）；管理台对应 `/v1/config/teams/*`（`config_teams.py`）。 +用户侧路由 `src/backend/edition_ee/routes/team_files.py`，挂 `multi_tenancy` 能力位（EE 路由表）；管理台对应 `/v1/config/teams/*`（`edition_ee/routes/config_teams.py`）。 | 方法 | 路径 | 说明 | |---|---|---| @@ -109,7 +109,7 @@ teams ─────────┬── team_members（role: owner/admin/memb | POST | `/v1/artifacts/{artifact_id}/move-to-team` | 个人文件转团队文件 | | GET / PUT | `/v1/teams/{team_id}/members/permissions`、`.../{user_id}/permission` | 成员文件权限查看 / 调整 | -权限模型：`TeamMember.role`（owner/admin/member）+ `file_permission`（viewer/editor，仅对 member 生效），鉴权封装在 `core/auth/team_permissions.py`。团队文件在沙箱侧有独立共享缓存 `team_cache_dir(team_id)`，同团队成员复用一份镜像。 +权限模型：`TeamMember.role`（owner/admin/member）+ `file_permission`（viewer/editor，仅对 member 生效），鉴权封装在 EE 专属的 `edition_ee/auth/team_permissions.py`。团队文件在沙箱侧有独立共享缓存 `team_cache_dir(team_id)`，同团队成员复用一份镜像。 ## 文件如何进入对话上下文 @@ -128,10 +128,11 @@ teams ─────────┬── team_members（role: owner/admin/memb | `src/backend/core/services/project_scope.py` | `ProjectScope`（沙箱路径作用域） | | `src/backend/api/routes/v1/myspace_folders.py` | 个人文件夹 API | | `src/backend/api/routes/v1/artifacts.py` | 资产列表 / 会话收藏 / 加入知识库 | -| `src/backend/api/routes/v1/team_files.py` | 团队文件夹与文件 API（商业版 EE） | +| `src/backend/edition_ee/routes/team_files.py` | 团队文件夹与文件 API（商业版 EE） | | `src/backend/api/routes/v1/file_upload.py` | 文件上传（可指定文件夹） | | `src/backend/core/db/models/project.py` | `Project` / `ProjectFavorite` ORM | -| `src/backend/core/db/models/identity.py` | `Team` / `TeamMember` / `TeamFolder` / `UserFolder` ORM | +| `src/backend/core/db/models/identity.py` | `UserFolder` 等共享身份 ORM | +| `src/backend/edition_ee/db/models/identity.py` | `Team` / `TeamMember` / `TeamFolder` ORM（仅 EE） | | `src/backend/core/db/models/artifact.py` | `Artifact` ORM | | `src/backend/core/llm/hooks.py` | 附件上下文注入（`_build_file_context` 等） | | `src/backend/core/llm/agent_factory.py` | 项目 section 注入 system prompt | diff --git a/install.sh b/install.sh index 5bf7f4bd..01de3ad1 100755 --- a/install.sh +++ b/install.sh @@ -343,6 +343,7 @@ info "Starting HugAgentOS" # The CE server seeds admin/admin on a fresh data directory and requires a # password change immediately after sign-in. Model providers are configured in # Settings, so the one-command path does not require an interactive wizard. +export HUGAGENT_BOOTSTRAP_DEFAULT_PLUGINS=1 if [[ -t 1 && -r /dev/tty ]]; then exec "${HUGAGENT_BIN}" serve dict: """Registry hit → allOf merge; otherwise fall back to a plain envelope ref.""" - data_schema = DATA_SCHEMAS.get((method_upper, path)) + data_schema = EDITION_DATA_SCHEMAS.get( + (method_upper, path), DATA_SCHEMAS.get((method_upper, path)) + ) if data_schema is None: return dict(envelope_ref) return { @@ -301,10 +304,9 @@ async def custom_redoc_html(): # --------------------------------------------------------------------------- # Registration order is the reverse of execution order (later registrations sit -# further out and run first). license_gate is registered first → innermost, so -# CORS wraps around it and the 402 returned in the disabled state still carries -# CORS headers, letting the browser read it. -setup_license_gate(app) +# further out and run first). Edition middleware is innermost so CORS can still +# decorate any edition-specific rejection response. +setup_edition_middleware(app) setup_cors(app) setup_logging_middleware(app) setup_error_handlers(app) @@ -354,19 +356,16 @@ async def root(): # Public site hosting (/site/{slug}/…): nginx location /site/ reverse-proxies here verbatim app.include_router(sites_serve_router) -# V1 API routers — edition registry (seam C1): CE first, then EE. -# When an EE module is physically absent from the CE derived tree, -# iter_edition_routers skips it automatically; in the main repo both tables are -# complete, so the registered set is equivalent to line-by-line includes. -# EE routes get a license guard per the feature flag in the table (M4 second -# line of defense); in internal deployments (no license file) the guard passes -# everything through, matching historical behavior. +# V1 API routers — community-capable routes first, then edition extensions. +# The CE registry contains no extension entries and the derived tree does not +# carry their modules. The full repository attaches edition policy dependencies +# while registering extension routers. for _name, _router, _ in iter_edition_routers(CE_ROUTERS): app.include_router(_router) for _name, _router, _feature in iter_edition_routers(EE_ROUTERS): app.include_router( _router, - dependencies=[requires_feature(Feature(_feature))] if _feature else None, + dependencies=edition_router_dependencies(_feature), ) # Unified login entry points: /login + /register (always on, coexisting with mock-sso) @@ -609,7 +608,7 @@ async def _startup_seed_roles(): """ try: from core.db.engine import SessionLocal - from core.services.role_service import seed_default_roles + from core.services.edition_startup import seed_default_roles db = SessionLocal() try: @@ -623,13 +622,21 @@ async def _startup_seed_roles(): async def _startup_local_sidecars(): - """Spawn MCP + script_runner sidecars in local profile (no-op otherwise).""" + """Spawn and verify MCP + script_runner sidecars in local profile. + + A local desktop process must not pass its health check while the MCP tools + promised by the default plugins are absent. Compose deployments are a + no-op inside ``start_local_sidecars`` and keep their existing best-effort + startup behavior. + """ try: from orchestration.local_subprocess import start_local_sidecars await start_local_sidecars() except Exception as exc: - logger.warning("[startup] local sidecars spawn failed: %s", exc) + logger.error("[startup] local sidecars failed readiness: %s", exc) + if settings.deploy.is_local: + raise async def _shutdown_local_sidecars(): @@ -653,10 +660,16 @@ async def _startup_seed_mcp_servers(): """ try: from core.db.engine import SessionLocal - from core.services.mcp_service import seed_builtin_mcp_servers_if_empty + from core.services.mcp_service import ( + prune_removed_builtin_mcp_servers, + seed_builtin_mcp_servers_if_empty, + ) db = SessionLocal() try: + pruned = prune_removed_builtin_mcp_servers(db) + if pruned: + logger.info("[startup] unavailable built-in MCP rows pruned: %s", ", ".join(pruned)) seeded = seed_builtin_mcp_servers_if_empty(db) if seeded: logger.info("[startup] built-in MCP catalog seeded: %s", ", ".join(seeded)) @@ -666,6 +679,39 @@ async def _startup_seed_mcp_servers(): logger.warning("[startup] MCP catalog seed failed: %s", exc) +async def _startup_seed_default_plugins(): + """Install the three credential-free plugins on first CE Compose boot. + + Local/desktop installs run the equivalent bootstrap in ``cli.py`` before + the API is imported. Compose needs a database-backed marker so a user who + later uninstalls one of the defaults does not have it resurrected after a + container restart. + """ + if settings.edition.edition != "ce" or settings.deploy.is_local: + return + + from core.db.engine import SessionLocal + from core.services.plugin_service import ( + DEFAULT_BOOTSTRAP_PLUGIN_SLUGS, + ensure_default_plugins_bootstrapped, + ) + + db = SessionLocal() + try: + if ensure_default_plugins_bootstrapped(db): + logger.info( + "[startup] default plugins bootstrapped: %s", + ", ".join(DEFAULT_BOOTSTRAP_PLUGIN_SLUGS), + ) + except Exception as exc: + logger.error("[startup] default plugin bootstrap failed: %s", exc) + # These plugins are part of the advertised CE baseline. Do not report a + # healthy web deployment with only a partial/default-missing toolset. + raise + finally: + db.close() + + async def _startup_recover_datasource_sidecars(): """Recover configured DBHub/ES MCP sidecars without blocking API startup.""" import asyncio diff --git a/src/backend/api/deps.py b/src/backend/api/deps.py index 5bae6d32..1457ad17 100644 --- a/src/backend/api/deps.py +++ b/src/backend/api/deps.py @@ -1,16 +1,13 @@ """Shared authentication dependencies.""" import os -from dataclasses import dataclass from typing import Optional -from core.auth.backend import UserContext, get_current_user, require_auth -from core.auth.permissions_iface import PermissionLevel, require_team_file_permission -from core.auth.roles import TeamRole, at_least +from core.auth.backend import UserContext, require_auth from core.config.settings import settings from core.db.engine import get_db from core.db.models import UserShadow -from core.db.repository import AuditLogRepository, TeamRepository +from core.db.repository import AuditLogRepository from fastapi import Depends, Header, HTTPException, Request from sqlalchemy.orm import Session @@ -353,102 +350,3 @@ async def require_super_admin( ) raise HTTPException(status_code=403, detail="需要 super_admin 权限") return user_id - - -def require_team_role(min_role: TeamRole = "member"): - """FastAPI dependency factory: require the current user to have >= min_role in the team given by the team_id path parameter.""" - - async def dependency( - team_id: str, - request: Request, - db: Session = Depends(get_db), - ) -> str: - user_id = _resolve_current_user_id(request) - if not user_id: - AuditLogRepository(db).log_denial( - user_id=None, - action="team_role.access_denied", - reason="not_logged_in", - required=min_role, - actual="anonymous", - resource_type="team", - resource_id=team_id, - request=request, - ) - raise HTTPException(status_code=401, detail="未登录") - if _is_super_admin(db, user_id): - return user_id - role = TeamRepository(db).get_member_role(team_id, user_id) - if role is None: - AuditLogRepository(db).log_denial( - user_id=user_id, - action="team_role.access_denied", - reason="not_a_member", - required=min_role, - actual="none", - resource_type="team", - resource_id=team_id, - request=request, - ) - raise HTTPException(status_code=403, detail="未加入该团队") - if not at_least(role, min_role): - AuditLogRepository(db).log_denial( - user_id=user_id, - action="team_role.access_denied", - reason="insufficient_role", - required=min_role, - actual=role, - resource_type="team", - resource_id=team_id, - request=request, - ) - raise HTTPException(status_code=403, detail=f"需要 {min_role} 以上角色") - return user_id - - return dependency - - -# ═══════════════════════════════════════════════════════════════════════ -# Team file permissions — unified dependency -# ═══════════════════════════════════════════════════════════════════════ - - -@dataclass -class TeamFileAccess: - """Access context produced by `require_team_file_perm()`. - - A route signature only needs a single - ``access: TeamFileAccess = Depends(require_team_file_perm("edit"))`` to get: - the resolved user / db / the validated permission level. - """ - - user: UserContext - db: Session - team_id: str - permission: PermissionLevel # the current user's actual permission (view/edit/admin) - - -def require_team_file_perm(min_permission: PermissionLevel, *, action: str = "team_file.access"): - """FastAPI dependency factory: verify the current user's file permission on the path ``team_id`` is >= ``min_permission``. - - When not satisfied, ``require_team_file_permission()`` is responsible for - raising 403/404 and writing the audit denial log. - """ - - async def dependency( - team_id: str, - request: Request, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), - ) -> TeamFileAccess: - perm = require_team_file_permission( - db, - str(user.user_id), - team_id, - min_permission, - request=request, - action=action, - ) - return TeamFileAccess(user=user, db=db, team_id=team_id, permission=perm) - - return dependency diff --git a/src/backend/api/middleware/edition.py b/src/backend/api/middleware/edition.py new file mode 100644 index 00000000..586fca44 --- /dev/null +++ b/src/backend/api/middleware/edition.py @@ -0,0 +1,30 @@ +"""Community-edition middleware and capability facade.""" + + +def setup_edition_middleware(app) -> None: + return None + + +def edition_router_dependencies(policy_key): + return None + + +def edition_probe_payload() -> dict: + return {"edition": "ce"} + + +def edition_only_route(route_decorator): + """Discard an enterprise route decorator without registering the handler.""" + + def _identity(handler): + return handler + + return _identity + + +__all__ = [ + "edition_probe_payload", + "edition_only_route", + "edition_router_dependencies", + "setup_edition_middleware", +] diff --git a/src/backend/api/middleware/license_gate.py b/src/backend/api/middleware/license_gate.py deleted file mode 100644 index 09ee0f5b..00000000 --- a/src/backend/api/middleware/license_gate.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Global license deactivation gate (the third line of defense for the commercial edition's "stop on expiry"). - -First line: the route registry (the CE tree physically contains no EE routes). -Second line: the ``requires_feature`` capability-bit guard (disables individual EE capabilities by entitlement). -This middleware is a **whole-product-level** gate: when the license is in :data:`DEAD_MODES` -(expired / invalid / missing), all requests except the allowlist are rejected with 402 — -even basic chat/data-fetching stops, delivering "the product stops the moment the license expires". - -Allowlist (still reachable in the deactivated state, otherwise self-service renewal would be impossible): -- health/liveness probes, root path, API docs -- ``/v1/config/license``: upload a new license to renew (CONFIG_TOKEN auth) -- ``/v1/meta``: the frontend renders the "deactivated" block page from this - -Decision goes through ``license_manager.is_active()``, and the result is cached by the license file's mtime, -so each request costs only one in-memory comparison and introduces no IO. ce / internal / licensed / grace -are always allowed — the community edition and internal/fully-hosted deployments are unaffected. -""" - -from __future__ import annotations - -import json - -from fastapi import FastAPI -from starlette.types import ASGIApp, Receive, Scope, Send - -from core.infra.responses import generate_trace_id -from core.licensing import license_manager - -# Path prefixes still allowed in the deactivated state (exact or prefix match). -_ALLOW_PREFIXES = ( - "/health", - "/ready", - "/live", - "/docs", - "/redoc", - "/openapi.json", - "/v1/config/license", - "/v1/meta", - # Desktop client auto-update manifest / installer distribution: public, no user data, allowed - # even when the license is invalid, so customers can pull a fixed client (otherwise - # "expired -> update -> still expired" deadlocks). - "/v1/desktop", -) - -_INACTIVE_CODE = 40203 -_INACTIVE_MESSAGE = "license 已失效或过期，产品已停用，请联系厂商续期后重新激活。" - - -def _allowed(path: str) -> bool: - if path == "/": - return True - return any(path == p or path.startswith(p + "/") for p in _ALLOW_PREFIXES) - - -class LicenseGateMiddleware: - """Pure ASGI middleware — gates business requests in the deactivated state, without wrapping the response body (does not break SSE).""" - - def __init__(self, app: ASGIApp) -> None: - self.app = app - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if scope["type"] != "http": - await self.app(scope, receive, send) - return - - method = scope.get("method", "GET") - path = scope.get("path", "") - - # CORS preflight and allowlist paths are always allowed; when the license is healthy, pass through directly. - if method == "OPTIONS" or _allowed(path) or license_manager.is_active(): - await self.app(scope, receive, send) - return - - body = json.dumps({ - "code": _INACTIVE_CODE, - "message": _INACTIVE_MESSAGE, - "data": {"mode": license_manager.mode()}, - "trace_id": generate_trace_id(), - }).encode("utf-8") - await send({ - "type": "http.response.start", - "status": 402, - "headers": [ - (b"content-type", b"application/json; charset=utf-8"), - (b"content-length", str(len(body)).encode()), - ], - }) - await send({"type": "http.response.body", "body": body}) - - -def setup_license_gate(app: FastAPI) -> None: - app.add_middleware(LicenseGateMiddleware) diff --git a/src/backend/api/openapi_data_schemas.py b/src/backend/api/openapi_data_schemas.py index 68449998..5feb3ef4 100644 --- a/src/backend/api/openapi_data_schemas.py +++ b/src/backend/api/openapi_data_schemas.py @@ -279,44 +279,12 @@ "nickname": {"type": "string"}, "real_name": {"type": "string"}, "department": {"type": "string"}, - "teams": {"type": "array", "items": {"type": "object", "additionalProperties": True}}, "expires_at": {"type": "string", "format": "date-time"}, "sso_token": {"type": "string"}, "allowed_apps": {"type": "array", "items": {"type": "string"}}, "lab_enabled": {"type": "boolean"}, }, }, - "TeamBrief": { - "type": "object", - "properties": { - "team_id": {"type": "string"}, - "name": {"type": "string"}, - "description": {"type": "string"}, - "member_count": {"type": "integer"}, - "role": {"type": "string"}, - "avatar": {"type": "string"}, - }, - }, - "TeamMemberItem": { - "type": "object", - "properties": { - "user_id": {"type": "string"}, - "username": {"type": "string"}, - "avatar_url": {"type": "string"}, - "role": {"type": "string"}, - "joined_at": {"type": "string", "format": "date-time"}, - "is_self": {"type": "boolean"}, - }, - }, - "UserSearchResult": { - "type": "object", - "properties": { - "user_id": {"type": "string"}, - "username": {"type": "string"}, - "real_name": {"type": "string"}, - "avatar_url": {"type": "string"}, - }, - }, "CurrentUserInfo": { "type": "object", "properties": { @@ -331,7 +299,6 @@ "phone": {"type": "string"}, "department": {"type": "string"}, "auth_source": {"type": "string", "enum": ["local", "external"]}, - "teams": {"type": "array", "items": {"type": "object", "additionalProperties": True}}, "created_at": {"type": "string", "format": "date-time"}, }, }, @@ -399,7 +366,10 @@ "type": "object", "properties": { "enabled": {"type": "boolean"}, - "relations": {"type": "array", "items": {"type": "object", "additionalProperties": True}}, + "relations": { + "type": "array", + "items": {"type": "object", "additionalProperties": True}, + }, "count": {"type": "integer"}, }, }, @@ -455,6 +425,7 @@ # 2. Endpoint registry: (METHOD, PATH) → data field schema # --------------------------------------------------------------------------- + # Shorthand for writing $ref def _ref(name: str) -> Dict[str, Any]: return {"$ref": f"#/components/schemas/{name}"} @@ -506,7 +477,10 @@ def _ref(name: str) -> Dict[str, Any]: "is_markdown": {"type": "boolean"}, "route": {"type": "string"}, "sources": {"type": "array", "items": {"type": "object", "additionalProperties": True}}, - "artifacts": {"type": "array", "items": {"type": "object", "additionalProperties": True}}, + "artifacts": { + "type": "array", + "items": {"type": "object", "additionalProperties": True}, + }, "warnings": {"type": "array", "items": {"type": "string"}}, }, }, @@ -683,47 +657,9 @@ def _ref(name: str) -> Dict[str, Any]: "type": "object", "properties": {"login_url": {"type": "string"}}, }, - # ===== Me / Teams ===== + # ===== Current user ===== ("GET", "/v1/me"): _ref("CurrentUserInfo"), ("PATCH", "/v1/me"): _ref("CurrentUserInfo"), - ("GET", "/v1/me/teams"): { - "type": "object", - "properties": { - "items": {"type": "array", "items": _ref("TeamBrief")}, - "total": {"type": "integer"}, - }, - }, - ("GET", "/v1/me/teams/{team_id}"): _ref("TeamBrief"), - ("GET", "/v1/me/teams/{team_id}/members"): { - "type": "object", - "properties": { - "items": {"type": "array", "items": _ref("TeamMemberItem")}, - "my_role": {"type": "string"}, - }, - }, - ("POST", "/v1/me/teams/{team_id}/members"): { - "type": "object", - "properties": { - "team_id": {"type": "string"}, - "user_id": {"type": "string"}, - "username": {"type": "string"}, - "role": {"type": "string"}, - }, - }, - ("DELETE", "/v1/me/teams/{team_id}/members/{member_user_id}"): { - "type": "object", - "properties": { - "team_id": {"type": "string"}, - "user_id": {"type": "string"}, - "self_leave": {"type": "boolean"}, - }, - }, - ("GET", "/v1/me/users/search"): { - "type": "object", - "properties": { - "items": {"type": "array", "items": _ref("UserSearchResult")}, - }, - }, # ===== User preferences ===== ("GET", "/v1/users/{user_id}/preferences"): _ref("UserPreferencesResponse"), ("PUT", "/v1/users/{user_id}/preferences"): {"type": "null"}, diff --git a/src/backend/api/openapi_edition_schemas.py b/src/backend/api/openapi_edition_schemas.py new file mode 100644 index 00000000..17b2e377 --- /dev/null +++ b/src/backend/api/openapi_edition_schemas.py @@ -0,0 +1,6 @@ +"""Community Edition has no organization-only OpenAPI contracts.""" + +EDITION_DATA_COMPONENTS = {} +EDITION_DATA_SCHEMAS = {} + +__all__ = ["EDITION_DATA_COMPONENTS", "EDITION_DATA_SCHEMAS"] diff --git a/src/backend/api/routes/files.py b/src/backend/api/routes/files.py index 89d9fb91..6c6a6e81 100644 --- a/src/backend/api/routes/files.py +++ b/src/backend/api/routes/files.py @@ -8,18 +8,17 @@ from pathlib import Path from typing import Any, Optional -from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query -from fastapi.responses import FileResponse, JSONResponse -from sqlalchemy.orm import Session - from core.artifacts.store import get_artifact from core.auth.backend import UserContext, require_auth -from core.auth.permissions_iface import resolve_artifact_access from core.content.office import find_libreoffice_binary from core.db.engine import get_db from core.db.repository import AuditLogRepository from core.infra.exceptions import StorageError +from core.services.artifact_edition import artifact_access_metadata, can_access_artifact_metadata from core.storage import get_storage +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query +from fastapi.responses import FileResponse, JSONResponse +from sqlalchemy.orm import Session logger = logging.getLogger(__name__) @@ -33,15 +32,13 @@ def _load_artifact_item(file_id: str, db: Session) -> dict[str, Any]: """Resolve an artifact from DB first, then local store. - DB is authoritative — it reflects moves between personal ↔ team folders, - which rewrite ``storage_key``. The local index is only used as a fallback - for tool-generated artifacts that never hit the DB. + DB is authoritative because edition-specific moves can rewrite + ``storage_key``. The local index is only used as a fallback for + tool-generated artifacts that never hit the DB. """ from core.db.models import Artifact as ArtifactModel - artifact_obj = db.query(ArtifactModel).filter( - ArtifactModel.artifact_id == file_id - ).first() + artifact_obj = db.query(ArtifactModel).filter(ArtifactModel.artifact_id == file_id).first() if artifact_obj is not None: return { "path": None, @@ -51,8 +48,7 @@ def _load_artifact_item(file_id: str, db: Session) -> dict[str, Any]: "storage_key": artifact_obj.storage_key, "metadata": { "from_database": True, - "user_id": artifact_obj.user_id, - "team_id": artifact_obj.team_id, + **artifact_access_metadata(artifact_obj), }, } @@ -75,14 +71,16 @@ def _record_audit( if not user: return try: - AuditLogRepository(db).create({ - "user_id": user.user_id, - "action": action, - "resource_type": "artifact", - "resource_id": file_id, - "status": status, - "details": details or {}, - }) + AuditLogRepository(db).create( + { + "user_id": user.user_id, + "action": action, + "resource_type": "artifact", + "resource_id": file_id, + "status": status, + "details": details or {}, + } + ) except Exception as exc: logger.warning("Failed to create audit log for %s: %s", action, exc) @@ -96,19 +94,10 @@ def _authorize_access( denied_action: str, ) -> None: metadata = item.get("metadata") or {} - owner_id = metadata.get("user_id") - team_id = metadata.get("team_id") if not user: return - - if owner_id or team_id: - # The owner ∪ team combined-permission single point is in permissions_iface (the - # owner can always access their own files — including team_id-tagged files in - # CE / left-team scenarios; team members with view+ can access) - if resolve_artifact_access(db, str(user.user_id), owner_id, team_id) != "none": - return - else: - return # legacy data: old files without owner/team metadata remain allowed + if can_access_artifact_metadata(db, str(user.user_id), metadata): + return _record_audit( user=user, @@ -179,7 +168,9 @@ def _is_powerpoint_file(item: dict[str, Any]) -> bool: name = str(item.get("name", "")) mime_type = str(item.get("mime_type", "")).lower() ext = Path(name).suffix.lower() - return ext in POWERPOINT_EXTENSIONS or any(marker in mime_type for marker in POWERPOINT_MIME_MARKERS) + return ext in POWERPOINT_EXTENSIONS or any( + marker in mime_type for marker in POWERPOINT_MIME_MARKERS + ) def _is_word_file(item: dict[str, Any]) -> bool: @@ -270,7 +261,9 @@ async def download_file( file_id: str, background_tasks: BackgroundTasks, mode: str = Query("direct", description="Download mode: direct or presigned"), - inline: bool = Query(False, description="If true, serve for inline display (Content-Disposition: inline)"), + inline: bool = Query( + False, description="If true, serve for inline display (Content-Disposition: inline)" + ), user: Optional[UserContext] = Depends(require_auth(required=False)), db: Session = Depends(get_db), ): @@ -351,20 +344,24 @@ async def download_file( except StorageError as exc: logger.error("Failed to generate presigned URL: %s", exc) - return JSONResponse({ + return JSONResponse( + { + "url": f"/files/{file_id}", + "expires_in": 900, + "filename": str(item.get("name", file_id)), + "note": "Failed to generate presigned URL, using direct download URL", + } + ) + else: + # Local artifact, return direct download URL + return JSONResponse( + { "url": f"/files/{file_id}", "expires_in": 900, "filename": str(item.get("name", file_id)), - "note": "Failed to generate presigned URL, using direct download URL" - }) - else: - # Local artifact, return direct download URL - return JSONResponse({ - "url": f"/files/{file_id}", - "expires_in": 900, - "filename": str(item.get("name", file_id)), - "note": "Local artifact, using direct download URL" - }) + "note": "Local artifact, using direct download URL", + } + ) else: return _build_direct_download_response( item=item, diff --git a/src/backend/api/routes/v1/__init__.py b/src/backend/api/routes/v1/__init__.py index 6413d3fc..6f5d3723 100644 --- a/src/backend/api/routes/v1/__init__.py +++ b/src/backend/api/routes/v1/__init__.py @@ -1,23 +1,9 @@ -"""API v1 routes — Edition router registry (CE/EE split seam C1). +"""Community-edition route registry.""" -This file is the **single source of truth shared by both the CE and EE trees**: -`api.app` registers routers from these tables, CE first then EE. -Entries are ``(module name, router attribute)`` or -``(module name, router attribute, license feature bit)``; an EE entry whose -feature bit is None is **explicitly exempt** from the feature guard. When a -module is missing (the CE derived tree physically deletes EE files), -``iter_edition_routers`` silently skips it — so this file goes into the CE -tree as-is, no overlay copy needed. +from importlib import import_module -Route modules no longer do named eager re-exports (no consumers repo-wide); -when a single module is needed, import it directly via -``from api.routes.v1 import chats``. -Table order preserves the historical include order (relative order within a -prefix family is immutable: the public config read must come before the -config_* admin consoles). -""" - -from .mock_sso import router as mock_sso_router, login_router +from .mock_sso import login_router +from .mock_sso import router as mock_sso_router CE_ROUTERS: tuple[tuple[str, str], ...] = ( ("chats", "router"), @@ -40,11 +26,6 @@ ("loops", "router"), ("automations", "router"), ("chat_runs", "router"), - ("me", "router"), - # Personal system settings / personal logs (CE hand-down: model access goes - # through the models.py gate swap; service configs and the user's own call - # logs go through these two routes. EE registers them too, but the frontend - # only shows the entry points on CE.) ("me_system", "router"), ("me_logs", "router"), ("myspace_folders", "router"), @@ -52,102 +33,32 @@ ("internal_batch", "router"), ("internal_sites", "router"), ("projects", "router"), - ("api_keys", "router"), ("me_capabilities", "router"), ("marketplace", "router"), ("agent_marketplace", "router"), ("plugins", "router"), ("integrations", "router"), ("channels", "router"), - ("lab_skill_distill", "router"), ("meta", "router"), ("sites", "router"), ("desktop", "router"), ) -# EE routers + license feature bits (M4 second line of defense; the first is -# that the CE tree physically does not contain these files). -# The three entries with feature bit None are explicit exemptions: config_verify -# is the console login check, config_license is the entry point for swapping the -# license, and auth is login/session infrastructure (session/check, logout, -# ticket exchange for local mock login) — these must stay reachable even when -# the license is invalid, otherwise users get stuck in a -# "402 → logout → login → 402" loop with no way to replace the license. -# SSO-specific features guard themselves: authorize-url attaches -# requires_feature on the auth route; the remote ticket exchange is checked -# inside the remote branch of core/auth/sso.exchange_ticket (mock/remote fork -# there — a route-level check keyed on login_mode would be bypassed by legacy -# configs). -EE_ROUTERS: tuple[tuple[str, str, str | None], ...] = ( - ("audit", "router", "audit"), - ("admin_skills", "router", "content_admin"), - ("admin_kb", "router", "content_admin"), - ("admin_prompts", "router", "content_admin"), - ("admin_mcp_servers", "router", "content_admin"), - ("admin_agents", "router", "content_admin"), - ("config_verify", "router", None), - ("admin_usage_logs", "router", "billing"), - ("admin_billing", "router", "billing"), - ("admin_chat_history", "router", "audit"), - ("auth", "router", None), - ("admin_logs", "router", "audit"), - ("admin_skill_drafts", "router", "content_admin"), - ("admin_sandbox", "router", "content_admin"), - ("config_users", "router", "multi_tenancy"), - ("config_user_distill", "router", "content_admin"), - ("config_teams", "router", "multi_tenancy"), - ("config_roles", "router", "multi_tenancy"), - ("config_invites", "router", "multi_tenancy"), - ("config_security", "router", "system_config"), - # Internal callback surface of the security management plugin - # (security-manager): security_ops MCP → double-gated read-only log queries - ("internal_security", "router", "audit"), - # All endpoints are CONFIG_TOKEN admin operations (including writes + - # connectivity tests), no public reads → the whole module belongs to EE - ("service_configs", "router", "system_config"), - # "Database tools" data source management (create/update/delete applies - # immediately: render dbhub.toml + restart sidecar + tool wiring) - ("data_sources", "router", "system_config"), - # "Metadata governance": table/column semantics + enum dictionaries + - # golden SQL (improves accuracy of direct-to-DB data retrieval) - ("db_metadata", "router", "system_config"), - ("team_files", "router", "multi_tenancy"), - ("admin_marketplace", "router", "content_admin"), - ("admin_agent_marketplace", "router", "content_admin"), - ("admin_plugins", "router", "content_admin"), - # Marketplace visibility scope: brief lists of subjects (users/teams/roles), - # data source for the admin console's visibility scope picker - ("admin_visibility", "router", "content_admin"), - ("config_license", "router", None), - # External model gateway control plane (LiteLLM Proxy): issue/revoke - # virtual keys, read usage - ("gateway_admin", "router", "model_gateway"), - # External gateway Anthropic-protocol data endpoints (self-built translation - # layer → litellm OpenAI upstream): public endpoints (virtual keys are - # validated by litellm, not CONFIG_TOKEN), for Claude Code / Cherry Studio - # agent access - ("gateway_anthropic", "router", "model_gateway"), -) - +EE_ROUTERS: tuple = () -def iter_edition_routers(specs): - """Yield (module name, router, feature bit|None) per registry entry; skip missing modules.""" - from importlib import import_module - for spec in specs: - module_name, attr = spec[0], spec[1] - feature = spec[2] if len(spec) > 2 else None - try: - module = import_module(f"{__name__}.{module_name}") - except ModuleNotFoundError: - continue +def iter_edition_routers(entries): + for entry in entries: + module_name, attr, *feature_items = entry + module = import_module(f"{__name__}.{module_name}") + feature = feature_items[0] if feature_items else None yield module_name, getattr(module, attr), feature __all__ = [ - "mock_sso_router", - "login_router", "CE_ROUTERS", "EE_ROUTERS", "iter_edition_routers", + "login_router", + "mock_sso_router", ] diff --git a/src/backend/api/routes/v1/agents.py b/src/backend/api/routes/v1/agents.py index 825b3b12..b4702612 100644 --- a/src/backend/api/routes/v1/agents.py +++ b/src/backend/api/routes/v1/agents.py @@ -1,7 +1,4 @@ -"""User-facing sub-agent API routes. - -Provides CRUD for user-owned agents and read access to admin agents. -""" +"""Personal sub-agent API for the community edition.""" from __future__ import annotations @@ -23,19 +20,14 @@ def _require_can_add_agent(user_id: str, db: Session) -> None: - # personal explicit (user management) → team default (team management) → off by default if not resolve_user_capabilities(db, user_id)["can_add_agent"]: raise AccessDeniedError( - message="管理员未开放自建/安装子智能体功能", reason="can_add_agent_disabled" + message="管理员未开放自建/安装子智能体功能", + reason="can_add_agent_disabled", ) -# ── Pydantic schemas ───────────────────────────────────────────────────────── - - class AgentCreateRequest(BaseModel): - # passing team_id = create a team sub-agent (requires being owner/admin of that team); omitting = personal sub-agent - team_id: Optional[str] = None name: str = Field(..., min_length=1, max_length=255) avatar: Optional[str] = None description: Optional[str] = Field("", max_length=20) @@ -76,18 +68,12 @@ class AgentUpdateRequest(BaseModel): extra_config: Optional[Dict[str, Any]] = None -# ── Endpoints ───────────────────────────────────────────────────────────────── - - @router.get("", summary="列出当前用户可见的所有子智能体") async def list_agents( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - """列出当前用户可见的所有子智能体（含本人创建的与管理员发布的）。需登录。""" - svc = UserAgentService(db) - agents = svc.list_for_user(user.user_id) - return success_response(data=agents) + return success_response(data=UserAgentService(db).list_for_user(user.user_id)) @router.get("/available-resources", summary="可绑定到子智能体的资源列表") @@ -95,10 +81,8 @@ async def available_resources( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - """列出可绑定到子智能体的资源（MCP 工具、技能、知识库等），供创建/编辑时选择。需登录。""" - svc = UserAgentService(db) - resources = svc.list_available_resources(owner_user_id=str(user.user_id)) - return success_response(data=resources) + data = UserAgentService(db).list_available_resources(owner_user_id=str(user.user_id)) + return success_response(data=data) @router.get("/{agent_id}", summary="子智能体详情") @@ -107,10 +91,8 @@ async def get_agent( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - """获取指定子智能体的详情。不存在返回 404，无权访问返回 403。需登录。""" - svc = UserAgentService(db) try: - agent = svc.get_by_id(agent_id, user_id=user.user_id) + agent = UserAgentService(db).get_by_id(agent_id, user_id=user.user_id) except LookupError: return error_response(code=404, message="Agent not found") except PermissionError: @@ -118,33 +100,19 @@ async def get_agent( return success_response(data=agent) -@router.post("", summary="创建子智能体（个人 / 团队）") +@router.post("", summary="创建个人子智能体") async def create_agent( body: AgentCreateRequest, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - """创建子智能体（名称、提示词、绑定的工具/技能/知识库等）。 - - - 不传 ``team_id`` → 个人子智能体（owner_type=user，仅本人可见）。 - - 传 ``team_id`` → 团队子智能体（owner_type=team，对团队成员可见可用），仅该团队 - owner/admin 可创建，否则返回 403。参数校验失败返回 400。需登录。 - - 需 ``can_add_agent`` 权限（与技能/MCP 自助一致，由 Config 后管按用户/团队开放）。 - 团队子智能体在此之上仍需该团队 owner/admin 身份。 - """ _require_can_add_agent(str(user.user_id), db) - svc = UserAgentService(db) - data = body.model_dump(exclude_none=True) - team_id = data.pop("team_id", None) - owner_type = "team" if team_id else "user" try: - agent = svc.create( + agent = UserAgentService(db).create( user_id=user.user_id, operator_name=user.username, - owner_type=owner_type, - data=data, - team_id=team_id, + owner_type="user", + data=body.model_dump(exclude_none=True), ) except PermissionError as exc: return error_response(code=403, message=str(exc)) @@ -153,23 +121,20 @@ async def create_agent( return success_response(data=agent) -@router.put("/{agent_id}", summary="更新用户子智能体") +@router.put("/{agent_id}", summary="更新个人子智能体") async def update_agent( agent_id: str, body: AgentUpdateRequest, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - """更新指定子智能体（仅传入字段被修改）。不存在 404，无权修改 403。需登录。""" - svc = UserAgentService(db) - data = body.model_dump(exclude_none=True) try: - agent = svc.update( + agent = UserAgentService(db).update( agent_id, user_id=user.user_id, operator_name=user.username, owner_type="user", - data=data, + data=body.model_dump(exclude_none=True), ) except LookupError: return error_response(code=404, message="Agent not found") @@ -178,18 +143,19 @@ async def update_agent( return success_response(data=agent) -@router.delete("/{agent_id}", summary="删除用户子智能体") +@router.delete("/{agent_id}", summary="删除个人子智能体") async def delete_agent( agent_id: str, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - """删除指定子智能体。不存在 404，无权删除 403。需登录。""" - svc = UserAgentService(db) try: - svc.delete(agent_id, user_id=user.user_id, owner_type="user") + UserAgentService(db).delete(agent_id, user_id=user.user_id, owner_type="user") except LookupError: return error_response(code=404, message="Agent not found") except PermissionError: return error_response(code=403, message="Access denied") return success_response(data={"deleted": True}) + + +__all__ = ["AgentCreateRequest", "AgentUpdateRequest", "router"] diff --git a/src/backend/api/routes/v1/api_keys.py b/src/backend/api/routes/v1/api_keys.py deleted file mode 100644 index f9142b6e..00000000 --- a/src/backend/api/routes/v1/api_keys.py +++ /dev/null @@ -1,214 +0,0 @@ -"""Per-user API-Key management API. - -GET /v1/me/api-keys List the current user's API-Keys -POST /v1/me/api-keys Create (plaintext returned only this once) -GET /v1/me/api-keys/{key_id}/reveal Retrieve the full plaintext again (decrypt key_enc, for copying) -PATCH /v1/me/api-keys/{key_id} Enable / disable -DELETE /v1/me/api-keys/{key_id} Revoke - -Available only when the user's capability bit ``can_use_api_key=true``, otherwise 403. This -switch is controlled by the user-management module of the Config admin platform. -""" - -from __future__ import annotations - -from typing import Optional - -from fastapi import APIRouter, Depends, status -from pydantic import BaseModel, Field -from sqlalchemy.orm import Session - -from core.auth.backend import get_current_user, UserContext -from core.auth.capabilities import resolve_user_capabilities -from core.db.engine import get_db -from core.infra.exceptions import AccessDeniedError, ResourceNotFoundError -from core.infra.responses import success_response, created_response -from core.licensing import Feature, license_manager -from core.services import ApiKeyService - -router = APIRouter(prefix="/v1/me/api-keys", tags=["API Keys"]) - -# Expiry options (days). None means never expires. The frontend dropdown renders based on this. -ALLOWED_EXPIRY_DAYS = {7, 30, 90, 180, 365} - - -class CreateApiKeyRequest(BaseModel): - name: str = Field("API Key", max_length=128, description="便于识别的名称") - expires_in_days: Optional[int] = Field( - None, description="过期天数，留空=永不过期；允许 7/30/90/180/365" - ) - for_gateway: bool = Field( - False, description="同时把此密钥用于对外模型网关（Cherry Studio 等可直接用它调用）" - ) - - -class ToggleApiKeyRequest(BaseModel): - enabled: bool - - -def _require_api_key_permission(user_id: str, db: Session) -> None: - """Check whether the user is allowed to use API-Keys: personal explicit → team default → off by default.""" - if not resolve_user_capabilities(db, user_id)["can_use_api_key"]: - raise AccessDeniedError( - message="管理员未开放 API-Key 功能", - reason="api_key_disabled", - ) - - -def _gateway_service_or_none(db: Session): - """Return the EE gateway service only when the feature is enabled.""" - if not license_manager.has(Feature.MODEL_GATEWAY): - return None - from core.services.litellm_gateway_service import LiteLLMGatewayService - - return LiteLLMGatewayService(db) - - -def _key_to_dict(row, *, plaintext: Optional[str] = None) -> dict: - return { - "id": row.id, - "name": row.name, - "key_prefix": row.key_prefix, - "enabled": row.enabled, - "expires_at": row.expires_at.isoformat() if row.expires_at else None, - "last_used_at": row.last_used_at.isoformat() if row.last_used_at else None, - "created_at": row.created_at.isoformat() if row.created_at else None, - # Whether the plaintext can be retrieved again (yes if ciphertext exists; old keys without ciphertext cannot) — the frontend uses this to decide whether to show "Copy" - "revealable": bool(getattr(row, "key_enc", None)), - # Plaintext is returned only on create/reveal; always None in listings - "api_key": plaintext, - } - - -@router.get("", summary="API-Key 列表") -async def list_api_keys( - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """列出当前用户全部未撤销的 API-Key（不含明文）。""" - _require_api_key_permission(str(user.user_id), db) - rows = ApiKeyService(db).list_keys(str(user.user_id)) - return success_response(data={"items": [_key_to_dict(r) for r in rows]}) - - -@router.post("", status_code=status.HTTP_201_CREATED, summary="新建 API-Key") -async def create_api_key( - body: CreateApiKeyRequest, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """生成一个新的 API-Key。明文 ``api_key`` 仅在本次响应返回，请妥善保存。""" - _require_api_key_permission(str(user.user_id), db) - - expires_in_days = body.expires_in_days - if expires_in_days is not None and expires_in_days not in ALLOWED_EXPIRY_DAYS: - from core.infra.exceptions import BadRequestError - - raise BadRequestError( - message="无效的过期天数", - data={"allowed": sorted(ALLOWED_EXPIRY_DAYS)}, - ) - - gw = None - if body.for_gateway: - from core.infra.exceptions import BadRequestError - - license_manager.require(Feature.MODEL_GATEWAY) - gw = _gateway_service_or_none(db) - if gw is None: - raise BadRequestError(message="对外模型网关未授权，无法将密钥用于网关") - if not gw.is_configured(): - raise BadRequestError(message="对外模型网关未启用，无法将密钥用于网关") - - row, raw = ApiKeyService(db).create_key( - user_id=str(user.user_id), - name=body.name, - expires_in_days=expires_in_days, - ) - - # Optional: also register this key into the outbound model gateway (plaintext is only available at this moment, so it can only be included at creation time). - if gw is not None: - await gw.register_user_key( - raw, user_api_key_id=row.id, - owner=getattr(user, "username", None) or str(user.user_id), - display_name=f"用户密钥：{row.name}", - ) - - data = _key_to_dict(row, plaintext=raw) - data["for_gateway"] = bool(body.for_gateway) - return created_response(data=data) - - -@router.get("/{key_id}/reveal", summary="再次取回 API-Key 明文") -async def reveal_api_key( - key_id: str, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """解密并返回某个 API-Key 的完整明文，供前端「再次复制」。 - - 旧密钥（本能力上线前创建、无密文）无法找回明文，返回 410 提示用户新建。 - """ - _require_api_key_permission(str(user.user_id), db) - row = ApiKeyService(db).get_key(str(user.user_id), key_id) - if not row: - raise ResourceNotFoundError("api_key", key_id) - - if not row.key_enc: - from core.infra.exceptions import BadRequestError - - raise BadRequestError( - message="该密钥创建于「再次复制」功能上线前，明文无法找回，请撤销后新建", - data={"reason": "api_key_not_revealable"}, - ) - - from core.infra.crypto import decrypt_secret - - plaintext = decrypt_secret(row.key_enc) - if not plaintext: - # Ciphertext exists but cannot be decrypted (usually a deployment key change) — treat as unrecoverable - from core.infra.exceptions import BadRequestError - - raise BadRequestError( - message="密钥明文解密失败（部署密钥可能已变更），请撤销后新建", - data={"reason": "api_key_decrypt_failed"}, - ) - - return success_response(data=_key_to_dict(row, plaintext=plaintext)) - - -@router.patch("/{key_id}", summary="启用/禁用 API-Key") -async def toggle_api_key( - key_id: str, - body: ToggleApiKeyRequest, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """切换某个 API-Key 的启用状态。""" - _require_api_key_permission(str(user.user_id), db) - row = ApiKeyService(db).set_enabled(str(user.user_id), key_id, body.enabled) - if not row: - raise ResourceNotFoundError("api_key", key_id) - # If this key is registered in the gateway, cascade block/unblock (no-op if not registered) - gw = _gateway_service_or_none(db) - if gw is not None: - await gw.set_user_key_active(key_id, body.enabled) - return success_response(data=_key_to_dict(row)) - - -@router.delete("/{key_id}", summary="撤销 API-Key") -async def revoke_api_key( - key_id: str, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """撤销（软删除）某个 API-Key，撤销后立即失效且不可恢复。""" - _require_api_key_permission(str(user.user_id), db) - ok = ApiKeyService(db).revoke_key(str(user.user_id), key_id) - if not ok: - raise ResourceNotFoundError("api_key", key_id) - # Cascade-delete the gateway-side mirror (no-op if not registered) - gw = _gateway_service_or_none(db) - if gw is not None: - await gw.unregister_user_key(key_id) - return success_response(data={"id": key_id, "revoked": True}) diff --git a/src/backend/api/routes/v1/artifacts.py b/src/backend/api/routes/v1/artifacts.py index aefc089c..38f59389 100644 --- a/src/backend/api/routes/v1/artifacts.py +++ b/src/backend/api/routes/v1/artifacts.py @@ -11,20 +11,25 @@ import threading from typing import Any, Dict, List, Optional -from fastapi import APIRouter, BackgroundTasks, Depends, Query -from pydantic import BaseModel -from sqlalchemy import desc, func -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Session - -from core.auth.backend import get_current_user, UserContext +from core.auth.backend import UserContext, get_current_user from core.content.kb_processing import vectorise_document_background from core.db.engine import get_db from core.db.models import Artifact, ChatMessage, ChatSession, KBDocument, KBSpace from core.db.repository import ArtifactRepository -from core.infra.responses import success_response, error_response +from core.infra.responses import error_response, success_response from core.services import KBService +from core.services.artifact_edition import ( + artifact_list_scope, + artifact_scope_fields, + can_access_artifact, + extend_artifact_item, +) from core.storage import get_storage +from fastapi import APIRouter, BackgroundTasks, Depends, Query +from pydantic import BaseModel +from sqlalchemy import desc, func +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session logger = logging.getLogger(__name__) @@ -97,15 +102,11 @@ def _backfill_artifacts_from_messages(user_id: str, db: Session) -> int: # ``Artifact.artifact_id`` is a single-column global primary key, so a # content-hash file id already owned by another user/chat would otherwise # slip past a user-filtered set and blow up the whole INSERT batch. - existing_ids = set( - row[0] for row in db.query(Artifact.artifact_id).all() - ) + existing_ids = set(row[0] for row in db.query(Artifact.artifact_id).all()) # Scan both assistant messages (tool_calls) and user messages (attachments). - # Also carry the chat session's project_id — under a team project, backfilled - # rows must land on team_id+team_folder_id; otherwise a fallback backfill - # triggered by a single list request would pour a team chat's historical - # files into the personal MySpace root. + # Also carry the chat session's project_id so edition-specific scope fields + # are preserved during the backfill. rows = ( db.query( ChatMessage.chat_id, @@ -123,58 +124,47 @@ def _backfill_artifacts_from_messages(user_id: str, db: Session) -> int: .all() ) - # Cache project_id → (team_id, team_folder_id, user_folder_id) to reduce - # repeated lookups. - # Any missing item → treated as non-project; on write all NULL, landing in - # the MySpace root (consistent with old behavior). - from core.services.project_scope import project_scope_from_chat_id # local: avoid top-level cycle - _project_columns_cache: Dict[str, tuple] = {} + # Cache project_id → artifact scope fields to reduce repeated lookups. + from core.services.project_scope import ( # local: avoid top-level cycle + project_scope_from_chat_id, + ) + + _project_columns_cache: Dict[str, Dict[str, Optional[str]]] = {} - def _columns_for_chat(chat_id: str) -> tuple: - # Reverse-lookup via project_scope_from_chat_id; personal hit → - # user_folder_id; team hit → team_id + team_folder_id; otherwise all None. + def _fields_for_chat(chat_id: str) -> Dict[str, Optional[str]]: scope = project_scope_from_chat_id(db, chat_id) - if scope is None: - return (None, None, None) - if scope.is_team: - return (None, scope.team_id, scope.root_folder_id) - return (scope.root_folder_id, None, None) + return artifact_scope_fields(scope) created = 0 for chat_id, role, tool_calls_col, extra_data, _project_id in rows: file_refs: List[Dict[str, Any]] = [] is_strict_message = ( - isinstance(extra_data, dict) - and extra_data.get("workspace_files") is not None + isinstance(extra_data, dict) and extra_data.get("workspace_files") is not None ) # Source 1: tool_calls[].result (legacy assistant messages only) if role == "assistant" and not is_strict_message: - for tc in (tool_calls_col or []): + for tc in tool_calls_col or []: file_refs.extend(extract_file_refs(tc.get("result"))) if isinstance(extra_data, dict): # Source 2: extra_data.artifacts — pinned-only under strict mode - for art in (extra_data.get("artifacts") or []): + for art in extra_data.get("artifacts") or []: file_refs.extend(extract_file_refs(art)) # Source 3: extra_data.attachments (user uploads — always) - for att in (extra_data.get("attachments") or []): + for att in extra_data.get("attachments") or []: file_refs.extend(extract_file_refs(att)) if not file_refs: continue - # Resolve ownership columns per chat (personal/team/no-project); multiple - # refs within the same chat reuse them. if _project_id and _project_id in _project_columns_cache: - _bf_user_folder, _bf_team_id, _bf_team_folder = _project_columns_cache[_project_id] + scope_fields = _project_columns_cache[_project_id] else: - _bf_user_folder, _bf_team_id, _bf_team_folder = _columns_for_chat(chat_id) + scope_fields = _fields_for_chat(chat_id) if _project_id: - _project_columns_cache[_project_id] = ( - _bf_user_folder, _bf_team_id, _bf_team_folder, - ) + _project_columns_cache[_project_id] = scope_fields for ref in file_refs: fid = ref["file_id"] @@ -187,22 +177,22 @@ def _columns_for_chat(chat_id: str) -> tuple: existing_ids.add(fid) # claim before insert so dup refs in-batch skip try: with db.begin_nested(): - db.add(Artifact( - artifact_id=fid, - chat_id=chat_id, - user_id=user_id, - user_folder_id=_bf_user_folder, - team_id=_bf_team_id, - team_folder_id=_bf_team_folder, - type=infer_artifact_type(ref["mime_type"]), - title=ref["name"], - filename=ref["name"], - size_bytes=max(ref.get("size", 0) or 0, 1), - mime_type=ref["mime_type"], - storage_key=ref.get("storage_key") or f"artifacts/{fid}", - storage_url=ref.get("url", ""), - extra_data={"source": "backfill"}, - )) + db.add( + Artifact( + artifact_id=fid, + chat_id=chat_id, + user_id=user_id, + type=infer_artifact_type(ref["mime_type"]), + title=ref["name"], + filename=ref["name"], + size_bytes=max(ref.get("size", 0) or 0, 1), + mime_type=ref["mime_type"], + storage_key=ref.get("storage_key") or f"artifacts/{fid}", + storage_url=ref.get("url", ""), + extra_data={"source": "backfill"}, + **scope_fields, + ) + ) created += 1 except IntegrityError: logger.debug("backfill skip dup %s", fid, exc_info=True) @@ -220,7 +210,9 @@ def _columns_for_chat(chat_id: str) -> tuple: return created -def _collect_artifact_kb_usage(db: Session, user_id: str, artifact_ids: List[str]) -> Dict[str, List[Dict[str, str]]]: +def _collect_artifact_kb_usage( + db: Session, user_id: str, artifact_ids: List[str] +) -> Dict[str, List[Dict[str, str]]]: """Collect private KB memberships for a batch of artifact IDs.""" if not artifact_ids: return {} @@ -243,10 +235,12 @@ def _collect_artifact_kb_usage(db: Session, user_id: str, artifact_ids: List[str source_artifact_id = meta.get("source_artifact_id") if not source_artifact_id or source_artifact_id not in artifact_id_set: continue - usage.setdefault(source_artifact_id, []).append({ - "kb_id": space.kb_id, - "name": space.name, - }) + usage.setdefault(source_artifact_id, []).append( + { + "kb_id": space.kb_id, + "name": space.name, + } + ) return usage @@ -275,19 +269,26 @@ async def list_favorite_chats( q = q.filter(ChatSession.title.ilike(f"%{keyword}%")) total = q.count() - sessions = q.order_by(desc(ChatSession.updated_at)).offset( - (page - 1) * page_size - ).limit(page_size).all() + sessions = ( + q.order_by(desc(ChatSession.updated_at)) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) # Batch-fetch last message preview for all sessions in one query chat_ids = [s.chat_id for s in sessions] previews: Dict[str, str] = {} if chat_ids: # Window function: row_number per chat_id ordered by created_at desc - rn = func.row_number().over( - partition_by=ChatMessage.chat_id, - order_by=desc(ChatMessage.created_at), - ).label("rn") + rn = ( + func.row_number() + .over( + partition_by=ChatMessage.chat_id, + order_by=desc(ChatMessage.created_at), + ) + .label("rn") + ) subq = ( db.query(ChatMessage.chat_id, ChatMessage.content, rn) .filter( @@ -302,23 +303,31 @@ async def list_favorite_chats( items = [] for s in sessions: - items.append({ - "id": s.chat_id, - "type": "favorite", - "name": s.title or "对话", - "source_chat_id": s.chat_id, - "source_chat_title": s.title, - "content_preview": previews.get(s.chat_id, ""), - "created_at": (s.updated_at or s.created_at).isoformat() if (s.updated_at or s.created_at) else None, - }) - - return success_response(data={ - "items": items, - "total": total, - "page": page, - "page_size": page_size, - "has_more": page * page_size < total, - }) + items.append( + { + "id": s.chat_id, + "type": "favorite", + "name": s.title or "对话", + "source_chat_id": s.chat_id, + "source_chat_title": s.title, + "content_preview": previews.get(s.chat_id, ""), + "created_at": ( + (s.updated_at or s.created_at).isoformat() + if (s.updated_at or s.created_at) + else None + ), + } + ) + + return success_response( + data={ + "items": items, + "total": total, + "page": page, + "page_size": page_size, + "has_more": page * page_size < total, + } + ) @router.get("", summary="用户文件/图片列表") @@ -326,7 +335,7 @@ async def list_user_artifacts( type: Optional[str] = Query(None, description="document | image"), source_kind: Optional[str] = Query(None, description="user_upload | ai_generated"), keyword: Optional[str] = Query(None, description="文件名搜索"), - scope: Optional[str] = Query("personal", description="personal（默认，仅个人文件）| all（含团队）"), + scope: str = Depends(artifact_list_scope), folder_id: Optional[str] = Query( None, description="仅 personal scope 生效：__root__=个人根目录，=该个人文件夹直接子文件，省略=全部个人文件（向后兼容）", @@ -336,7 +345,7 @@ async def list_user_artifacts( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - """获取用户文件/图片列表（从 Artifact 表）。默认仅返回个人（非团队归属）文件。""" + """获取当前用户有权查看的文件与图片。""" uid = str(user.user_id) # One-time backfill for historical data. Claim under a lock BEFORE running @@ -367,9 +376,12 @@ async def list_user_artifacts( personal_only = scope != "all" rows, total = repo.list_by_user_with_chat( - user_id=uid, mime_prefix=mime_prefix, keyword=keyword, + user_id=uid, + mime_prefix=mime_prefix, + keyword=keyword, source_kind=normalized_source_kind, - page=page, page_size=page_size, + page=page, + page_size=page_size, personal_only=personal_only, folder_id=folder_id if personal_only else None, ) @@ -384,7 +396,7 @@ async def list_user_artifacts( linked_kbs = artifact_kb_usage.get(artifact.artifact_id, []) extra_data = artifact.extra_data if isinstance(artifact.extra_data, dict) else {} source_kind = "user_upload" if extra_data.get("source") == "user_upload" else "ai_generated" - items.append({ + item = { "id": artifact.artifact_id, "type": "image" if is_image else "document", "name": artifact.filename or artifact.title, @@ -396,19 +408,24 @@ async def list_user_artifacts( "knowledge_bases": linked_kbs, "source_chat_id": artifact.chat_id, "source_chat_title": row["chat_title"] or "对话", - "team_id": artifact.team_id, - "team_folder_id": artifact.team_folder_id, "user_folder_id": artifact.user_folder_id, - "created_at": (artifact.updated_at or artifact.created_at).isoformat() if (artifact.updated_at or artifact.created_at) else None, - }) - - return success_response(data={ - "items": items, - "total": total, - "page": page, - "page_size": page_size, - "has_more": page * page_size < total, - }) + "created_at": ( + (artifact.updated_at or artifact.created_at).isoformat() + if (artifact.updated_at or artifact.created_at) + else None + ), + } + items.append(extend_artifact_item(artifact, item)) + + return success_response( + data={ + "items": items, + "total": total, + "page": page, + "page_size": page_size, + "has_more": page * page_size < total, + } + ) @router.post("/{artifact_id}/knowledge-base", summary="资源加入知识库") @@ -419,7 +436,7 @@ async def add_artifact_to_knowledge_base( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - """将指定资源加入目标知识库（kb_id），校验归属/团队权限后后台异步向量化索引；文件已存在时直接返回。""" + """将有权访问的资源加入目标知识库；已存在时直接返回。""" uid = str(user.user_id) kb_service = KBService(db) @@ -438,15 +455,10 @@ async def add_artifact_to_knowledge_base( return success_response(data=document, message="该文件已在目标知识库中") try: - from core.auth.permissions_iface import has_permission, resolve_artifact_access - artifact = ArtifactRepository(db).get_by_id(artifact_id) if artifact is None: return error_response(message="资源不存在或无权限", code=404, status_code=404) - # Single point that combines owner ∪ team permissions (owner always - # allowed; team members with view+ can reference) - perm = resolve_artifact_access(db, uid, artifact.user_id, artifact.team_id) - if not has_permission(perm, "view"): + if not can_access_artifact(db, uid, artifact): return error_response(message="资源不存在或无权限", code=404, status_code=404) file_bytes = get_storage().download_bytes(artifact.storage_key) background_tasks.add_task( diff --git a/src/backend/api/routes/v1/auth.py b/src/backend/api/routes/v1/auth.py index 7252f613..52d43282 100644 --- a/src/backend/api/routes/v1/auth.py +++ b/src/backend/api/routes/v1/auth.py @@ -69,7 +69,6 @@ def _serialize_user(db: Session, user_data: dict, ttl_seconds: Optional[int] = N "nickname": local.nickname if local else user_data.get("nickname"), "real_name": local.real_name if local else user_data.get("real_name"), "department": None, - "teams": [], "expires_at": expires_at_iso(ttl_seconds or user_data.get("ttl_seconds")), "sso_token": None, "must_change_password": bool(meta.get("must_change_password")), diff --git a/src/backend/api/routes/v1/catalog.py b/src/backend/api/routes/v1/catalog.py index b6c9136e..36d4a7e9 100644 --- a/src/backend/api/routes/v1/catalog.py +++ b/src/backend/api/routes/v1/catalog.py @@ -12,31 +12,31 @@ from core.db.repository import KBRepository from core.infra.exceptions import BadRequestError from core.infra.responses import success_response -from core.kb.dify_kb import is_dify_enabled, list_datasets +from core.kb.external_provider import is_enabled, list_collections from core.services import CatalogService from fastapi import APIRouter, Depends, Path from pydantic import BaseModel, Field from sqlalchemy.orm import Session -# ── Dify dataset list cache (avoids 3s timeout on every page load) ── -_dify_cache_lock = Lock() -_dify_cache: Optional[tuple] = None # (expires_at, items) -_DIFY_CACHE_TTL = 30.0 +# ── External knowledge collection cache ── +_external_cache_lock = Lock() +_external_cache: Optional[tuple] = None # (expires_at, items) +_EXTERNAL_CACHE_TTL = 30.0 def _list_datasets_cached() -> List[Dict[str, Any]]: - """Return Dify datasets with 30s in-memory cache.""" - global _dify_cache + """Return external collections with a short in-memory cache.""" + global _external_cache now = monotonic() - with _dify_cache_lock: - if _dify_cache is not None: - expires_at, items = _dify_cache + with _external_cache_lock: + if _external_cache is not None: + expires_at, items = _external_cache if now < expires_at: return items - items = list_datasets(page=1, limit=100) - with _dify_cache_lock: - _dify_cache = (now + _DIFY_CACHE_TTL, items) + items = list_collections(page=1, limit=100) + with _external_cache_lock: + _external_cache = (now + _EXTERNAL_CACHE_TTL, items) return items @@ -309,17 +309,17 @@ def merge_items(base_items: List[Dict], override_items: List[Dict]) -> List[Dict is_ce = settings.edition.edition == "ce" - # ── Public KB (Dify; EE only) ───────────────────────────────────────────── + # ── Externally managed public knowledge collections ─────────────────────── public_kb_items: List[Dict[str, Any]] = [] - if not is_ce and is_dify_enabled(): + if not is_ce and is_enabled(): try: - dify_items = _list_datasets_cached() + external_items = _list_datasets_cached() # Permission assignment: narrow to datasets visible to the current user (public + granted scoped; defaults to public when unset). from core.auth.kb_permissions import get_dataset_levels - ds_ids = [str(it.get("id", "")).strip() for it in dify_items if it.get("id")] + ds_ids = [str(item.get("id", "")).strip() for item in external_items if item.get("id")] ds_levels = get_dataset_levels(db, user.user_id, ds_ids) - for item in dify_items: + for item in external_items: ds_id = str(item.get("id", "")).strip() level = ds_levels.get(ds_id) if not level: @@ -329,7 +329,7 @@ def merge_items(base_items: List[Dict], override_items: List[Dict]) -> List[Dict item["access_level"] = level public_kb_items.append(item) except Exception as exc: - logger.warning("Failed to load Dify KB datasets: %s", exc) + logger.warning("Failed to load external knowledge collections: %s", exc) # ── Private KB (local Milvus) ───────────────────────────────────────────── try: diff --git a/src/backend/api/routes/v1/chats.py b/src/backend/api/routes/v1/chats.py index 4c560fee..0a516aca 100644 --- a/src/backend/api/routes/v1/chats.py +++ b/src/backend/api/routes/v1/chats.py @@ -9,7 +9,7 @@ import anyio from api.schemas import AttachmentItem, ChatRequest, ChatResponse from core.auth.backend import UserContext, get_current_user -from core.auth.permissions_iface import can_delete_session, can_modify_share_scope +from core.auth.permissions_iface import can_delete_session from core.chat.context import build_effective_user_message as _build_effective_user_message from core.chat.context import ( build_runtime_context, @@ -92,13 +92,7 @@ class UpdateChatRequest(BaseModel): def _session_to_dict(s) -> dict: - """Convert a ChatSession ORM object to API response dict. - - Legacy callers keep seeing the old fields. The new fields (share_scope/ - owner_user_id/is_owner/access_level/per-user pin&favorite) are assembled by - :func:`_session_view_for_user`, so old call sites never receive incorrect - pin/favorite values. - """ + """Convert a ChatSession ORM object to the edition-neutral API response.""" return { "chat_id": s.chat_id, "title": s.title, @@ -108,49 +102,17 @@ def _session_to_dict(s) -> dict: "favorite": s.favorite, # Project attachment (if any) — the frontend uses this to bind the chat back to the project and auto-attaches project_id when sending new messages "project_id": s.project_id, - "share_scope": getattr(s, "share_scope", "private") or "private", "metadata": s.extra_data or {}, "created_at": s.created_at.isoformat(), "updated_at": s.updated_at.isoformat(), } -def _is_team_share_session(db: Session, s) -> bool: - """Whether the session is attached to a team project (sessions inside a team project render with shared semantics by default).""" - if not getattr(s, "project_id", None): - return False - from core.db.models import Project - - p = ( - db.query(Project) - .filter(Project.project_id == s.project_id, Project.deleted_at.is_(None)) - .first() - ) - return bool(p and p.kind == "team") - - def _session_view_for_user(db: Session, s, user_id: str, level: str) -> dict: - """Session view rendered for this specific user: under team-share, pin/favorite come from the per-user table.""" - base = _session_to_dict(s) - team_share = _is_team_share_session(db, s) - if team_share: - from core.db.models import ChatSessionUserState - - state = ( - db.query(ChatSessionUserState) - .filter( - ChatSessionUserState.chat_id == s.chat_id, - ChatSessionUserState.user_id == user_id, - ) - .first() - ) - base["pinned"] = bool(state.pinned) if state is not None else False - base["favorite"] = bool(state.favorite) if state is not None else False - base["owner_user_id"] = s.user_id - base["is_owner"] = s.user_id == user_id - base["access_level"] = level - base["is_team_project"] = team_share - return base + """Render a session for the current user, then let the edition extend it.""" + from core.services.chat_edition import extend_session_view + + return extend_session_view(db, s, user_id, level, _session_to_dict(s)) def _message_to_dict(m) -> dict: @@ -326,13 +288,7 @@ async def list_pending_confirms( async def get_chat( chat_id: str, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db) ): - """获取会话详情。 - - - 会话 owner 永远可读。 - - 团队项目里 ``share_scope ∈ {team_read, team_edit}`` 且项目开关 ON 时， - 项目成员也可读，响应里携带 ``access_level`` / ``share_scope`` / - ``is_owner``，前端据此渲染只读 banner 等。 - """ + """获取当前用户有权读取的会话详情。""" chat_service = ChatService(db) pair = chat_service.get_session_with_access(chat_id, str(user.user_id)) @@ -353,14 +309,7 @@ async def update_chat( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - """更新会话元信息。共享场景下权限矩阵： - - - **title**：``admin`` 或 ``edit`` 都可改。 - - **pinned / favorite**：team-share 会话写 ``chat_session_user_states``（per-user 独立）； - 非共享会话写老字段 ``ChatSession.pinned/favorite``。 - - **metadata**：仅 owner（``admin`` 级）可改 —— 业务上 metadata 可能含会话级 - 设置，不放给协作成员。 - """ + """更新当前用户有权修改的会话元信息。""" chat_service = ChatService(db) user_id = str(user.user_id) @@ -383,18 +332,18 @@ async def update_chat( raise HTTPException(status_code=403, detail="会话元数据仅创建者可改") metadata_change = request.metadata - # pinned / favorite: take different paths depending on team-share or not - team_share = _is_team_share_session(db, session) - if (request.pinned is not None or request.favorite is not None) and team_share: - from core.services.project_service import ProjectService + from core.services.chat_edition import update_member_state - ProjectService(db).upsert_chat_user_state( - chat_id, + member_state_updated = False + if request.pinned is not None or request.favorite is not None: + member_state_updated = update_member_state( + db, + session, user_id, pinned=request.pinned, favorite=request.favorite, ) - # Under team-share, no longer write ChatSession.pinned/favorite + if member_state_updated: pin_change = None fav_change = None else: @@ -432,8 +381,7 @@ async def update_chat( async def delete_chat( chat_id: str, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db) ): - """软删会话。会话 owner 永远可删；团队项目 admin（在项目开关 ON 时）也可 - 删共享会话；普通成员即便是 ``team_edit`` 也不能删。""" + """软删当前用户有权管理的会话。""" chat_service = ChatService(db) user_id = str(user.user_id) @@ -458,7 +406,7 @@ async def list_messages( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - """获取会话消息列表。共享会话（含 team_read）成员可读；非成员 404。""" + """获取当前用户有权读取的会话消息列表；无权访问时返回 404。""" chat_service = ChatService(db) user_id = str(user.user_id) @@ -477,61 +425,6 @@ async def list_messages( ) -# ── Share scope ────────────────────────────────────────────────────────── - - -class UpdateShareScopeBody(BaseModel): - share_scope: str = Field(..., description="'private' | 'team_read' | 'team_edit'") - - -@router.post("/{chat_id}/share", summary="设置/取消会话在团队项目内的共享范围") -async def update_share_scope( - chat_id: str, - body: UpdateShareScopeBody, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """会话级共享开关。仅会话 owner 可改；必须挂在团队项目下。""" - chat_service = ChatService(db) - user_id = str(user.user_id) - - if body.share_scope not in ("private", "team_read", "team_edit"): - raise HTTPException(status_code=400, detail="share_scope 取值非法") - - session = chat_service.session_repo.get_by_id(chat_id) - if session is None: - raise ResourceNotFoundError(resource_type="chat_session", resource_id=chat_id) - - if not can_modify_share_scope(db, user_id, session): - raise HTTPException(status_code=403, detail="仅会话创建者可调整共享范围") - - # A shared state requires the session to be attached to a team project; private needs no project-type check (users may revoke sharing at any time) - if body.share_scope != "private": - if not session.project_id: - raise HTTPException(status_code=400, detail="会话未挂载到项目，无法共享") - from core.db.models import Project - - project = ( - db.query(Project) - .filter(Project.project_id == session.project_id, Project.deleted_at.is_(None)) - .first() - ) - if project is None or project.kind != "team": - raise HTTPException(status_code=400, detail="仅团队项目内的会话可共享") - - chat_service.update_session_fields( - chat_id, {"share_scope": body.share_scope}, actor_user_id=user_id - ) - pair = chat_service.get_session_with_access(chat_id, user_id) - if pair is None: - raise ResourceNotFoundError(resource_type="chat_session", resource_id=chat_id) - s2, level2 = pair - return success_response( - data=_session_view_for_user(db, s2, user_id, level2), - message="共享范围已更新", - ) - - @router.get("/{chat_id}/messages/{message_id}/followups", summary="获取追问问题") async def get_followups( chat_id: str, @@ -856,96 +749,50 @@ def _build_ctx( # When chat_mode is not explicitly given, default to "thinking: medium" resolved_chat_mode = request.chat_mode or "medium" - # Project metadata (if any): name + instructions + linked folder name + file listing, injected into the system prompt. - # A project is now essentially a view over a MySpace folder; the agent's access domain ≈ that folder's subtree. + # Project metadata is edition-owned; the shared chat path consumes only the + # returned context map and never imports organization models. project_id = getattr(request, "project_id", None) - project_name: Optional[str] = None - project_instructions: Optional[str] = None - project_folder_name: Optional[str] = None - project_folder_kind: Optional[str] = None # 'personal' | 'team' - project_folder_id: Optional[str] = None # linked_folder_id / linked_team_folder_id - project_team_id: Optional[str] = ( - None # Set only for team kind; lets agent file tools resolve via TeamFolder - ) - project_files: Optional[List[Dict[str, Any]]] = None + project_ctx: Dict[str, Any] = { + "project_id": project_id, + "project_name": None, + "project_instructions": None, + "project_folder_name": None, + "project_folder_kind": None, + "project_folder_id": None, + "project_files": None, + } if project_id: try: from core.db.engine import SessionLocal as _Sess - from core.db.models import Project as _Project - from core.db.models import TeamFolder as _TF - from core.db.models import UserFolder as _UF - from core.services.project_file_service import ProjectFileService as _PFS + from core.services.project_scope import build_project_ctx with _Sess() as _db: - _p = ( - _db.query(_Project) - .filter( - _Project.project_id == project_id, - _Project.deleted_at.is_(None), - ) - .first() - ) - if _p is not None: - project_name = _p.name - project_instructions = (_p.instructions or "").strip() or None - # Project-level memory switches fully override the user-level ones: inside a project only the project's own settings apply; - # when absent (old projects / never explicitly turned off) they default to True. - _proj_extra = _p.extra_data or {} - memory_enabled = bool(_proj_extra.get("memory_enabled", True)) - memory_write_enabled = bool(_proj_extra.get("memory_write_enabled", True)) - if _p.kind == "personal" and _p.linked_folder_id: - row = ( - _db.query(_UF.name).filter(_UF.folder_id == _p.linked_folder_id).first() - ) - project_folder_name = row[0] if row else None - project_folder_kind = "personal" - project_folder_id = _p.linked_folder_id - elif _p.kind == "team" and _p.linked_team_folder_id: - row = ( - _db.query(_TF.name) - .filter(_TF.folder_id == _p.linked_team_folder_id) - .first() - ) - project_folder_name = row[0] if row else None - project_folder_kind = "team" - project_folder_id = _p.linked_team_folder_id - project_team_id = _p.team_id - # File listing: reuse the service's existing expansion of the linked folder subtree - try: - project_files = _PFS(_db).list_files(_p) - except Exception: - logger.warning( - "[chat] project file list failed for %s", project_id, exc_info=True - ) - project_files = [] + resolved_project_ctx = build_project_ctx(_db, project_id) + if resolved_project_ctx: + project_ctx.update(resolved_project_ctx) + memory_enabled = bool(project_ctx.pop("_memory_enabled", True)) + memory_write_enabled = bool(project_ctx.pop("_memory_write_enabled", True)) except Exception: logger.warning("[chat] project ctx lookup failed for %s", project_id, exc_info=True) - # Project mode → mem0 isolates memories in a dedicated workspace namespace (avoids polluting the default space) workspace_id_value = f"project:{project_id}" if project_id else "default" - # In team projects, use "team:" as the mem0 user_id so all team members' reads/writes share one bucket; - # personal projects and the default space keep the real user_id (private to the user). The real author goes into - # metadata author_user_id, and audit still records the real user_id. - memory_scope_user_id_value = ( - f"team:{project_team_id}" if project_folder_kind == "team" and project_team_id else None - ) + memory_scope_user_id_value = project_ctx.pop("memory_scope_user_id", None) - from core.services.ontology_service import disabled_ontology_runtime + from core.services.ontology_service import ( + build_ontology_runtime_for_preference, + disabled_ontology_runtime, + ) ontology_runtime: Dict[str, Any] = disabled_ontology_runtime() if ontology_enabled: try: - from core.services.ontology_service import OntologyService - with SessionLocal() as _ontology_db: - ontology_runtime = OntologyService(_ontology_db).build_runtime( + ontology_enabled, ontology_runtime = build_ontology_runtime_for_preference( + enabled=True, task=request.message, + db=_ontology_db, pack_ids=ontology_pack_ids or None, ) - if not ontology_runtime.get("enabled"): - raise ServiceUnavailableError( - "本体校验已开启，但当前没有可用的已激活 Domain Pack" - ) except Exception as exc: # noqa: BLE001 logger.exception("[ontology] failed to build runtime policy") raise ServiceUnavailableError( @@ -990,15 +837,7 @@ def _build_ctx( "plan_chat": request.plan_chat, "batch_chat": request.batch_chat, "disable_batch_plan": request.disable_batch_plan, - # ── Project mode ── - "project_id": project_id, - "project_name": project_name, - "project_instructions": project_instructions, - "project_folder_name": project_folder_name, - "project_folder_kind": project_folder_kind, - "project_folder_id": project_folder_id, - "project_team_id": project_team_id, - "project_files": project_files, + **project_ctx, } return ctx @@ -1023,12 +862,12 @@ def _ensure_chat_session( extra_data["plan_chat"] = True if batch_chat: extra_data["batch_chat"] = True - # Prefer resolving in the shared context first (covers team_edit members sending messages into someone else's shared session) + # Prefer the edition-aware access resolver before creating a session. pair = chat_service.get_session_with_access(chat_id, user_id) if pair is not None: session, level = pair if level not in ("admin", "edit"): - # Read-only levels such as team_read are not allowed to write + # Read-only access levels are not allowed to write. raise HTTPException(status_code=403, detail="只读共享会话不可写入消息") if level != "admin": # Non-owner member: reuse the session, but never modify the metadata / project_id set by the owner @@ -1637,8 +1476,8 @@ async def _stream_sse_response( # Build a ProjectScope from the workflow context and pass it explicitly. # This is the core fix for the personal MySpace root leak in trace 9d218075…: # workflow.py's finally block has already cleared the internal scope by this - # point, so without passing it explicitly _persist_artifacts gets no team info - # and would file team AI output as personal-chat output into an orphan row + # point, so it must be passed explicitly to keep generated output + # inside the project-owned file scope. # with user_folder_id=NULL. _stream_scope = project_scope_from_context(context) _persist_artifacts( diff --git a/src/backend/api/routes/v1/internal_sites.py b/src/backend/api/routes/v1/internal_sites.py index adc96a5a..3812df62 100644 --- a/src/backend/api/routes/v1/internal_sites.py +++ b/src/backend/api/routes/v1/internal_sites.py @@ -26,20 +26,23 @@ import uuid from typing import List, Optional, Tuple +from core.infra.responses import success_response +from core.services.artifact_edition import personal_artifact_predicates +from core.services.site_access_policy import SitePublishScopeFields, site_scope_ref from fastapi import APIRouter, Header, HTTPException from pydantic import BaseModel, Field -from core.infra.responses import success_response - logger = logging.getLogger(__name__) router = APIRouter(prefix="/v1/internal/sites", tags=["internal-sites"]) -MAX_PACK_BYTES = 40 * 1024 * 1024 # tar archive cap (a separate 30MB total quota applies after unpacking) -_UNPACK_MAX_FILES = 400 # unpack fuse (service layer caps at 300; slightly looser here) +MAX_PACK_BYTES = ( + 40 * 1024 * 1024 +) # tar archive cap (a separate 30MB total quota applies after unpacking) +_UNPACK_MAX_FILES = 400 # unpack fuse (service layer caps at 300; slightly looser here) -class PublishBody(BaseModel): +class PublishBody(SitePublishScopeFields): src_dir: str = Field( "", description=( @@ -58,11 +61,9 @@ class PublishBody(BaseModel): title: str = "" slug: str = "" site_id: str = "" - visibility: str = "public" description: str = "" - team_id: str = "" - user_id: str = "" # parsed by the MCP side from X-Current-User-Id - chat_id: str = "" # parsed by the MCP side from X-Conversation-Id (sandbox session key) + user_id: str = "" # parsed by the MCP side from X-Current-User-Id + chat_id: str = "" # parsed by the MCP side from X-Conversation-Id (sandbox session key) def _check_internal_token(token: Optional[str]) -> None: @@ -96,11 +97,7 @@ def _resolve_project_context(chat_id: str, user_id: str): from core.db.models import ChatSession, Project, UserFolder with SessionLocal() as db: - sess = ( - db.query(ChatSession.project_id) - .filter(ChatSession.chat_id == chat_id) - .first() - ) + sess = db.query(ChatSession.project_id).filter(ChatSession.chat_id == chat_id).first() project_id = sess[0] if sess else None if not project_id: return None, None @@ -122,7 +119,9 @@ def _resolve_project_context(chat_id: str, user_id: str): if not folder_name: return None, None return project_id, f"/workspace/myspace/{user_id}/{folder_name}" - except Exception: # noqa: BLE001 — on resolve failure, treat as no bound project and take the legacy path + except ( + Exception + ): # noqa: BLE001 — on resolve failure, treat as no bound project and take the legacy path logger.warning("[internal-sites] project context resolve failed", exc_info=True) return None, None @@ -195,7 +194,7 @@ def _mirror_files_to_project_folder( now = _dt.utcnow() db.query(Artifact).filter( Artifact.user_id == user_id, - Artifact.team_id.is_(None), + *personal_artifact_predicates(Artifact), Artifact.user_folder_id.in_(subtree_ids), Artifact.deleted_at.is_(None), ).update({Artifact.deleted_at: now}, synchronize_session=False) @@ -215,7 +214,8 @@ def _mirror_files_to_project_folder( pfs.upload(proj, user_id, content, rel_path, mime or "text/plain") except Exception: # noqa: BLE001 — one file failing does not affect the rest logger.warning( - "[internal-sites] mirror file to project failed: %s", rel_path, + "[internal-sites] mirror file to project failed: %s", + rel_path, exc_info=True, ) except Exception: # noqa: BLE001 @@ -255,12 +255,16 @@ def _ensure_project_for_site(site_id: str, user_id: str, title: str) -> Optional # Otherwise create a new personal project named after the site title (create_personal creates a same-named folder) name = (title or site.title or "站点").strip()[:200] or "站点" proj = ProjectService(db).create_personal( - user_id, name=name, description="对话建站源码工程（发布后可继续编辑）", + user_id, + name=name, + description="对话建站源码工程（发布后可继续编辑）", ) site.project_id = proj.project_id db.commit() return proj.project_id - except Exception: # noqa: BLE001 — project creation failure does not affect an already-successful publish + except ( + Exception + ): # noqa: BLE001 — project creation failure does not affect an already-successful publish logger.warning("[internal-sites] ensure project for site failed", exc_info=True) return None @@ -285,7 +289,7 @@ def _project_root_has_package_json(project_id: str, user_id: str) -> bool: db.query(Artifact.artifact_id) .filter( Artifact.user_id == user_id, - Artifact.team_id.is_(None), + *personal_artifact_predicates(Artifact), Artifact.user_folder_id == proj.linked_folder_id, Artifact.filename == "package.json", Artifact.deleted_at.is_(None), @@ -307,11 +311,9 @@ async def _pack_and_fetch_dir( ) -> Tuple[Optional[List[Tuple[str, bytes]]], Optional[str]]: """tar the directory inside the sandbox → fetch it back → safely unpack. Returns exactly one of (files, error).""" from core.llm.tools._common import sandbox_exec_bash, shell_quote - from core.sandbox import ( - SandboxConnectError as _SandboxConnectError, - SandboxError as _SandboxError, - get_sandbox_provider as _get_provider, - ) + from core.sandbox import SandboxConnectError as _SandboxConnectError + from core.sandbox import SandboxError as _SandboxError + from core.sandbox import get_sandbox_provider as _get_provider excludes = (".git", "node_modules", "__pycache__") + tuple(extra_excludes) exclude_args = " ".join(f"--exclude={shell_quote(e)}" for e in excludes) @@ -422,11 +424,15 @@ async def publish( if source_dir == src: # source dir == output dir = the caller is publishing source as the build output — # error out explicitly; silently degrading would publish unbuilt source as the live site (the page is simply broken). - return success_response(data={"error": ( - "src_dir 与 source_dir 不能是同一个目录：src_dir 必须指向构建产物" - "（先 npm run build，产物在 /workspace/.site-dist/ 下，见 init 脚本输出），" - "source_dir 指向源码工程目录。请构建后带两个不同的目录重试。" - )}) + return success_response( + data={ + "error": ( + "src_dir 与 source_dir 不能是同一个目录：src_dir 必须指向构建产物" + "（先 npm run build，产物在 /workspace/.site-dist/ 下，见 init 脚本输出），" + "source_dir 指向源码工程目录。请构建后带两个不同的目录重试。" + ) + } + ) # Target site: explicit site_id > the project's already-associated live site (edit / new version) > create new target_site_id = (body.site_id or "").strip() @@ -450,7 +456,9 @@ async def publish( (files, err), (src_files, src_err) = await asyncio.gather( _pack_and_fetch_dir(src, _sess, user_id), _pack_and_fetch_dir( - source_dir, _sess, user_id, + source_dir, + _sess, + user_id, extra_excludes=("dist", ".vite", "*.log"), ), ) @@ -465,11 +473,15 @@ async def publish( # session forgot to pass src_dir and src fell back to the project folder) — the published # live page would simply be broken (index.html referencing /src/main.jsx). if not source_dir and SiteService.looks_like_source_tree(p for p, _ in files): - return success_response(data={"error": ( - f"目录 {src} 是一个未构建的源码工程（含 package.json/src/），" - "不能直接发布。请先在工程目录 npm run build，再调用 " - "publish_site(src_dir='<构建产物目录>', source_dir='<源码工程目录>')。" - )}) + return success_response( + data={ + "error": ( + f"目录 {src} 是一个未构建的源码工程（含 package.json/src/），" + "不能直接发布。请先在工程目录 npm run build，再调用 " + "publish_site(src_dir='<构建产物目录>', source_dir='<源码工程目录>')。" + ) + } + ) db = SessionLocal() try: @@ -482,7 +494,7 @@ async def publish( chat_id=body.chat_id or None, visibility=body.visibility, description=body.description, - team_id=(body.team_id or None), + scope_id=site_scope_ref(body), build_info=( {"kind": "build", "published_from": src, "source_dir": source_dir} if source_dir @@ -508,15 +520,11 @@ async def publish( if effective_project_id: if source_dir: if src_files: - _mirror_files_to_project_folder( - effective_project_id, user_id, src_files - ) + _mirror_files_to_project_folder(effective_project_id, user_id, src_files) mirrored_from = source_dir else: mirror_note = f"（注意：源码目录打包失败，未镜像进项目：{src_err}）" - logger.warning( - "[internal-sites] source mirror failed: %s", src_err - ) + logger.warning("[internal-sites] source mirror failed: %s", src_err) elif _project_root_has_package_json(effective_project_id, user_id): # Build-style project + no source_dir passed: never mirror (replace semantics # would wipe out the project's source workspace with the published content). diff --git a/src/backend/api/routes/v1/kb.py b/src/backend/api/routes/v1/kb.py index 1830c7b4..1c1ee2de 100644 --- a/src/backend/api/routes/v1/kb.py +++ b/src/backend/api/routes/v1/kb.py @@ -211,12 +211,7 @@ async def create_kb_space( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - """创建知识库空间，归属当前登录用户。 - - visibility=private 需 ``can_create_private_kb``（仅本人可见）；visibility=public 需 - ``can_create_public_kb``——公有库默认对所有人不可见，创建后**自动授权给创建者所属的 - 所有团队（view）**，即对其团队可见；之后可在「用户管理 / 团队管理」继续调整授权。 - """ + """创建归属当前登录用户的知识库空间；可见性规则由当前版本决定。""" visibility = (request.visibility or "private").strip() if visibility not in ("private", "public"): raise BadRequestError( @@ -242,12 +237,9 @@ async def create_kb_space( if request.indexing_config: metadata["indexing_config"] = request.indexing_config.model_dump() - # Public KB -> automatically grant visibility to the creator's teams (members inherit view); private KB is owner-only. - grant_team_ids: list[str] = [] - if visibility == "public": - from core.auth.kb_permissions import _user_team_ids + from core.services.kb_edition import initial_visibility_grants - grant_team_ids = sorted(_user_team_ids(db, user.user_id)) + initial_grants = initial_visibility_grants(db, user.user_id, visibility) space = kb_service.create_space( user_id=user.user_id, @@ -256,7 +248,7 @@ async def create_kb_space( chunk_method=request.chunk_method or "semantic", metadata=metadata, visibility=visibility, - grant_team_ids=grant_team_ids, + initial_grants=initial_grants, granted_by=user.user_id, ) return created_response(data=space, message="Knowledge base space created successfully") @@ -408,24 +400,23 @@ async def upload_document( @router.get("/{kb_id}/documents", summary="获取知识库文档列表") async def list_documents( - kb_id: str = Path(..., description="KB space ID or Dify dataset ID"), + kb_id: str = Path(..., description="Local or external knowledge collection ID"), page: int = Query(1, ge=1, description="Page number"), page_size: int = Query(20, ge=1, le=100, description="Items per page"), user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - """分页获取指定知识库空间下的文档列表。仅空间所有者可查看本地知识库；当 kb_id 不是本地空间且启用了 Dify 时，回退到 Dify 数据集获取文档列表。""" + """分页获取知识库文档；非本地 ID 交由当前版本的外部知识提供方处理。""" from core.db.repository import KBRepository kb_repo = KBRepository(db) kb_space = kb_repo.get_space(kb_id) if not kb_space: - from core.kb.dify_kb import is_dify_enabled - from core.kb.dify_kb import list_documents as dify_list_docs + from core.kb.external_provider import is_enabled, list_documents - if is_dify_enabled(): - result = dify_list_docs(kb_id, page=page, limit=page_size) + if is_enabled(): + result = list_documents(kb_id, page=page, limit=page_size) return paginated_response( items=result.get("items", []), page=result.get("page", page), @@ -467,23 +458,22 @@ async def list_documents( @router.get("/{kb_id}/documents/{document_id}", summary="获取知识库文档详情") async def get_document_detail( - kb_id: str = Path(..., description="KB space ID or Dify dataset ID"), + kb_id: str = Path(..., description="Local or external knowledge collection ID"), document_id: str = Path(..., description="Document ID"), user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - """获取指定文档的详情，包括标题、文件名、类型及拼接后的全文内容（按分块顺序合并）。仅空间所有者可查看本地知识库；当 kb_id 不是本地空间且启用了 Dify 时，回退到 Dify 获取文档详情。""" + """获取文档详情；非本地 ID 交由当前版本的外部知识提供方处理。""" from core.db.repository import KBRepository kb_repo = KBRepository(db) kb_space = kb_repo.get_space(kb_id) if not kb_space: - from core.kb.dify_kb import get_document_detail as dify_get_document_detail - from core.kb.dify_kb import is_dify_enabled + from core.kb.external_provider import get_document_detail, is_enabled - if is_dify_enabled(): - detail = dify_get_document_detail(kb_id, document_id) + if is_enabled(): + detail = get_document_detail(kb_id, document_id) return success_response(data=detail, message="Document detail retrieved successfully") raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) diff --git a/src/backend/api/routes/v1/kb_models.py b/src/backend/api/routes/v1/kb_models.py index 6ff6b65a..2f73c306 100644 --- a/src/backend/api/routes/v1/kb_models.py +++ b/src/backend/api/routes/v1/kb_models.py @@ -1,60 +1,56 @@ -"""Pydantic request/response models for Knowledge Base routes.""" +"""Community knowledge-base request models.""" + +from typing import List, Literal, Optional -from typing import List, Optional from pydantic import BaseModel, Field class IndexingConfig(BaseModel): - """Indexing configuration for KB space.""" - parent_chunk_size: int = Field(1024, ge=256, le=4096, description="Parent chunk size in chars") - child_chunk_size: int = Field(128, ge=64, le=512, description="Child chunk size in chars") - overlap_tokens: int = Field(20, ge=0, le=100, description="Overlap between child chunks") - parent_child_indexing: bool = Field(True, description="Enable parent-child indexing; False = index parent chunks only") - auto_keywords_count: int = Field(0, ge=0, le=10, description="LLM auto-extract keyword count per chunk (0=disabled)") - auto_questions_count: int = Field(0, ge=0, le=5, description="LLM auto-generate question count per chunk (0=disabled)") - separators: Optional[List[str]] = Field( - None, - description="父分块分隔符层级（仅对递归分块及语义分块的递归兜底生效）；为空用内置默认层级", - ) - child_separators: Optional[List[str]] = Field( - None, - description="子分块分隔符层级（父子分块时按此切子块，再按 child_size 打包）；为空走定长滑窗", - ) + parent_chunk_size: int = Field(1024, ge=256, le=4096) + child_chunk_size: int = Field(128, ge=64, le=512) + overlap_tokens: int = Field(20, ge=0, le=100) + parent_child_indexing: bool = True + auto_keywords_count: int = Field(0, ge=0, le=10) + auto_questions_count: int = Field(0, ge=0, le=5) + separators: Optional[List[str]] = None + child_separators: Optional[List[str]] = None class CreateKBSpaceRequest(BaseModel): - """Request model for creating a KB space.""" - name: str = Field(..., min_length=1, max_length=255, description="KB space name") - description: Optional[str] = Field(None, description="KB space description") - chunk_method: Optional[str] = Field("semantic", description="Chunking strategy: semantic|laws|qa") - metadata: Optional[dict] = Field(default_factory=dict, description="Additional metadata") - indexing_config: Optional[IndexingConfig] = Field(None, description="Advanced indexing configuration") - visibility: Optional[str] = Field("private", description="private（仅本人）| public（共享，按授权可见）") - grant_team_ids: Optional[List[str]] = Field(None, description="创建后授予可见的团队 ID 列表（内容管理台创建公有库时指定）") + name: str = Field(..., min_length=1, max_length=255) + description: Optional[str] = None + chunk_method: Optional[str] = "semantic" + metadata: Optional[dict] = Field(default_factory=dict) + indexing_config: Optional[IndexingConfig] = None + visibility: Literal["private"] = "private" class UpdateKBSpaceRequest(BaseModel): - """Request model for updating a KB space.""" - name: Optional[str] = Field(None, min_length=1, max_length=255, description="KB space name") - description: Optional[str] = Field(None, description="KB space description") + name: Optional[str] = Field(None, min_length=1, max_length=255) + description: Optional[str] = None class PolishKBDescriptionRequest(BaseModel): - """Request model for AI-polishing a KB description.""" - name: str = Field(..., min_length=1, max_length=255, description="KB space name") - description: Optional[str] = Field(None, description="Current KB description") + name: str = Field(..., min_length=1, max_length=255) + description: Optional[str] = None class UpdateChunkRequest(BaseModel): - """Request model for updating chunk content, tags and questions.""" - content: Optional[str] = Field(None, description="Full chunk text (admin manual edit; triggers re-embedding)") - tags: Optional[List[str]] = Field(None, description="Tag list for BM25 augmentation") - questions: Optional[List[str]] = Field(None, description="Question list for multi-surface indexing") + content: Optional[str] = None + tags: Optional[List[str]] = None + questions: Optional[List[str]] = None class ReindexRequest(BaseModel): - """Optional indexing config override for reindexing.""" - indexing_config: Optional[IndexingConfig] = Field(None, description="Override indexing configuration") - chunk_method: Optional[str] = Field(None, description="Override chunking method: structured|recursive|embedding_semantic|laws|qa") - - + indexing_config: Optional[IndexingConfig] = None + chunk_method: Optional[str] = None + + +__all__ = [ + "CreateKBSpaceRequest", + "IndexingConfig", + "PolishKBDescriptionRequest", + "ReindexRequest", + "UpdateChunkRequest", + "UpdateKBSpaceRequest", +] diff --git a/src/backend/api/routes/v1/lab_skill_distill.py b/src/backend/api/routes/v1/lab_skill_distill.py deleted file mode 100644 index 4efcabb0..00000000 --- a/src/backend/api/routes/v1/lab_skill_distill.py +++ /dev/null @@ -1,179 +0,0 @@ -"""Lab · Personal skill-distillation API. - -POST /v1/lab/skill-distill/jobs Create a distillation job (chosen chats / all chats) -GET /v1/lab/skill-distill/jobs My job list -GET /v1/lab/skill-distill/jobs/{job_id} Job detail (with progress and artifact) -POST /v1/lab/skill-distill/jobs/{job_id}/save Persist the artifact as my private skill -POST /v1/lab/skill-distill/jobs/{job_id}/cancel Cancel -DELETE /v1/lab/skill-distill/jobs/{job_id} Delete - -A job is a PersonaDistillJob with kind='personal': target_user = requested_by = self. -""" - -from __future__ import annotations - -from typing import List, Optional, Union - -from fastapi import APIRouter, Depends, Query, status -from pydantic import BaseModel, Field -from sqlalchemy.orm import Session - -from core.auth.backend import UserContext, get_current_user -from core.db.engine import get_db -from core.db.models import PersonaDistillJob -from core.infra.exceptions import AccessDeniedError, BadRequestError, ResourceNotFoundError -from core.infra.responses import created_response, success_response -from core.services import persona_distillation_service as pds - -router = APIRouter(prefix="/v1/lab/skill-distill", tags=["LabSkillDistill"]) - -_MAX_SELECTED_CHATS = 500 - - -class CreateJobRequest(BaseModel): - # Explicit list = chosen chats; "all" = all chats (server samples recent-first up to the cap) - chat_ids: Union[List[str], str] = Field(..., description='会话 ID 列表，或字符串 "all"') - hint: Optional[str] = Field(None, description="蒸馏侧重提示", max_length=500) - include_project_memories: bool = Field(True, description="纳入所选会话关联项目中本人的记忆") - - -class SaveJobRequest(BaseModel): - skill_content: Optional[str] = Field(None, description="编辑后的 SKILL.md 全文（不传用原产物）") - enable: bool = Field(True, description="保存后立即启用") - - -def _ensure_lab_enabled(db: Session, user_id: str) -> None: - """Server-side fallback check of lab permission: personal explicit → team default → default on.""" - from core.auth.capabilities import resolve_user_capabilities - - if not resolve_user_capabilities(db, str(user_id))["lab_enabled"]: - raise AccessDeniedError("实验室功能未对当前账号开放") - - -def _job_to_dict(job: PersonaDistillJob, include_result: bool = False) -> dict: - return pds.job_to_dict(job, include_result=include_result) - - -def _get_own_job(db: Session, job_id: str, user_id: str) -> PersonaDistillJob: - job = pds.get_job(db, job_id) - if job is None or job.kind != "personal" or job.target_user_id != str(user_id): - raise ResourceNotFoundError("persona_distill_job", job_id) - return job - - -@router.post("/jobs", status_code=status.HTTP_201_CREATED, summary="创建个人技能蒸馏作业") -async def create_job( - body: CreateJobRequest, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - _ensure_lab_enabled(db, user.user_id) - - if isinstance(body.chat_ids, str): - if body.chat_ids != "all": - raise BadRequestError('chat_ids 必须是会话 ID 列表或 "all"') - scope_chat_ids = None - else: - ids = [c.strip() for c in body.chat_ids if c and c.strip()] - if not ids: - raise BadRequestError("至少选择一个会话") - if len(ids) > _MAX_SELECTED_CHATS: - raise BadRequestError(f"最多选择 {_MAX_SELECTED_CHATS} 个会话") - invalid = pds.validate_chat_ids(db, ids, str(user.user_id)) - if invalid: - raise BadRequestError(f"以下会话不存在或不属于你：{', '.join(invalid[:5])}") - scope_chat_ids = ids - - # Only one in-progress job allowed per user at a time - active = ( - db.query(PersonaDistillJob) - .filter( - PersonaDistillJob.kind == "personal", - PersonaDistillJob.target_user_id == str(user.user_id), - PersonaDistillJob.status.in_(("queued", "running")), - ) - .first() - ) - if active: - raise BadRequestError("已有进行中的蒸馏作业，请等待完成或先取消") - - scope = { - "chat_ids": scope_chat_ids, - "hint": (body.hint or "").strip(), - "include_project_memories": body.include_project_memories, - } - job = pds.create_job( - db, - kind="personal", - target_user_id=str(user.user_id), - requested_by=str(user.user_id), - scope=scope, - ) - pds.start_job_background(job.job_id) - return created_response(data=_job_to_dict(job)) - - -@router.get("/jobs", summary="我的蒸馏作业列表") -async def list_jobs( - limit: int = Query(20, ge=1, le=100), - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - jobs = pds.list_jobs( - db, kind="personal", target_user_id=str(user.user_id), limit=limit - ) - return success_response(data={"items": [_job_to_dict(j) for j in jobs], "count": len(jobs)}) - - -@router.get("/jobs/{job_id}", summary="作业详情（含产物）") -async def get_job( - job_id: str, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - job = _get_own_job(db, job_id, user.user_id) - return success_response(data=_job_to_dict(job, include_result=True)) - - -@router.post("/jobs/{job_id}/save", summary="保存产物为我的私有技能") -async def save_job( - job_id: str, - body: SaveJobRequest, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - _ensure_lab_enabled(db, user.user_id) - job = _get_own_job(db, job_id, user.user_id) - skill = pds.save_personal_skill( - db, job, edited_content=body.skill_content, enable=body.enable - ) - return success_response( - data={ - "skill_id": skill.skill_id, - "display_name": skill.display_name, - "is_enabled": skill.is_enabled, - "job": _job_to_dict(job), - } - ) - - -@router.post("/jobs/{job_id}/cancel", summary="取消作业") -async def cancel_job( - job_id: str, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - job = _get_own_job(db, job_id, user.user_id) - job = pds.cancel_job(db, job.job_id) - return success_response(data=_job_to_dict(job)) - - -@router.delete("/jobs/{job_id}", summary="删除作业") -async def delete_job( - job_id: str, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - job = _get_own_job(db, job_id, user.user_id) - pds.delete_job(db, job.job_id) - return success_response(data={"deleted": job_id}) diff --git a/src/backend/api/routes/v1/me.py b/src/backend/api/routes/v1/me.py deleted file mode 100644 index bf9dac96..00000000 --- a/src/backend/api/routes/v1/me.py +++ /dev/null @@ -1,224 +0,0 @@ -"""User-facing (non Config / Admin) team management endpoints: /v1/me/*""" - -from __future__ import annotations - -from typing import Optional - -from fastapi import APIRouter, Depends, HTTPException, Query -from pydantic import BaseModel, Field -from sqlalchemy.orm import Session - -from core.auth.backend import UserContext, get_current_user -from core.auth.roles import at_least, rank, role_label -from core.db.engine import get_db -from core.db.models import LocalUser, Team, UserShadow -from core.db.repository import TeamRepository -from core.infra.responses import success_response -# Seam: team serialization is EE-only — on the CE single-tenant tree this module -# is missing, so team endpoints degrade to 404 -try: - from core.services.team_service import serialize_team_membership -except ModuleNotFoundError: - serialize_team_membership = None - -router = APIRouter(prefix="/v1/me", tags=["My Profile"]) - - -class InviteBody(BaseModel): - user_id: Optional[str] = None - username: Optional[str] = None - role: str = Field("member", pattern="^(member|admin)$") - - -def _require_team_member(db: Session, team_id: str, user_id: str) -> str: - role = TeamRepository(db).get_member_role(team_id, user_id) - if role is None: - raise HTTPException(status_code=403, detail="你不在该团队中") - return role - - -def _require_team_admin(db: Session, team_id: str, user_id: str) -> str: - role = _require_team_member(db, team_id, user_id) - if not at_least(role, "admin"): - raise HTTPException(status_code=403, detail="需要团队管理员权限") - return role - - -def _team_brief_with_count(team: Team, role: str) -> dict: - if serialize_team_membership is None: - raise HTTPException(status_code=404, detail="团队功能在当前版本不可用") - member_count = len(team.members) if team.members is not None else 0 - return serialize_team_membership(team, role, member_count=member_count) - - -# ── Team list (brief) ───────────────────────────────────────────────── - -@router.get("/teams", summary="我的团队列表") -async def my_teams( - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """列出当前用户加入的所有团队（含角色与成员数的简要信息）。""" - team_repo = TeamRepository(db) - rows = team_repo.list_for_user(user.user_id) - teams = [_team_brief_with_count(t, r) for t, r in rows] - return success_response(data={"items": teams, "total": len(teams)}) - - -# ── Single team detail ────────────────────────────────────────────────────── - -@router.get("/teams/{team_id}", summary="团队详情") -async def team_detail( - team_id: str, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """获取指定团队的详情（含当前用户角色与成员数）。需为该团队成员。""" - my_role = _require_team_member(db, team_id, user.user_id) - team = db.query(Team).filter(Team.team_id == team_id).first() - if not team: - raise HTTPException(status_code=404, detail="团队不存在") - return success_response(data=_team_brief_with_count(team, my_role)) - - -@router.get("/teams/{team_id}/members", summary="团队成员列表") -async def team_members( - team_id: str, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """列出指定团队的成员（含角色、加入时间，并标记是否为本人）。需为该团队成员。""" - my_role = _require_team_member(db, team_id, user.user_id) - team_repo = TeamRepository(db) - rows = team_repo.list_members(team_id) - items = [] - for member, shadow in rows: - items.append( - { - "user_id": shadow.user_id, - "username": shadow.username, - "avatar_url": shadow.avatar_url, - "role": member.role, - "joined_at": member.joined_at.isoformat() if member.joined_at else None, - "is_self": shadow.user_id == user.user_id, - } - ) - return success_response(data={"items": items, "my_role": my_role}) - - -# ── Invite member (admin / owner) ─────────────────────────────────────── - -@router.post("/teams/{team_id}/members", summary="邀请成员加入团队") -async def invite_member( - team_id: str, - body: InviteBody, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """按 user_id 或 username 邀请用户加入团队。需团队管理员；仅所有者可直接授予 admin 角色。""" - my_role = _require_team_admin(db, team_id, user.user_id) - - target: Optional[UserShadow] = None - if body.user_id: - target = db.query(UserShadow).filter(UserShadow.user_id == body.user_id).first() - elif body.username: - target = db.query(UserShadow).filter(UserShadow.username == body.username.strip()).first() - if target is None: - raise HTTPException(status_code=404, detail="未找到该用户，请先注册") - - requested_role = body.role - if requested_role == "admin" and my_role != "owner": - raise HTTPException(status_code=403, detail="仅团队所有者可将成员设为管理员") - - team_repo = TeamRepository(db) - existing = team_repo.get_member_role(team_id, target.user_id) - if existing: - raise HTTPException(status_code=409, detail=f"该用户已是团队{role_label(existing)}") - - team_repo.add_member(team_id, target.user_id, requested_role) - - return success_response( - data={ - "team_id": team_id, - "user_id": target.user_id, - "username": target.username, - "role": requested_role, - } - ) - - -# ── Kick member / voluntary leave ───────────────────────────────────────────────── - -@router.delete("/teams/{team_id}/members/{member_user_id}", summary="移除成员/退出团队") -async def remove_member( - team_id: str, - member_user_id: str, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """移除团队成员或本人退出团队。移除他人需管理员且角色高于对方；唯一所有者退出前须先转让所有权。""" - my_role = _require_team_member(db, team_id, user.user_id) - team_repo = TeamRepository(db) - target_role = team_repo.get_member_role(team_id, member_user_id) - if target_role is None: - raise HTTPException(status_code=404, detail="该成员不在团队中") - - is_self = member_user_id == user.user_id - - if is_self: - if my_role == "owner": - from core.db.models import TeamMember - - owner_count = ( - db.query(TeamMember) - .filter(TeamMember.team_id == team_id, TeamMember.role == "owner") - .count() - ) - if owner_count <= 1: - raise HTTPException(status_code=409, detail="你是唯一所有者，请先转让所有权再退出") - else: - if not at_least(my_role, "admin"): - raise HTTPException(status_code=403, detail="需要团队管理员权限") - if rank(my_role) <= rank(target_role): - raise HTTPException(status_code=403, detail=f"无法移除{role_label(target_role)}") - - team_repo.remove_member(team_id, member_user_id) - return success_response( - data={"team_id": team_id, "user_id": member_user_id, "self_leave": is_self} - ) - - -# ── User search (for finding invite targets) ────────────────────────────────── - -@router.get("/users/search", summary="搜索用户（用于邀请）") -async def search_users( - q: str = Query(..., min_length=2, max_length=64, description="用户名或真实姓名，≥2 字"), - limit: int = Query(10, ge=1, le=20), - user: UserContext = Depends(get_current_user), # noqa: ARG001 — login required only - db: Session = Depends(get_db), -): - """按用户名或真实姓名模糊搜索用户，供邀请成员时查找目标。仅要求已登录。""" - pattern = f"%{q.strip()}%" - query = ( - db.query(UserShadow, LocalUser) - .outerjoin(LocalUser, LocalUser.user_id == UserShadow.user_id) - .filter( - (UserShadow.username.ilike(pattern)) - | (LocalUser.real_name.ilike(pattern)) - ) - .order_by(UserShadow.username) - .limit(limit) - ) - items = [] - for shadow, local in query.all(): - items.append( - { - "user_id": shadow.user_id, - "username": shadow.username, - "real_name": local.real_name if local else None, - "avatar_url": shadow.avatar_url, - } - ) - return success_response(data={"items": items}) - - diff --git a/src/backend/api/routes/v1/me_system.py b/src/backend/api/routes/v1/me_system.py index dcd1e87c..7d93b93b 100644 --- a/src/backend/api/routes/v1/me_system.py +++ b/src/backend/api/routes/v1/me_system.py @@ -52,7 +52,7 @@ def _personal_groups() -> Dict[str, str]: """Return the edition-specific settings surface. - CE provides only owner-isolated local knowledge bases, so the shared/Dify + CE provides only owner-isolated local knowledge bases, so the external knowledge-base connector is not configurable there. """ if settings.edition.edition == "ce": diff --git a/src/backend/api/routes/v1/memories.py b/src/backend/api/routes/v1/memories.py index fc7c6a13..0b5ada64 100644 --- a/src/backend/api/routes/v1/memories.py +++ b/src/backend/api/routes/v1/memories.py @@ -12,22 +12,17 @@ from typing import Optional -from fastapi import APIRouter, Depends, Query -from sqlalchemy.orm import Session -from pydantic import BaseModel - -from core.auth.backend import get_current_user, UserContext -from core.db.engine import get_db +from core.auth.backend import UserContext, get_current_user from core.config.settings import settings as _jx_settings +from core.db.engine import get_db +from core.infra.responses import error_response, success_response from core.memory.profile import get as profile_get -from core.memory.service import ( - get_all_memories, - delete_memory, - delete_all_memories, -) -from core.infra.responses import success_response, error_response +from core.memory.service import delete_all_memories, delete_memory, get_all_memories from core.services import UserService from core.services.memory_settings_service import MemorySettingsService +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel +from sqlalchemy.orm import Session router = APIRouter(prefix="/v1/memories", tags=["memories"]) @@ -40,16 +35,19 @@ class MemorySettingsRequest(BaseModel): # ── Register fixed paths first so they aren't mis-matched by /{memory_id} ── + def _is_reranker_available() -> bool: """Check if reranker endpoint is configured at the infra level.""" try: from core.services.model_config import ModelConfigService + cfg = ModelConfigService.get_instance().resolve("reranker") if cfg and cfg.base_url and cfg.model_name: return True except Exception: pass import os + return bool(os.getenv("RERANKER_URL") and os.getenv("RERANKER_MODEL")) @@ -62,13 +60,15 @@ async def get_memory_settings( svc = UserService(db) settings = svc.get_user_settings(str(user.user_id)) availability = MemorySettingsService(db).availability() - return success_response(data={ - "memory_enabled": settings.get("memory_enabled", False), - "memory_write_enabled": settings.get("memory_write_enabled", False), - **availability, - "reranker_enabled": settings.get("reranker_enabled", False), - "reranker_available": _is_reranker_available(), - }) + return success_response( + data={ + "memory_enabled": settings.get("memory_enabled", False), + "memory_write_enabled": settings.get("memory_write_enabled", False), + **availability, + "reranker_enabled": settings.get("reranker_enabled", False), + "reranker_available": _is_reranker_available(), + } + ) @router.patch("/settings", summary="更新记忆设置") @@ -89,18 +89,31 @@ async def update_memory_settings( if patch: MemorySettingsService(db).validate_patch(patch) svc.update_user_metadata(user_id=str(user.user_id), patch=patch) - return success_response(data={ - **({"memory_enabled": body.memory_enabled} if body.memory_enabled is not None else {}), - **({"memory_write_enabled": body.memory_write_enabled} if body.memory_write_enabled is not None else {}), - **({"reranker_enabled": body.reranker_enabled} if body.reranker_enabled is not None else {}), - }) + return success_response( + data={ + **({"memory_enabled": body.memory_enabled} if body.memory_enabled is not None else {}), + **( + {"memory_write_enabled": body.memory_write_enabled} + if body.memory_write_enabled is not None + else {} + ), + **( + {"reranker_enabled": body.reranker_enabled} + if body.reranker_enabled is not None + else {} + ), + } + ) # ── List / clear / delete single ──────────────────────────────── + @router.get("", summary="查询事实记忆列表") async def list_memories( - project_id: Optional[str] = Query(None, description="若指定，只返回属于该项目 workspace 的记忆"), + project_id: Optional[str] = Query( + None, description="若指定，只返回属于该项目 workspace 的记忆" + ), user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): @@ -118,23 +131,14 @@ async def list_memories( if not _jx_settings.memory.enabled: return success_response(data={"enabled": False, "items": [], "count": 0}) - # Resolve the workspace-level read switch + compute the mem0 scope_user_id - # (for team projects scope = "team:" enables sharing; personal projects - # and the default space use the real user_id) scope_user_id = str(user.user_id) if project_id: - from core.db.models import Project as _Project - _p = ( - db.query(_Project) - .filter(_Project.project_id == project_id, _Project.deleted_at.is_(None)) - .first() - ) - # Missing project → treat as disabled, avoiding returning data for an unknown project_id - if _p is None: + from core.services.project_scope import project_memory_policy + + policy = project_memory_policy(db, project_id, scope_user_id) + if policy is None: return success_response(data={"enabled": False, "items": [], "count": 0}) - ws_enabled = bool((_p.extra_data or {}).get("memory_enabled", True)) - if _p.kind == "team" and _p.team_id: - scope_user_id = f"team:{_p.team_id}" + ws_enabled, scope_user_id = policy else: settings = UserService(db).get_user_settings(str(user.user_id)) ws_enabled = bool(settings.get("memory_enabled", False)) @@ -155,7 +159,8 @@ async def list_memories( else: raw_items = await get_all_memories(scope_user_id) filtered = [ - it for it in raw_items + it + for it in raw_items if ((it.get("metadata") or {}).get("workspace_id") or "default") == "default" ] items = [_flatten_fact_metadata(it) for it in filtered] @@ -165,9 +170,7 @@ async def list_memories( def _flatten_fact_metadata(item: dict) -> dict: """Flatten a mem0 item's metadata fields to the top level; unknown fields pass through as-is. - In team projects ``author_user_id`` denotes the memory's real author; in personal - projects and the default space it usually equals the top-level user_id. May be - missing in legacy data. + ``author_user_id`` denotes the memory's real author and may be missing in legacy data. """ if not isinstance(item, dict): return item @@ -204,6 +207,7 @@ async def remove_memory(memory_id: str, user: UserContext = Depends(get_current_ # ── Layered memory details ──────────────────────────────────────────────── + @router.get("/profile", summary="查询用户档案记忆") async def get_profile_memory( user: UserContext = Depends(get_current_user), @@ -214,21 +218,25 @@ async def get_profile_memory( 返回：{ enabled, workspace_id, content_md, length, max_chars } """ if not _jx_settings.memory.enabled: - return success_response(data={ - "enabled": False, + return success_response( + data={ + "enabled": False, + "workspace_id": workspace_id, + "content_md": "", + "length": 0, + "max_chars": _jx_settings.memory.profile_max_chars, + } + ) + content = await profile_get(str(user.user_id), workspace_id) + return success_response( + data={ + "enabled": True, "workspace_id": workspace_id, - "content_md": "", - "length": 0, + "content_md": content or "", + "length": len(content or ""), "max_chars": _jx_settings.memory.profile_max_chars, - }) - content = await profile_get(str(user.user_id), workspace_id) - return success_response(data={ - "enabled": True, - "workspace_id": workspace_id, - "content_md": content or "", - "length": len(content or ""), - "max_chars": _jx_settings.memory.profile_max_chars, - }) + } + ) @router.get("/audit", summary="查询记忆审计记录") @@ -236,7 +244,9 @@ async def list_memory_audit( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), limit: int = Query(50, ge=1, le=500, description="返回行数上限"), - action: str | None = Query(None, description="按 action 过滤：read/write/update/delete/write_rejected/forget"), + action: str | None = Query( + None, description="按 action 过滤：read/write/update/delete/write_rejected/forget" + ), layer: str | None = Query(None, description="按 layer 过滤：L1/L2/L3/session"), ): """审计记录：谁在什么时间对自己的记忆做了什么操作。 @@ -247,6 +257,7 @@ async def list_memory_audit( return success_response(data={"enabled": False, "items": [], "count": 0}) from core.db.models import MemoryAudit + q = db.query(MemoryAudit).filter(MemoryAudit.user_id == str(user.user_id)) if action: q = q.filter(MemoryAudit.action == action) @@ -254,19 +265,22 @@ async def list_memory_audit( q = q.filter(MemoryAudit.layer == layer) rows = q.order_by(MemoryAudit.ts.desc()).limit(limit).all() - items = [{ - "id": r.id, - "ts": r.ts.isoformat() if r.ts else None, - "actor": r.actor, - "action": r.action, - "layer": r.layer, - "memory_id": r.memory_id, - "workspace_id": r.workspace_id, - "chat_id": r.chat_id, - "confidentiality": r.confidentiality, - "content_hash": r.content_hash, - "reason": r.reason, - } for r in rows] + items = [ + { + "id": r.id, + "ts": r.ts.isoformat() if r.ts else None, + "actor": r.actor, + "action": r.action, + "layer": r.layer, + "memory_id": r.memory_id, + "workspace_id": r.workspace_id, + "chat_id": r.chat_id, + "confidentiality": r.confidentiality, + "content_hash": r.content_hash, + "reason": r.reason, + } + for r in rows + ] return success_response(data={"enabled": True, "items": items, "count": len(items)}) diff --git a/src/backend/api/routes/v1/meta.py b/src/backend/api/routes/v1/meta.py index 9829f170..8d5b3b1a 100644 --- a/src/backend/api/routes/v1/meta.py +++ b/src/backend/api/routes/v1/meta.py @@ -1,26 +1,12 @@ -"""Edition / license probe (shared by CE/EE, no auth—— only exposes the edition, mode, and feature-flag boolean map). +"""Public build-edition probe.""" -An unauthenticated endpoint; it must never return license details -(license_id/customer name/seats/expiry date)—— those fields are only exposed by the -CONFIG_TOKEN-authenticated /v1/config/license. ``mode`` is reserved for scenarios -like the login page hinting "license has expired". -""" - -from fastapi import APIRouter - -from core.config.settings import settings +from api.middleware.edition import edition_probe_payload from core.infra.responses import success_response -from core.licensing import license_manager +from fastapi import APIRouter router = APIRouter(prefix="/v1/meta", tags=["meta"]) @router.get("/edition", summary="当前部署的版本与能力位") async def get_edition(): - return success_response( - data={ - "edition": settings.edition.edition, - "mode": license_manager.mode(), - "features": license_manager.features_map(), - } - ) + return success_response(data=edition_probe_payload()) diff --git a/src/backend/api/routes/v1/mock_sso.py b/src/backend/api/routes/v1/mock_sso.py index 27171e1a..953e3f4e 100644 --- a/src/backend/api/routes/v1/mock_sso.py +++ b/src/backend/api/routes/v1/mock_sso.py @@ -25,7 +25,7 @@ router = APIRouter(prefix="/mock-sso", tags=["Mock SSO"]) # ── Mock-SSO ticket store ───────────────────────────────────────────────── -# Moved to core.auth.mock_ticket_store so core.auth.sso can validate tickets +# Moved to core.auth.mock_ticket_store so edition authentication can validate tickets # without importing this route module. Re-exported under original names. from core.auth.mock_ticket_store import consume_ticket from core.auth.mock_ticket_store import generate_ticket as _generate_ticket # noqa: E402 diff --git a/src/backend/api/routes/v1/myspace_folders.py b/src/backend/api/routes/v1/myspace_folders.py index 35fdec26..fcc8fd1f 100644 --- a/src/backend/api/routes/v1/myspace_folders.py +++ b/src/backend/api/routes/v1/myspace_folders.py @@ -1,7 +1,6 @@ """My Space · personal folders API. -Mirrors the shape of /v1/teams/{team_id}/folders/* but with the scope switched to "the -current logged-in user's personal space". All routes take the current user via +All routes operate on the current logged-in user's personal space and take the user via Depends(get_current_user); accepting user_id from path/body is forbidden. Endpoints: @@ -19,14 +18,13 @@ from typing import Optional -from fastapi import APIRouter, Depends, HTTPException, Query -from pydantic import BaseModel, Field -from sqlalchemy.orm import Session - from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from core.infra.responses import success_response from core.services.user_folder_service import UserFolderService +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session router = APIRouter(prefix="/v1/myspace/folders", tags=["MySpace Folders"]) @@ -57,18 +55,20 @@ async def list_folders( user_id = str(user.user_id) if as_ == "flat": folders = service.list_by_user(user_id) - return success_response(data={ - "items": [ - { - "folder_id": f.folder_id, - "user_id": f.user_id, - "parent_folder_id": f.parent_folder_id, - "name": f.name, - "created_at": f.created_at.isoformat() if f.created_at else None, - } - for f in folders - ] - }) + return success_response( + data={ + "items": [ + { + "folder_id": f.folder_id, + "user_id": f.user_id, + "parent_folder_id": f.parent_folder_id, + "name": f.name, + "created_at": f.created_at.isoformat() if f.created_at else None, + } + for f in folders + ] + } + ) return success_response(data={"tree": service.get_tree(user_id)}) @@ -182,7 +182,9 @@ async def move_artifact_to_folder( if not result.ok: # Distinguishing 401/404/400 adds little value; use a uniform 400 so the frontend toast passes through message raise HTTPException(status_code=400, detail=result.message) - return success_response(data={"artifact_id": body.artifact_id, "folder_id": body.folder_id}, message=result.message) + return success_response( + data={"artifact_id": body.artifact_id, "folder_id": body.folder_id}, message=result.message + ) @router.post("/copy-artifact", summary="复制个人文件到文件夹") diff --git a/src/backend/api/routes/v1/projects.py b/src/backend/api/routes/v1/projects.py index b17fe1f7..fc3b8d90 100644 --- a/src/backend/api/routes/v1/projects.py +++ b/src/backend/api/routes/v1/projects.py @@ -1,8 +1,6 @@ """Projects (Claude-style 工作空间) API — CE 子集（split：§5.2）. -社区版为单租户：只有个人项目（项目 ↔ user_folders 强挂钩）。 -团队项目（kind=team、/teams 目标列表、team_folders 挂钩）属商业版 -多租户能力，不在社区版提供。 +社区版为单租户，只提供与个人文件夹绑定的个人项目。 端点： @@ -24,10 +22,6 @@ from typing import Literal, Optional -from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile, status -from pydantic import BaseModel, Field -from sqlalchemy.orm import Session - from core.auth.backend import UserContext, get_current_user from core.auth.permissions_iface import ProjectAccess, require_project_access from core.db.engine import get_db @@ -35,6 +29,9 @@ from core.infra.responses import created_response, paginated_response, success_response from core.services.project_file_service import ProjectFileService from core.services.project_service import ProjectService +from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile, status +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session router = APIRouter(prefix="/v1/projects", tags=["projects"]) @@ -206,12 +203,14 @@ async def list_project_files( """列出项目文件（递归遍历挂钩文件夹子树），附带容量已用 / 上限。""" pf_svc = ProjectFileService(db) items = pf_svc.list_files(access.project) - return success_response(data={ - "items": items, - "total": len(items), - "capacity_used": pf_svc.capacity_used(access.project), - "capacity_limit": pf_svc.capacity_limit(), - }) + return success_response( + data={ + "items": items, + "total": len(items), + "capacity_used": pf_svc.capacity_used(access.project), + "capacity_limit": pf_svc.capacity_limit(), + } + ) @router.post("/{project_id}/files/upload", summary="直传文件到项目（写入挂钩文件夹）") diff --git a/src/backend/api/routes/v1/sites.py b/src/backend/api/routes/v1/sites.py index 86df1a2b..9b2dd324 100644 --- a/src/backend/api/routes/v1/sites.py +++ b/src/backend/api/routes/v1/sites.py @@ -7,27 +7,24 @@ from typing import Optional -from fastapi import APIRouter, Depends, Query -from pydantic import BaseModel, Field -from sqlalchemy.orm import Session - from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from core.infra.responses import paginated_response, success_response +from core.services.site_access_policy import ( + SiteUpdateScopeFields, + serialize_site_scope, + site_scope_ref, +) from core.services.site_service import SiteService +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session router = APIRouter(prefix="/v1/sites", tags=["Sites"]) -class UpdateSiteRequest(BaseModel): +class UpdateSiteRequest(SiteUpdateScopeFields): title: Optional[str] = Field(None, description="站点标题", max_length=200) - visibility: Optional[str] = Field( - None, description="可见性：public / private / team", - pattern="^(public|private|team)$", - ) - team_id: Optional[str] = Field( - None, description="visibility=team 时的授权团队", max_length=64 - ) slug: Optional[str] = Field(None, description="访问地址 slug", max_length=64) description: Optional[str] = Field(None, description="站点描述", max_length=2000) @@ -43,8 +40,7 @@ def _site_to_dict(site) -> dict: "url": f"/site/{site.slug}/", "title": site.title, "description": site.description, - "visibility": site.visibility, - "team_id": site.team_id, + **serialize_site_scope(site), "entry_file": site.entry_file, "current_version": site.current_version, "file_count": site.file_count, @@ -104,7 +100,7 @@ async def update_site( visibility=body.visibility, slug=body.slug, description=body.description, - team_id=body.team_id, + scope_id=site_scope_ref(body), ) return success_response(data=_site_to_dict(site)) @@ -178,17 +174,19 @@ async def list_site_kv( service = SiteService(db) site = service.get_owned(site_id, user.user_id) rows = service.repo.kv_list(site.site_id) - return success_response(data={ - "items": [ - { - "key": r.k, - "value": r.v, - "updated_at": r.updated_at.isoformat() if r.updated_at else None, - } - for r in rows - ], - "total": service.repo.kv_count(site.site_id), - }) + return success_response( + data={ + "items": [ + { + "key": r.k, + "value": r.v, + "updated_at": r.updated_at.isoformat() if r.updated_at else None, + } + for r in rows + ], + "total": service.repo.kv_count(site.site_id), + } + ) @router.delete("/{site_id}/kv/{key}", summary="删除站点 KV 键") diff --git a/src/backend/api/routes/v1/users.py b/src/backend/api/routes/v1/users.py index 056b510f..c3dad277 100644 --- a/src/backend/api/routes/v1/users.py +++ b/src/backend/api/routes/v1/users.py @@ -73,11 +73,11 @@ class UserPreferences(BaseModel): ) -@router.get("/me", summary="获取当前用户信息（含部门、团队、本地账号资料）") +@router.get("/me", summary="获取当前用户资料") async def get_current_user_info( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db) ): - """获取当前登录用户信息，包含部门、所属团队及本地账号资料（昵称/真实姓名/电话）。""" + """获取当前登录用户的账号与个人资料。""" user_repo = UserRepository(db) user_shadow = user_repo.get_by_id(user.user_id) @@ -86,15 +86,7 @@ async def get_current_user_info( from core.db.models import LocalUser - try: - from core.services.team_service import list_user_teams_brief - except ( - ModuleNotFoundError - ): # CE: single-tenant has no teams -- only degrade the teams field, user info returns as usual - list_user_teams_brief = None - local = db.query(LocalUser).filter(LocalUser.user_id == user.user_id).first() - teams = list_user_teams_brief(db, user.user_id) if list_user_teams_brief else [] meta = dict(user_shadow.extra_data or {}) from core.services.local_user_service import ce_onboarding_required @@ -113,10 +105,13 @@ async def get_current_user_info( "auth_source": "local" if local else "external", "must_change_password": bool(meta.get("must_change_password")), "onboarding_required": ce_onboarding_required(meta), - "teams": teams, "created_at": user_shadow.created_at.isoformat(), } + from core.auth.account_view import extend_current_account + + data = extend_current_account(db, str(user.user_id), data) + return success_response(data=data, message="User information retrieved successfully") @@ -398,7 +393,7 @@ async def get_user_avatar_raw( _user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - """同源 cookie 鉴权后任意登录用户都可读取（用于团队成员之间互相看到头像）。""" + """同源 cookie 鉴权后，已登录用户可读取其他用户的头像。""" user_repo = UserRepository(db) target = user_repo.get_by_id(user_id) if not target: diff --git a/src/backend/api/schemas.py b/src/backend/api/schemas.py index 656736dd..a4f7060b 100644 --- a/src/backend/api/schemas.py +++ b/src/backend/api/schemas.py @@ -194,31 +194,3 @@ class HealthResponse(BaseModel): status: str service: str timestamp: str - - -class KBGrantItem(BaseModel): - """单条知识库授权项（用户/团队管理页共用）。""" - - resource_id: str = Field(..., max_length=64) - resource_type: str = Field("local", pattern="^(local|dify)$") - level: str = Field("view", pattern="^(view|edit|admin)$") - - -class KBGrantsBody(BaseModel): - """全量替换语义：提交后该用户/团队的知识库授权 = grants 列表。""" - - grants: List[KBGrantItem] = Field(default_factory=list) - - -class MarketVisibilityGrantItem(BaseModel): - """单条市场可见范围授权（三大市场 admin 端共用）。""" - - principal_type: str = Field(..., pattern="^(user|team|role)$") - principal_id: str = Field(..., min_length=1, max_length=64) - - -class MarketVisibilityRequest(BaseModel): - """全量替换语义：public=全员可见（grants 忽略）；scoped=仅 grants 白名单可见。""" - - visibility: str = Field(..., pattern="^(public|scoped)$") - grants: List[MarketVisibilityGrantItem] = Field(default_factory=list) diff --git a/src/backend/cli.py b/src/backend/cli.py index 13b2ed8b..93f5c7a0 100644 --- a/src/backend/cli.py +++ b/src/backend/cli.py @@ -76,6 +76,11 @@ def apply_local_env(port: int) -> dict: "PLAYWRIGHT_BROWSERS_PATH": str(dd / "node" / "browsers"), "JX_FONT_DIR": str(dd / "fonts"), "MCP_HOST": "127.0.0.1", + # A local/desktop install is expected to be useful immediately after a + # zero-state boot. Keep the three credential-free first-party plugins + # as the local-profile default even when the caller is plain + # ``hugagent serve`` rather than one of the installer wrappers. + "HUGAGENT_BOOTSTRAP_DEFAULT_PLUGINS": "1", "STORAGE_TYPE": "local", "STORAGE_PATH": str(dd / "storage"), "LOG_FILE_PATH": str(dd / "logs" / "backend.log"), @@ -275,6 +280,8 @@ def mark_web_onboarding_complete(user_id: str) -> None: # scheduling, skill authoring/market, and conversational site-building. Others # (IM / email / low-code) need per-user credentials, so they're opt-in only. _DEFAULT_PLUGINS = ["automation", "skill-manager", "sites"] +_DEFAULT_PLUGINS_MARKER = ".default-plugins-v1" +_DEFAULT_PLUGIN_BOOTSTRAP_ENV = "HUGAGENT_BOOTSTRAP_DEFAULT_PLUGINS" def list_installable_plugins() -> list: @@ -312,6 +319,37 @@ def install_plugins(slugs: list) -> list: return done +def ensure_default_plugins_once() -> bool: + """Install local-install defaults once without overriding later user choices. + + The marker intentionally lives beside the local SQLite database instead of + being inferred from installed rows: after the first bootstrap, uninstalling + a default plugin is an explicit user choice and must survive future desktop + restarts and upgrades. A failed or partial install leaves no marker, so the + next launch retries the complete default set. + + Returns ``True`` when this call performed the first successful bootstrap. + """ + marker = data_dir() / _DEFAULT_PLUGINS_MARKER + if marker.is_file(): + # Plugin choices stay untouched after first bootstrap, while the bundled + # site template may still receive compatible fixes in desktop upgrades. + provision_site_template(verbose=False) + return False + + installed = install_plugins(_DEFAULT_PLUGINS) + missing = [slug for slug in _DEFAULT_PLUGINS if slug not in installed] + if missing: + raise RuntimeError(f"默认插件安装不完整：{', '.join(missing)}") + if not provision_site_template(verbose=True): + raise RuntimeError("站点模板初始化失败") + + temporary = marker.with_name(f"{marker.name}.{os.getpid()}.tmp") + temporary.write_text("\n".join(_DEFAULT_PLUGINS) + "\n", encoding="utf-8") + os.replace(temporary, marker) + return True + + def _select_plugins_interactively(available: list) -> list: """Show the plugin menu, return the slugs the user picked.""" print("\n[插件] 选择要安装的插件（可在插件市场随时增减）") @@ -693,6 +731,15 @@ def _open_browser_when_ready(port: int) -> None: def cmd_serve(args) -> int: apply_local_env(args.port) _ensure_schema_and_seed() + if os.getenv(_DEFAULT_PLUGIN_BOOTSTRAP_ENV) == "1": + try: + if ensure_default_plugins_once(): + print("✓ 默认插件已就绪：定时任务、技能管理、站点发布") + except Exception as exc: # noqa: BLE001 + # The desktop readiness endpoint must not go healthy with only a + # partial default-plugin set. No marker is written on failure, so + # the next installer-managed start will retry the full bootstrap. + raise RuntimeError(f"默认插件初始化失败，下次启动将重试：{exc}") from exc import uvicorn from api.app import app diff --git a/src/backend/core/agent_skills/registry.py b/src/backend/core/agent_skills/registry.py index f21c380b..700fc7ec 100644 --- a/src/backend/core/agent_skills/registry.py +++ b/src/backend/core/agent_skills/registry.py @@ -15,7 +15,6 @@ from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple - _SKILLS_ROOT = Path(__file__).resolve().parent.parent / "skill_bundles" _ID_RE = re.compile(r"^[a-z0-9_-]{1,63}$") @@ -23,6 +22,7 @@ @dataclass(frozen=True) class AgentSkillMetadata: """Lightweight skill metadata (no instructions loaded).""" + id: str name: str description: str @@ -36,6 +36,7 @@ class AgentSkillMetadata: @dataclass(frozen=True) class AgentSkillSpec: """Full skill spec with instructions.""" + id: str name: str description: str @@ -46,10 +47,12 @@ class AgentSkillSpec: tags: List[str] = field(default_factory=list) allowed_tools: List[str] = field(default_factory=list) # Tool filtering support mcp_server_ids: List[str] = field(default_factory=list) - extra_files: List[str] = field(default_factory=list) # Available resource file names - base_dir: str = "" # Materialized directory path (for {baseDir} substitution) + extra_files: List[str] = field(default_factory=list) # Available resource file names + base_dir: str = "" # Materialized directory path (for {baseDir} substitution) examples: List[Dict[str, Any]] = field(default_factory=list) - executable_scripts: List[Dict[str, Any]] = field(default_factory=list) # Scripts declared in _scripts.json + executable_scripts: List[Dict[str, Any]] = field( + default_factory=list + ) # Scripts declared in _scripts.json skill_path: str = "" @@ -65,7 +68,10 @@ def _require_id(value: str) -> str: def _split_frontmatter(raw: str) -> Tuple[Dict[str, str], str]: - text = raw or "" + # ZIP uploads are commonly created on Windows, where Markdown files use + # CRLF line endings. Normalize all newline styles before matching the + # frontmatter delimiters so a valid ``---`` header is platform-agnostic. + text = (raw or "").replace("\r\n", "\n").replace("\r", "\n") if not text.startswith("---\n"): raise SkillSpecError("SKILL.md missing YAML frontmatter") @@ -368,16 +374,18 @@ def parse_scripts_json(raw: str) -> List[Dict[str, Any]]: raise SkillSpecError(f"_scripts.json[{idx}] 必须是对象") if "name" not in item: raise SkillSpecError(f"_scripts.json[{idx}] 缺少 name 字段") - scripts.append({ - "name": item["name"], - "description": item.get("description", ""), - "language": item.get("language", "python"), - "timeout": min(int(item.get("timeout", 30)), 120), - "params_schema": item.get("params_schema"), - # "stdin_json" (default): params sent via stdin as JSON - # "cli_args": params converted to CLI arguments (_args) - "input_mode": item.get("input_mode", "stdin_json"), - }) + scripts.append( + { + "name": item["name"], + "description": item.get("description", ""), + "language": item.get("language", "python"), + "timeout": min(int(item.get("timeout", 30)), 120), + "params_schema": item.get("params_schema"), + # "stdin_json" (default): params sent via stdin as JSON + # "cli_args": params converted to CLI arguments (_args) + "input_mode": item.get("input_mode", "stdin_json"), + } + ) return scripts diff --git a/src/backend/core/auth/__init__.py b/src/backend/core/auth/__init__.py index 475cc860..86edd465 100644 --- a/src/backend/core/auth/__init__.py +++ b/src/backend/core/auth/__init__.py @@ -1 +1,20 @@ -from core.auth.backend import AuthService, UserContext, get_current_user, require_auth # noqa: F401 +"""Authentication package with lazy compatibility exports. + +Importing an edition policy module must not eagerly import ``backend``: the backend +depends on the service package, while service admission policies live below this +package. Lazy attributes keep the historical ``from core.auth import ...`` API +without recreating that cycle. +""" + +from importlib import import_module + +_BACKEND_EXPORTS = {"AuthService", "UserContext", "get_current_user", "require_auth"} + + +def __getattr__(name: str): + if name in _BACKEND_EXPORTS: + return getattr(import_module("core.auth.backend"), name) + raise AttributeError(name) + + +__all__ = sorted(_BACKEND_EXPORTS) diff --git a/src/backend/core/auth/account_policy.py b/src/backend/core/auth/account_policy.py new file mode 100644 index 00000000..1454ce94 --- /dev/null +++ b/src/backend/core/auth/account_policy.py @@ -0,0 +1,52 @@ +"""Single-tenant account policy for the community edition.""" + +from core.infra.exceptions import AppException + + +class AccountCapacityExceeded(AppException): + """Neutral account-admission exception; CE never raises it.""" + + +def account_capacity_block_reason(db) -> None: + return None + + +def validate_registration_credential(db, credential: str): + return True, None, None + + +def claim_registration_credential(db, credential: str, user_id: str): + return True, None + + +def registration_credential_id(validated) -> None: + return None + + +def add_invited_account_to_scope(db, validated, user_id: str) -> None: + return None + + +def validate_account_scope(db, scope_id) -> bool: + return scope_id is None + + +def add_account_to_scope(db, scope_id, user_id: str, role: str) -> None: + return None + + +def list_account_scopes(db, user_id: str) -> list: + return [] + + +__all__ = [ + "AccountCapacityExceeded", + "account_capacity_block_reason", + "add_account_to_scope", + "add_invited_account_to_scope", + "claim_registration_credential", + "list_account_scopes", + "registration_credential_id", + "validate_account_scope", + "validate_registration_credential", +] diff --git a/src/backend/core/auth/account_view.py b/src/backend/core/auth/account_view.py new file mode 100644 index 00000000..26dff3e4 --- /dev/null +++ b/src/backend/core/auth/account_view.py @@ -0,0 +1,8 @@ +"""Community current-account response has no organization fields.""" + + +def extend_current_account(db, user_id: str, data: dict) -> dict: + return data + + +__all__ = ["extend_current_account"] diff --git a/src/backend/core/auth/backend.py b/src/backend/core/auth/backend.py index 049d4956..29f09cd4 100644 --- a/src/backend/core/auth/backend.py +++ b/src/backend/core/auth/backend.py @@ -6,25 +6,26 @@ - session: SSO ticket mode, validates jx_session Cookie against Redis """ -from typing import Optional, Dict, Any -from fastapi import Request, HTTPException, Depends -from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials -import requests -from pydantic import BaseModel +from typing import Any, Dict, Optional +import requests +from core.auth.account_policy import AccountCapacityExceeded from core.config.settings import settings from core.db.engine import get_db -from sqlalchemy.orm import Session -from core.services import UserService from core.db.repository import AuditLogRepository from core.infra.logging import get_logger -from core.licensing import SeatLimitExceeded +from core.services import UserService +from fastapi import Depends, HTTPException, Request +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from pydantic import BaseModel +from sqlalchemy.orm import Session logger = get_logger(__name__) class UserContext(BaseModel): """User context injected into requests after authentication.""" + user_id: str user_center_id: str username: str @@ -55,11 +56,7 @@ def verify_token_remote(self, token: str) -> Optional[Dict[str, Any]]: if not self.user_center_url: raise HTTPException( status_code=500, - detail={ - "code": 52002, - "message": "AUTH_API_URL not configured", - "data": {} - } + detail={"code": 52002, "message": "AUTH_API_URL not configured", "data": {}}, ) headers = {"Authorization": f"Bearer {token}"} @@ -67,9 +64,7 @@ def verify_token_remote(self, token: str) -> Optional[Dict[str, Any]]: for attempt in range(self.retry_count): try: response = requests.get( - f"{self.user_center_url}/verify", - headers=headers, - timeout=self.timeout + f"{self.user_center_url}/verify", headers=headers, timeout=self.timeout ) if response.status_code == 200: @@ -86,8 +81,8 @@ def verify_token_remote(self, token: str) -> Optional[Dict[str, Any]]: detail={ "code": 52001, "message": "User center unavailable", - "data": {"error": str(e)} - } + "data": {"error": str(e)}, + }, ) continue @@ -100,13 +95,13 @@ def verify_token_mock(self, token: str) -> Dict[str, Any]: "user_center_id": token, "username": token, "email": f"{token}@mock.local", - "avatar_url": None + "avatar_url": None, } return { "user_center_id": settings.auth.mock_user_id, "username": settings.auth.mock_username, "email": "dev@example.com", - "avatar_url": None + "avatar_url": None, } def verify_token(self, token: str, db: Session = None) -> Dict[str, Any]: @@ -119,14 +114,16 @@ def verify_token(self, token: str, db: Session = None) -> Dict[str, Any]: if db: try: audit_repo = AuditLogRepository(db) - audit_repo.create({ - "user_id": "unknown", - "action": "auth.login.failed", - "resource_type": "user", - "resource_id": "unknown", - "status": "failed", - "details": {"reason": "invalid_or_expired_token"} - }) + audit_repo.create( + { + "user_id": "unknown", + "action": "auth.login.failed", + "resource_type": "user", + "resource_id": "unknown", + "status": "failed", + "details": {"reason": "invalid_or_expired_token"}, + } + ) except Exception as e: logger.warning(f"Failed to log auth failure: {e}") @@ -135,8 +132,8 @@ def verify_token(self, token: str, db: Session = None) -> Dict[str, Any]: detail={ "code": 30002, "message": "Invalid or expired token", - "data": {"login_url": _sso_login_url()} - } + "data": {"login_url": _sso_login_url()}, + }, ) return user_info @@ -216,15 +213,15 @@ async def _resolve_session_user(request: Request) -> Optional[UserContext]: detail={ "code": 30003, "message": "Session expired", - "data": {"login_url": _sso_login_url()} - } + "data": {"login_url": _sso_login_url()}, + }, ) async def get_current_user( request: Request, credentials: Optional[HTTPAuthorizationCredentials] = Depends(security), - db: Session = Depends(get_db) + db: Session = Depends(get_db), ) -> UserContext: """Dependency to get current authenticated user. @@ -256,8 +253,8 @@ async def get_current_user( detail={ "code": 30001, "message": "Authorization required", - "data": {"login_url": _sso_login_url()} - } + "data": {"login_url": _sso_login_url()}, + }, ) # ── 3. mock / remote: Bearer token path ── @@ -270,8 +267,8 @@ async def get_current_user( detail={ "code": 30001, "message": "Authorization header required", - "data": {"login_url": _sso_login_url()} - } + "data": {"login_url": _sso_login_url()}, + }, ) token = "mock_token" else: @@ -284,25 +281,27 @@ async def get_current_user( user_center_id=user_info["user_center_id"], username=user_info["username"], email=user_info.get("email"), - avatar_url=user_info.get("avatar_url") + avatar_url=user_info.get("avatar_url"), ) # Audit log for successful authentication audit_repo = AuditLogRepository(db) - audit_repo.create({ - "user_id": user_shadow.user_id, - "action": "auth.login.success", - "resource_type": "user", - "resource_id": user_shadow.user_id, - "status": "success" - }) + audit_repo.create( + { + "user_id": user_shadow.user_id, + "action": "auth.login.success", + "resource_type": "user", + "resource_id": user_shadow.user_id, + "status": "success", + } + ) return UserContext( user_id=user_shadow.user_id, user_center_id=user_shadow.user_center_id, username=user_shadow.username, email=user_shadow.email, - avatar_url=user_shadow.avatar_url + avatar_url=user_shadow.avatar_url, ) @@ -321,7 +320,10 @@ def private_endpoint(user: UserContext = Depends(require_auth(True))): if required: return get_current_user else: - async def optional_user(request: Request, db: Session = Depends(get_db)) -> Optional[UserContext]: + + async def optional_user( + request: Request, db: Session = Depends(get_db) + ) -> Optional[UserContext]: auth_mode = _auth_mode() # ── Try Cookie session first (all modes) ── @@ -367,7 +369,7 @@ async def optional_user(request: Request, db: Session = Depends(get_db)) -> Opti user_center_id=user_info["user_center_id"], username=user_info["username"], email=user_info.get("email"), - avatar_url=user_info.get("avatar_url") + avatar_url=user_info.get("avatar_url"), ) return UserContext( @@ -375,12 +377,10 @@ async def optional_user(request: Request, db: Session = Depends(get_db)) -> Opti user_center_id=user_shadow.user_center_id, username=user_shadow.username, email=user_shadow.email, - avatar_url=user_shadow.avatar_url + avatar_url=user_shadow.avatar_url, ) - except SeatLimitExceeded: - # Seat/license blocks must be explicitly raised as 402 — silently downgrading to - # anonymous would disguise a seat problem as "some features available", diverging - # from the behavior of required-auth endpoints + except AccountCapacityExceeded: + # Edition admission blocks must not degrade to anonymous access. try: db.rollback() except Exception: diff --git a/src/backend/core/auth/capabilities.py b/src/backend/core/auth/capabilities.py index 58d325c3..7ddff260 100644 --- a/src/backend/core/auth/capabilities.py +++ b/src/backend/core/auth/capabilities.py @@ -1,4 +1,4 @@ -"""Capability-flag resolution: personal explicit → team default → system default. +"""Capability-flag resolution with edition-provided default layers. The system has 6 user capability flags (consistent with the Config admin console "User Management" / "Team Management"): @@ -28,14 +28,13 @@ various ``_require_*`` gates, ``config_users`` display) go through here uniformly, so bare metadata reads cannot bypass team defaults. -Under CE (no teams), ``team_default_permissions_for_user`` returns ``{}`` -and resolution degrades to "personal → system default", i.e. exactly the -same as before this feature was introduced. +CE supplies no organization-scoped layers, so resolution is simply personal +explicit values followed by system defaults. """ from __future__ import annotations -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional from core.config.settings import settings from sqlalchemy.orm import Session @@ -98,75 +97,6 @@ def page_admin_flags( return {flag: bool(is_super or caps.get(flag)) for flag in PAGE_ADMIN_FLAGS} -def normalize_team_permissions(payload: Any) -> Dict[str, Any]: - """Normalize team default permissions: keep only the 6 valid keys, coerce types, drop the rest. - - The returned dict contains only flags the team **explicitly imposes** - (PUT full-replacement semantics): - - - The 5 boolean flags: values coerced to ``bool``; missing key = the - team does not impose that flag. - - ``allowed_apps``: ``list`` → deduplicated list of strings (empty list - = restricted to "none"); missing key = the team does not restrict app - visibility. - """ - if not isinstance(payload, dict): - return {} - out: Dict[str, Any] = {} - for key in BOOL_CAPABILITY_DEFAULTS: - if key in payload and payload[key] is not None: - out[key] = bool(payload[key]) - raw_apps = payload.get("allowed_apps") - if isinstance(raw_apps, list): - out["allowed_apps"] = list(dict.fromkeys(str(x) for x in raw_apps)) - return out - - -def merge_team_permissions(raw_perms_list: Any) -> Dict[str, Any]: - """Merge multiple teams' default permissions (union / most permissive across teams). - - The argument is an iterable of the raw ``default_permissions`` values of - each team. Only flags explicitly imposed by at least one team are - produced: boolean flags are "True if any team is True"; - ``allowed_apps`` is the union of the teams' whitelists. - """ - merged: Dict[str, Any] = {} - merged_apps: Optional[List[str]] = None - for raw in raw_perms_list: - perms = normalize_team_permissions(raw) - for key in BOOL_CAPABILITY_DEFAULTS: - if key in perms: - merged[key] = bool(merged.get(key, False)) or bool(perms[key]) - if "allowed_apps" in perms: - merged_apps = list(dict.fromkeys((merged_apps or []) + perms["allowed_apps"])) - if merged_apps is not None: - merged["allowed_apps"] = merged_apps - return merged - - -def team_default_permissions_for_user(db: Session, user_id: str) -> Dict[str, Any]: - """Aggregate the default permissions of all teams the user belongs to (union / most permissive). - - No teams / CE / query exception → return ``{}`` (resolution degrades to - "personal → system default"). - """ - try: - # CE's PostgreSQL baseline intentionally omits the team tables while - # keeping this shared module importable. Isolate the optional lookup - # in a SAVEPOINT: catching an undefined-table error without rolling it - # back leaves PostgreSQL's outer transaction aborted, breaking the - # caller's next otherwise-valid query. - with db.begin_nested(): - from core.db.repository import TeamRepository - - rows = TeamRepository(db).list_for_user(user_id) - except Exception: # noqa: BLE001 — CE has no team table / any exception degrades safely - return {} - return merge_team_permissions( - getattr(team, "default_permissions", None) for team, _role in rows - ) - - def resolve_capabilities( meta: Optional[Dict[str, Any]], *default_layers: Optional[Dict[str, Any]], @@ -236,12 +166,11 @@ def user_has_capability(db: Session, user_id: str, flag: str) -> bool: meta = shadow.extra_data if isinstance(shadow.extra_data, dict) else {} if meta.get("role") == "super_admin": return True - from core.auth.role_permissions import role_permissions_for_user + from core.auth.edition_capabilities import default_capability_layers_for_user caps = resolve_capabilities( meta, - role_permissions_for_user(db, user_id), - team_default_permissions_for_user(db, user_id), + *default_capability_layers_for_user(db, user_id), ) return bool(caps.get(flag)) @@ -255,14 +184,12 @@ def resolve_user_capabilities(db: Session, user_id: str) -> Dict[str, Any]: chain is "personal → role → team → system" (both role and team degrade defensively to ``{}``). """ - from core.auth.role_permissions import role_permissions_for_user + from core.auth.edition_capabilities import default_capability_layers_for_user from core.db.models import UserShadow shadow = db.query(UserShadow).filter(UserShadow.user_id == user_id).first() meta = dict(shadow.extra_data) if (shadow and isinstance(shadow.extra_data, dict)) else {} - role_defaults = role_permissions_for_user(db, user_id) - team_defaults = team_default_permissions_for_user(db, user_id) - caps = resolve_capabilities(meta, role_defaults, team_defaults) + caps = resolve_capabilities(meta, *default_capability_layers_for_user(db, user_id)) if settings.edition.edition == "ce": caps["can_create_public_kb"] = False return caps diff --git a/src/backend/core/auth/edition_capabilities.py b/src/backend/core/auth/edition_capabilities.py new file mode 100644 index 00000000..9c906878 --- /dev/null +++ b/src/backend/core/auth/edition_capabilities.py @@ -0,0 +1,14 @@ +"""Community-edition capability defaults.""" + + +def default_capability_layers_for_user(db, user_id: str) -> tuple[()]: + """CE has no organization-scoped role or team default layers.""" + return () + + +def extend_agent_visibility_filters(db, user_id: str, agent_model, filters: list): + """CE exposes only the caller's personal agents and enabled built-ins.""" + return filters + + +__all__ = ["default_capability_layers_for_user", "extend_agent_visibility_filters"] diff --git a/src/backend/core/auth/kb_permissions.py b/src/backend/core/auth/kb_permissions.py index 590d4247..26102bdb 100644 --- a/src/backend/core/auth/kb_permissions.py +++ b/src/backend/core/auth/kb_permissions.py @@ -1,48 +1,13 @@ -"""Knowledge-base access permission resolution (single source of truth for the shared-KB permission-grant system). - -Every determination of "which knowledge bases a user can see/retrieve, and at what -permission level" funnels through this module, ensuring: - - capability catalog display (catalog.py) - - agent retrieval (retrieve_local_kb / retrieve_dataset_content) - - read/write validation (kb.py / kb_service.py) -all see exactly the same visible set — eliminating the privilege escalation of -"hidden in the UI but retrievable by the agent". - -Resources come in two kinds, uniformly identified by ``resource_id``: - - local shared KBs: ``kb_id`` (created in the admin console, owned by the system - owner, ``KBSpace.visibility`` not private) - - external Dify KBs: ``dataset_id`` - -**Hidden-by-default / whitelist model (permissions are assigned only in "User -management / Team management"; KB management assigns no permissions)**: - - Shared KBs are **hidden from everyone by default**; only granted users/teams see them at their level. - - Precedence: **a personal grant overrides a team grant** (when a user has both a - personal grant and a team-mediated grant on a KB, the personal one applies). - - Owner / super admin are always admin; a private user KB is visible only to its owner. - -Permission tiers (modeled on team-folder permissions admin>edit>view>none): -view < edit < admin. view already means visible and retrievable. - -CE compatibility: ``kb_grants`` is an EE-only table, not created in CE. All queries -against it in this module are wrapped in try/except fallbacks — when the table is -missing, treat as "no grants" (shared KBs invisible to regular users; private KBs -still visible to their owner). -""" +"""Single-tenant knowledge-base authorization for the community edition.""" from __future__ import annotations -from typing import Dict, List, Literal, Optional, Set +from typing import Dict, List, Literal, Set -from sqlalchemy import or_ from sqlalchemy.orm import Session KBLevel = Literal["none", "view", "edit", "admin"] - -_RANK: Dict[str, int] = {"none": 0, "view": 1, "edit": 2, "admin": 3} - - -def _max_level(a: str, b: str) -> str: - return a if _RANK.get(a, 0) >= _RANK.get(b, 0) else b +_RANK = {"none": 0, "view": 1, "edit": 2, "admin": 3} def has_kb_permission(current: str, required: str) -> bool: @@ -50,198 +15,68 @@ def has_kb_permission(current: str, required: str) -> bool: def is_shared_visibility(visibility: str) -> bool: - """Shared-KB determination: anything non-private is a shared KB (globally retrievable by kb_id); only private is owner-isolated. - - Retrieval classification and catalog visibility share this single definition, - avoiding scattered visibility-string checks at call sites that would drift apart. - """ return visibility != "private" def level_to_caps(level: str) -> Dict[str, bool]: - """Grant level → frontend capability bits (shared policy between catalog items and other consumers).""" return { - "editable": level == "admin", - "deletable": level == "admin", - "uploadable": level in ("edit", "admin"), + "can_view": has_kb_permission(level, "view"), + "can_edit": has_kb_permission(level, "edit"), + "can_admin": has_kb_permission(level, "admin"), } -# ── Principals (user + their teams + super-admin status) ──────────────────── - def _is_super_admin(db: Session, user_id: str) -> bool: - try: - from core.db.models import UserShadow - row = db.query(UserShadow.extra_data).filter(UserShadow.user_id == user_id).first() - meta = row[0] if row and isinstance(row[0], dict) else {} - return str(meta.get("role") or "") == "super_admin" - except Exception: - return False - - -def _user_team_ids(db: Session, user_id: str) -> Set[str]: - try: - from core.db.models import TeamMember - rows = db.query(TeamMember.team_id).filter(TeamMember.user_id == user_id).all() - return {r[0] for r in rows if r[0]} - except Exception: - return set() - - -def _effective_grants(db: Session, resource_type: str, user_id: str, team_ids: Set[str]) -> Dict[str, str]: - """Return {resource_id: the user's effective grant level}, with **personal grants taking precedence over team grants**. - - If a resource has both a personal grant and a team-mediated grant → take the - personal one; multiple team grants → take the highest. Table missing → {}. - """ - user_grants: Dict[str, str] = {} - team_grants: Dict[str, str] = {} - try: - from core.db.models import KBGrant - conds = [(KBGrant.principal_type == "user") & (KBGrant.principal_id == user_id)] - if team_ids: - conds.append((KBGrant.principal_type == "team") & (KBGrant.principal_id.in_(list(team_ids)))) - rows = ( - db.query(KBGrant.resource_id, KBGrant.principal_type, KBGrant.level) - .filter(KBGrant.resource_type == resource_type) - .filter(or_(*conds)) - .all() - ) - for resource_id, principal_type, level in rows: - lvl = level if level in _RANK else "view" - if principal_type == "user": - user_grants[resource_id] = lvl # personal grant is unique (PK), take directly - else: - team_grants[resource_id] = _max_level(team_grants.get(resource_id, "none"), lvl) - except Exception: - pass - # Personal overrides team - effective = dict(team_grants) - effective.update(user_grants) - return effective - - -# ── Local KBs ──────────────────────────────────────────────────────────────── - -def _level_for( - owner: str, user_id: str, is_admin: bool, visibility: str, grant: Optional[str], -) -> Optional[KBLevel]: - """Single KB → effective level (None means invisible). The batch and single-KB paths share this rule (hidden-by-default / whitelist). - - Owner/super admin are always admin; private is owner-only; shared KBs (non-private) - are **hidden from everyone by default** — only granted users/teams see them at their - grant level (``grant``, None = not granted → invisible). Personal grants take - precedence over team grants. - """ - if is_admin or owner == user_id: - return "admin" - if visibility == "private": - return None - # Shared KB: hidden by default; only grantees see it at their level - return grant # type: ignore[return-value] + from core.db.models import UserShadow + row = db.query(UserShadow.extra_data).filter(UserShadow.user_id == user_id).first() + metadata = row[0] if row and isinstance(row[0], dict) else {} + return metadata.get("role") == "super_admin" -def get_accessible_local_kb_levels(db: Session, user_id: str) -> Dict[str, KBLevel]: - """Return local KBs as {kb_id: effective level} (KBs the user can at least view). Rules in ``_level_for``.""" - user_id = str(user_id or "") - levels: Dict[str, KBLevel] = {} - if not user_id: - return levels +def get_accessible_local_kb_levels(db: Session, user_id: str) -> Dict[str, KBLevel]: from core.db.models import KBSpace + if not user_id: + return {} is_admin = _is_super_admin(db, user_id) - team_ids = _user_team_ids(db, user_id) - effective = _effective_grants(db, "local", user_id, team_ids) - rows = ( db.query(KBSpace.kb_id, KBSpace.visibility, KBSpace.user_id) .filter(KBSpace.deleted_at.is_(None)) .all() ) - for kb_id, visibility, owner in rows: - lvl = _level_for(owner, user_id, is_admin, visibility, effective.get(kb_id)) - if lvl: - levels[kb_id] = lvl - return levels + return { + kb_id: "admin" if is_admin or owner == user_id else "view" + for kb_id, visibility, owner in rows + if is_admin or owner == user_id or is_shared_visibility(visibility) + } def get_accessible_local_kb_ids(db: Session, user_id: str) -> Set[str]: - return set(get_accessible_local_kb_levels(db, user_id).keys()) + return set(get_accessible_local_kb_levels(db, user_id)) -# ── Dify datasets ──────────────────────────────────────────────────────────── +def get_dataset_levels(db: Session, user_id: str, dataset_ids: List[str]) -> Dict: + return {} -def get_dataset_levels(db: Session, user_id: str, dataset_ids: List[str]) -> Dict[str, KBLevel]: - """For the given list of Dify dataset ids, return {dataset_id: effective level} (accessible ones only). - - Hidden-by-default / whitelist model: only granted datasets are visible (personal - grants take precedence over team grants); super admin is always admin. - """ - user_id = str(user_id or "") - out: Dict[str, KBLevel] = {} - if not user_id or not dataset_ids: - return out - - is_admin = _is_super_admin(db, user_id) - team_ids = _user_team_ids(db, user_id) - effective = _effective_grants(db, "dify", user_id, team_ids) if not is_admin else {} - - for ds_id in dataset_ids: - ds_id = str(ds_id or "").strip() - if not ds_id: - continue - if is_admin: - out[ds_id] = "admin" - else: - lvl = effective.get(ds_id) - if lvl: - out[ds_id] = lvl # type: ignore[assignment] - return out - - -# ── Mixed filtering (used by agent_factory: enabled_kb_ids mixes local kb_ and dify dataset kinds) ── def filter_accessible_kb_ids(db: Session, user_id: str, kb_ids: List[str]) -> List[str]: - """Strip ids the current user cannot access from the client-supplied enabled_kb_ids, preserving order. - - Local KB ids are distinguished by the ``kb_`` prefix; the rest are treated as Dify - datasets. Guards against the frontend passing unauthorized ids. - """ - if not kb_ids: - return [] - local_ids = [x for x in kb_ids if str(x).startswith("kb_")] - dify_ids = [x for x in kb_ids if not str(x).startswith("kb_")] - - allowed: Set[str] = set() - if local_ids: - local_levels = get_accessible_local_kb_levels(db, user_id) - allowed |= {k for k in local_ids if k in local_levels} - if dify_ids: - ds_levels = get_dataset_levels(db, user_id, dify_ids) - allowed |= set(ds_levels.keys()) - return [x for x in kb_ids if x in allowed] + allowed = get_accessible_local_kb_ids(db, user_id) + return [kb_id for kb_id in kb_ids if kb_id in allowed] def resolve_local_kb_level(db: Session, user_id: str, kb_id: str) -> KBLevel: - """Permission level for a single KB (used by kb.py / kb_service read/write validation; single-KB query, avoids a full-table scan).""" - user_id = str(user_id or "") - if not user_id or not kb_id: - return "none" - from core.db.models import KBSpace - - row = ( - db.query(KBSpace.kb_id, KBSpace.visibility, KBSpace.user_id) - .filter(KBSpace.kb_id == kb_id, KBSpace.deleted_at.is_(None)) - .first() - ) - if not row: - return "none" - - _, visibility, owner = row - is_admin = _is_super_admin(db, user_id) - grant: Optional[str] = None - # Only look up grants when "not owner / not super admin / shared KB" (hidden by default; ungranted means none) - if not is_admin and owner != user_id and visibility != "private": - grant = _effective_grants(db, "local", user_id, _user_team_ids(db, user_id)).get(kb_id) - return _level_for(owner, user_id, is_admin, visibility, grant) or "none" + return get_accessible_local_kb_levels(db, user_id).get(kb_id, "none") + + +__all__ = [ + "KBLevel", + "filter_accessible_kb_ids", + "get_accessible_local_kb_ids", + "get_accessible_local_kb_levels", + "get_dataset_levels", + "has_kb_permission", + "is_shared_visibility", + "level_to_caps", + "resolve_local_kb_level", +] diff --git a/src/backend/core/auth/marketplace_visibility.py b/src/backend/core/auth/marketplace_visibility.py index cd7b4b2d..8419cb90 100644 --- a/src/backend/core/auth/marketplace_visibility.py +++ b/src/backend/core/auth/marketplace_visibility.py @@ -1,109 +1,12 @@ -"""Marketplace item visibility-scope resolution (single source of truth shared across the skill / plugin / sub-agent marketplaces). +"""Community marketplace items are public when enabled.""" -The decision of "which items a given user can see in the marketplace" is funneled into this -module, ensuring the visible set seen across the three paths — marketplace listing, detail, and -install — is fully consistent, preventing the privilege escalation of "hidden in the listing but -still installable by guessing the slug". -**Visible-to-all-by-default / blacklist-exception model** (the opposite direction of the knowledge base's whitelist model): - - ``marketplace_listing_states.visibility`` missing a row or ``public`` → visible to everyone; - - ``scoped`` → visible only to principals granted in ``marketplace_visibility_grants``, - ``principal_type`` = ``user`` | ``team`` | ``role``, visible if any one matches (union); - - Super admins are always visible. Role principals include both personally-assigned roles and department default roles obtained via teams - (consistent with the capability-bit resolution rules of core/auth/role_permissions.py). +def get_hidden_item_ids(db, kind: str, user_id) -> set[str]: + return set() -Governs only marketplace browsing and installation; does not retroactively track installed instances (later restrictions do not revoke what is already installed). -CE compatibility: ``marketplace_visibility_grants`` is an EE-only table, not created in CE; and CE -has no admin marketplace route, so no scoped rows are produced. This module wraps all grant/role -table queries in try/except fallbacks — when a table is missing it is treated as "no grant" -(scoped items are invisible, but CE in practice has no scoped rows). -""" +def is_item_visible(db, kind: str, item_id: str, user_id) -> bool: + return True -from __future__ import annotations -from typing import Optional, Set - -from sqlalchemy import or_ -from sqlalchemy.orm import Session - -from core.auth.kb_permissions import _is_super_admin, _user_team_ids - - -def _user_role_ids(db: Session, user_id: str, team_ids: Set[str]) -> Set[str]: - """The user's effective role set: personally-assigned ∪ department default roles of each team. Table missing (CE) → empty set.""" - try: - from core.db.models import RoleAssignment - conds = [ - (RoleAssignment.principal_type == "user") - & (RoleAssignment.principal_id == user_id) - ] - if team_ids: - conds.append( - (RoleAssignment.principal_type == "team") - & (RoleAssignment.principal_id.in_(list(team_ids))) - ) - rows = db.query(RoleAssignment.role_id).filter(or_(*conds)).all() - return {r[0] for r in rows if r[0]} - except Exception: - return set() - - -def _scoped_item_ids(db: Session, kind: str) -> Set[str]: - """The set of item_ids in this marketplace set to scoped (visible to a specified scope). Missing column / query failure → empty set.""" - try: - from core.db.models import MarketplaceListingState - rows = ( - db.query(MarketplaceListingState.item_id) - .filter( - MarketplaceListingState.kind == kind, - MarketplaceListingState.visibility == "scoped", - ) - .all() - ) - return {r[0] for r in rows} - except Exception: - return set() - - -def _granted_item_ids(db: Session, kind: str, scoped_ids: Set[str], user_id: str) -> Set[str]: - """The set of item_ids among scoped items for which the current user is granted access via any user/team/role principal.""" - try: - from core.db.models import MarketplaceVisibilityGrant as G - team_ids = _user_team_ids(db, user_id) - role_ids = _user_role_ids(db, user_id, team_ids) - conds = [(G.principal_type == "user") & (G.principal_id == user_id)] - if team_ids: - conds.append((G.principal_type == "team") & (G.principal_id.in_(list(team_ids)))) - if role_ids: - conds.append((G.principal_type == "role") & (G.principal_id.in_(list(role_ids)))) - rows = ( - db.query(G.item_id) - .filter(G.kind == kind, G.item_id.in_(list(scoped_ids))) - .filter(or_(*conds)) - .all() - ) - return {r[0] for r in rows} - except Exception: - return set() - - -def get_hidden_item_ids(db: Session, kind: str, user_id: Optional[str]) -> Set[str]: - """The set of item_ids in this marketplace that are **invisible** to the current user (for batch filtering, one query on the listing path). - - Zero extra overhead when there are no scoped items (the norm for the vast majority of deployments); always an empty set for super admins. - An empty ``user_id`` (anonymous / system call) is treated as an ordinary unauthorized user. - """ - scoped = _scoped_item_ids(db, kind) - if not scoped: - return set() - user_id = str(user_id or "") - if user_id and _is_super_admin(db, user_id): - return set() - granted = _granted_item_ids(db, kind, scoped, user_id) if user_id else set() - return scoped - granted - - -def is_item_visible(db: Session, kind: str, item_id: str, user_id: Optional[str]) -> bool: - """Single-item visibility (for guarding the detail / install paths).""" - return str(item_id) not in get_hidden_item_ids(db, kind, user_id) +__all__ = ["get_hidden_item_ids", "is_item_visible"] diff --git a/src/backend/core/auth/mock_ticket_store.py b/src/backend/core/auth/mock_ticket_store.py index b35cefd2..2a2bcb0c 100644 --- a/src/backend/core/auth/mock_ticket_store.py +++ b/src/backend/core/auth/mock_ticket_store.py @@ -1,7 +1,7 @@ """In-process ticket store for local-account and mock-SSO login. Holds the one-time ticket state used by the local-account and mock SSO flows. -Relocated out of ``api/routes/v1/mock_sso.py`` so that ``core.auth.sso`` can +Relocated out of ``api/routes/v1/mock_sso.py`` so that edition authentication can validate tickets without importing an API route module (breaks the ``core/auth → api`` upward dependency). The unified login route and mock SSO route both reuse this store. diff --git a/src/backend/core/auth/permissions_iface.py b/src/backend/core/auth/permissions_iface.py index d2ada994..4247c4db 100644 --- a/src/backend/core/auth/permissions_iface.py +++ b/src/backend/core/auth/permissions_iface.py @@ -1,90 +1,51 @@ -"""权限接口层 —— 社区版单租户 stub。 - -CE 没有团队/多租户：当前用户对自己的资源恒为最高权限，对他人资源恒不可见。 -与商业版的真实现保持同一组符号与签名（接缝 C3），调用方零改动。 - -退化原则（对齐主仓施工图附录 D）： - - owner 语义保留：自己的资源恒最高权限（由各调用方的 owner 判定承担）； - - 团队权限恒 ``none``——CE 无团队概念，team_id 标记的他人资源一律不可见 - （从 EE 迁移来的存量团队数据不能因 stub 放行而对全员可读）； - - 资源「存在性 404」保留； - - 不写审计日志（审计属商业版）。 -""" +"""Community single-owner authorization contracts.""" from __future__ import annotations from dataclasses import dataclass -from typing import Any, Literal - -from fastapi import Depends, HTTPException, Path, Request -from sqlalchemy.orm import Session +from typing import Literal from core.db.engine import get_db from core.db.models import ChatSession, Project - -# ── 团队文件权限（CE：单租户恒 admin） ──────────────────────────────────────── +from fastapi import Depends, HTTPException, Path, Request +from sqlalchemy.orm import Session PermissionLevel = Literal["none", "view", "edit", "admin"] _RANK = {"none": 0, "view": 1, "edit": 2, "admin": 3} -def resolve_team_file_permission(db: Session, user_id: str, team_id: str) -> PermissionLevel: - # CE 无团队：team_id 标记的资源不经团队通道放行（owner 通道由调用方保留）。 - # 恒 "admin" 会让 EE 迁移来的团队文件对所有登录用户可读——必须是 "none"。 - return "none" - - def has_permission(current: PermissionLevel, required: PermissionLevel) -> bool: return _RANK[current] >= _RANK[required] -def resolve_artifact_access(db: Session, user_id: str, owner_id, team_id) -> PermissionLevel: - """owner ∪ team 合成的 artifact 访问级（与 EE 版同签名）。 - - CE：owner 恒 admin；团队通道恒 none——自己的资源（含 EE 迁移来的 - team_id 标记文件）始终可访问，他人的团队资源不可见。 - """ +def resolve_artifact_access(db: Session, user_id: str, owner_id, scope_id) -> PermissionLevel: + """Resolve access solely from personal ownership.""" if owner_id and str(owner_id) == str(user_id): return "admin" - if team_id: - return resolve_team_file_permission(db, str(user_id), str(team_id)) return "none" -def require_team_file_permission( - db: Session, - user_id: str, - team_id: str, - required: PermissionLevel, - *, - request: Any = None, - action: str = "team_file.access", -) -> PermissionLevel: - raise HTTPException(status_code=404, detail="团队功能在当前版本不可用") - - # ── 项目权限（CE：个人项目 owner 判定保留） ─────────────────────────────────── ProjectPermissionLevel = Literal["none", "view", "edit", "admin"] -def resolve_project_permission(db: Session, user_id: str, project: Project) -> ProjectPermissionLevel: +def resolve_project_permission( + db: Session, user_id: str, project: Project +) -> ProjectPermissionLevel: if project is None or project.deleted_at is not None: return "none" if project.kind == "personal": return "admin" if project.owner_user_id == user_id else "none" - # CE 不存在团队项目；历史数据兜底为不可见 + # Non-personal legacy data is never visible in CE. return "none" -def can_create_team_project(db: Session, user_id: str, team_id: str) -> bool: - return True - - @dataclass class ProjectAccess: """传给路由的访问上下文：project + 用户权限 + user_id。""" + project: Project level: ProjectPermissionLevel user_id: str @@ -120,7 +81,7 @@ async def _dep( return _dep -# ── 会话共享权限（CE：owner-only） ─────────────────────────────────────────── +# ── 会话权限（CE：owner-only） ─────────────────────────────────────────────── ChatAccessLevel = Literal["none", "read", "edit", "admin"] @@ -131,12 +92,6 @@ def resolve_chat_access(db: Session, user_id: str, session: ChatSession) -> Chat return "admin" if session.user_id == user_id else "none" -def can_modify_share_scope(db: Session, user_id: str, session: ChatSession) -> bool: - if session is None or session.deleted_at is not None: - return False - return session.user_id == user_id - - def can_delete_session(db: Session, user_id: str, session: ChatSession) -> bool: if session is None or session.deleted_at is not None: return False @@ -146,16 +101,12 @@ def can_delete_session(db: Session, user_id: str, session: ChatSession) -> bool: __all__ = [ "ChatAccessLevel", "can_delete_session", - "can_modify_share_scope", "resolve_chat_access", "ProjectAccess", "ProjectPermissionLevel", - "can_create_team_project", "require_project_access", "resolve_project_permission", "PermissionLevel", "has_permission", - "require_team_file_permission", "resolve_artifact_access", - "resolve_team_file_permission", ] diff --git a/src/backend/core/auth/role_permissions.py b/src/backend/core/auth/role_permissions.py deleted file mode 100644 index 1aaef229..00000000 --- a/src/backend/core/auth/role_permissions.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Role capability bundles: normalize / merge / aggregate per user. - -A role ([[Role]]) is a reusable, named **capability-grant bundle** — a set of -capability bits packaged together and assigned to a team (= department default -role, inherited by members in real time) or an individual. This module is the -single source of truth for "role → capability bits" and the sole extension seam -for a "later rework into an org tree" (see the note at the end of -``role_permissions_for_user``). - -Resolution chain (see ``core/auth/capabilities.py``): - personal explicit override → **union of roles** → team default → system default - -Role semantics are "grants" (additive): - -- Boolean bits only store what is granted (``True``); multiple roles take the - union (a grant from any role takes effect), reusing the union logic of - ``capabilities.merge_team_permissions``. -- ``allowed_apps`` takes the union of each role's whitelist (more permissive). - -Defensive degradation: CE (no roles table) / any query exception → return ``{}``; -resolution degrades to "personal → team default → system default", exactly as -before roles were introduced. -""" - -from __future__ import annotations - -from typing import Any, Dict, List, Optional - -from core.auth.capabilities import BOOL_CAPABILITY_DEFAULTS, merge_team_permissions -from sqlalchemy.orm import Session - - -def normalize_role_permissions(payload: Any) -> Dict[str, Any]: - """Normalize a role capability bundle: keep only valid capability bits, with "grant" semantics. - - - Boolean bits: store ``True`` only when the payload key is truthy (a role - never expresses "off" — additive grants; "off" is left to personal explicit - overrides); - - ``allowed_apps``: ``list`` → de-duplicated string list (empty list = - restricted to "none"); key absent = the role does not restrict app visibility. - """ - if not isinstance(payload, dict): - return {} - out: Dict[str, Any] = {} - for key in BOOL_CAPABILITY_DEFAULTS: - if payload.get(key): - out[key] = True - raw_apps = payload.get("allowed_apps") - if isinstance(raw_apps, list): - out["allowed_apps"] = list(dict.fromkeys(str(x) for x in raw_apps)) - return out - - -def merge_role_permissions(raw_perms_list: Any) -> Dict[str, Any]: - """Merge multiple roles' capability bundles (union across roles / most permissive). - - Directly reuses the team-default union logic: boolean bits are "True if any is - True", ``allowed_apps`` takes the union. - """ - return merge_team_permissions(raw_perms_list) - - -def role_permissions_for_user( - db: Session, user_id: str, team_ids: Optional[List[str]] = None -) -> Dict[str, Any]: - """Aggregate the union of capability bundles across all of a user's roles (direct assignments + department default roles of their teams). - - - Direct roles: ``role_assignments(principal_type='user', principal_id=user_id)`` - - Department default roles: ``role_assignments(principal_type='team', principal_id=team_id)`` - for each team the user belongs to — inherited by members in real time (no - provisioning needed; new SSO members take effect immediately). - - ``team_ids`` can be passed in by callers that already loaded the teams - (login/session serialization), saving one team_members query. - No roles / CE without tables / any exception → ``{}`` (resolution degrades to - "personal → team default → system default"). - """ - try: - # The CE PostgreSQL schema omits role/team tables. A plain try/except - # is insufficient there because an undefined-table error poisons the - # current transaction even after Python catches it. A nested - # transaction rolls the optional lookup back to its SAVEPOINT and - # leaves subsequent model/API-key queries usable. - with db.begin_nested(): - from core.db.models import Role, RoleAssignment, TeamMember - - if team_ids is None: - team_ids = [ - tid - for (tid,) in db.query(TeamMember.team_id) - .filter(TeamMember.user_id == user_id) - .all() - ] - - conds = [ - (RoleAssignment.principal_type == "user") & (RoleAssignment.principal_id == user_id) - ] - if team_ids: - conds.append( - (RoleAssignment.principal_type == "team") - & (RoleAssignment.principal_id.in_(team_ids)) - ) - - from sqlalchemy import or_ - - role_ids = [ - rid - for (rid,) in db.query(RoleAssignment.role_id).filter(or_(*conds)).distinct().all() - ] - if not role_ids: - return {} - perms_list = [ - p for (p,) in db.query(Role.permissions).filter(Role.role_id.in_(role_ids)).all() - ] - except Exception: # noqa: BLE001 — CE without role tables / any exception degrades safely - return {} - return merge_role_permissions(perms_list) diff --git a/src/backend/core/auth/roles.py b/src/backend/core/auth/roles.py deleted file mode 100644 index fa3dfafd..00000000 --- a/src/backend/core/auth/roles.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Team role constants and helpers — single source of truth, to avoid duplicating definitions across modules.""" - -from __future__ import annotations - -from typing import Literal - -TeamRole = Literal["owner", "admin", "member"] - -TEAM_ROLES: tuple[TeamRole, ...] = ("owner", "admin", "member") - -ROLE_RANK: dict[str, int] = {"member": 1, "admin": 2, "owner": 3} - -ROLE_LABELS_ZH: dict[str, str] = {"owner": "所有者", "admin": "管理员", "member": "成员"} - - -def rank(role: str) -> int: - return ROLE_RANK.get(role, 0) - - -def role_label(role: str) -> str: - return ROLE_LABELS_ZH.get(role, role) - - -def at_least(role: str, minimum: str) -> bool: - return rank(role) >= rank(minimum) diff --git a/src/backend/core/config/catalog_loader.py b/src/backend/core/config/catalog_loader.py index 6b88f350..41337b74 100644 --- a/src/backend/core/config/catalog_loader.py +++ b/src/backend/core/config/catalog_loader.py @@ -33,6 +33,17 @@ ) +def _database_query_capability_available() -> bool: + """Whether this edition ships a runnable database-query implementation.""" + try: + from mcp_servers._ports import PORTS + + return "query_database" in PORTS + except Exception as exc: + _LOGGER.warning("Database-query runtime registry unavailable: %s", exc) + return False + + def _private_skill_ids() -> set: """Set of private skill ids in admin_skills owned by some user (owner_user_id non-null). @@ -126,8 +137,8 @@ def _default_catalog() -> Dict[str, Any]: # Build MCP items from mcp_config.py with auto-extracted detail field try: - from core.config.mcp_config import MCP_SERVER_DISPLAY_NAMES as _MCP_ZH_NAMES from core.config.mcp_config import MCP_SERVER_DESCRIPTIONS as _MCP_ZH_DESC + from core.config.mcp_config import MCP_SERVER_DISPLAY_NAMES as _MCP_ZH_NAMES except Exception: _MCP_ZH_NAMES = {} _MCP_ZH_DESC = {} @@ -144,16 +155,17 @@ def _default_catalog() -> Dict[str, Any]: for k in mcp_servers.keys() if k not in DB_HIDDEN_SERVERS ] - mcp_items.append( - _item( - item_id=DB_UMBRELLA_ID, - kind="mcp_server", - name=DB_UMBRELLA_NAME, - description=DB_UMBRELLA_DESC, - enabled=True, - config={"server": DB_UMBRELLA_ID}, + if _database_query_capability_available(): + mcp_items.append( + _item( + item_id=DB_UMBRELLA_ID, + kind="mcp_server", + name=DB_UMBRELLA_NAME, + description=DB_UMBRELLA_DESC, + enabled=True, + config={"server": DB_UMBRELLA_ID}, + ) ) - ) skill_items: List[Dict[str, Any]] = [] for metadata in _load_builtin_skill_metadata(): @@ -377,9 +389,9 @@ def _load_dynamic_mcp_specs() -> Dict[str, Dict[str, str]]: """ try: from core.config.mcp_config import ( - MCP_SERVERS, MCP_SERVER_DESCRIPTIONS, MCP_SERVER_DISPLAY_NAMES, + MCP_SERVERS, ) except Exception as e: _LOGGER.warning(f"Failed to load MCP configs: {e}") @@ -438,13 +450,16 @@ def _load_dynamic_mcp_specs() -> Dict[str, Dict[str, str]]: "detail": row.user_intro or MCP_SERVER_USER_INTROS.get(sid, ""), } - # Inject the synthetic "database query" umbrella capability (corresponds to no admin_mcp_servers row). - result[DB_UMBRELLA_ID] = { - "id": DB_UMBRELLA_ID, - "name": DB_UMBRELLA_NAME, - "description": DB_UMBRELLA_DESC, - "detail": MCP_SERVER_USER_INTROS.get(DB_UMBRELLA_ID, ""), - } + # Inject the synthetic umbrella only when this edition ships a database + # query runtime. Otherwise the sync layer would recreate an item that the + # CE build deliberately removed from catalog.json. + if _database_query_capability_available(): + result[DB_UMBRELLA_ID] = { + "id": DB_UMBRELLA_ID, + "name": DB_UMBRELLA_NAME, + "description": DB_UMBRELLA_DESC, + "detail": MCP_SERVER_USER_INTROS.get(DB_UMBRELLA_ID, ""), + } return result diff --git a/src/backend/core/config/catalog_runtime.py b/src/backend/core/config/catalog_runtime.py index d63f8ebd..d28e827a 100644 --- a/src/backend/core/config/catalog_runtime.py +++ b/src/backend/core/config/catalog_runtime.py @@ -14,11 +14,15 @@ from time import monotonic from typing import Any, Dict, List, Optional -from sqlalchemy.orm import Session - from core.config.catalog import get_catalog from core.config.catalog_common import _item -from core.config.catalog_loader import DB_HIDDEN_SERVERS, skill_body_from_raw +from core.config.catalog_loader import ( + DB_HIDDEN_SERVERS, + DB_UMBRELLA_ID, + _database_query_capability_available, + skill_body_from_raw, +) +from sqlalchemy.orm import Session logger = logging.getLogger(__name__) @@ -32,7 +36,6 @@ "report_export_mcp": "/home/mcp/报告.svg", "web_fetch": "/home/mcp/来源.svg", } -_DATABASE_QUERY_ID = "database_query" _DATABASE_QUERY_ENABLED_CONFIG = "database_query.capability_enabled" _RUNTIME_DB_CACHE_TTL = 30.0 _runtime_db_cache: Dict[bool, tuple[float, bool, List[Dict[str, Any]], List[Dict[str, Any]]]] = {} @@ -165,7 +168,7 @@ def _apply_database_query_state(catalog: Dict[str, Any], db: Session) -> None: def _set_database_query_state(catalog: Dict[str, Any], enabled: bool) -> None: for item in catalog.get("mcp") or []: - if not isinstance(item, dict) or item.get("id") != _DATABASE_QUERY_ID: + if not isinstance(item, dict) or item.get("id") != DB_UMBRELLA_ID: continue item["enabled"] = enabled return @@ -238,6 +241,17 @@ def get_runtime_catalog( if not isinstance(catalog.get(key), list): catalog[key] = [] + # A deployment can retain old catalog data or an in-process cache from a + # build that shipped database-query support. The edition's runnable MCP + # registry is authoritative: never surface the synthetic umbrella when its + # implementation is absent. + if not _database_query_capability_available(): + catalog["mcp"] = [ + item + for item in catalog["mcp"] + if not isinstance(item, dict) or item.get("id") != DB_UMBRELLA_ID + ] + try: db_query_enabled, db_skills, db_mcps = _public_db_overlay( db, diff --git a/src/backend/core/config/display_names.py b/src/backend/core/config/display_names.py index 4fed02fc..84841d19 100644 --- a/src/backend/core/config/display_names.py +++ b/src/backend/core/config/display_names.py @@ -8,40 +8,42 @@ from typing import Dict +from core.config.edition_display_names import edition_tool_display_names + # ── MCP server-level names ─────────────────────────────────────────────────── # MCP server ID -> Chinese name (used for capability-center panel titles) MCP_SERVER_DISPLAY_NAMES: Dict[str, str] = { - "query_database": "数据库查询", - "db_query": "数据库直连查询", - "retrieve_dataset_content": "知识库检索", - "internet_search": "互联网搜索", - "ai_chain_information_mcp": "产业知识中心查询", - "generate_chart_tool": "数据可视化", - "report_export_mcp": "报告导出", + "query_database": "数据库查询", + "db_query": "数据库直连查询", + "retrieve_dataset_content": "知识库检索", + "internet_search": "互联网搜索", + "ai_chain_information_mcp": "产业知识中心查询", + "generate_chart_tool": "数据可视化", + "report_export_mcp": "报告导出", # (Word capability migrated to the word-editing skill; no longer goes by an MCP tool name) # (Excel capability migrated to the excel-editing skill; no longer goes by an MCP tool name) # (PPT capability migrated to the ppt-design skill; no longer goes by an MCP tool name) # (PDF capability migrated to the pdf-editing skill; no longer goes by an MCP tool name) - "web_fetch": "网站信息抓取", - "batch_runner": "批量执行", + "web_fetch": "网站信息抓取", + "batch_runner": "批量执行", } # MCP server ID -> one-line feature description (used for capability-center panel description text) MCP_SERVER_DESCRIPTIONS: Dict[str, str] = { - "query_database": "查询数据仓库中的行业指标与统计数值，支持自然语言提问直接获取精确数据。", - "db_query": "通过 DBHub 网关只读直连 MySQL/PostgreSQL/SQL Server/MariaDB/SQLite 等数据库，自动探查表结构并执行 SQL 取数。", - "retrieve_dataset_content": "从公有/私有知识库中语义检索政策文件、产业报告及用户上传文档，支持混合检索与重排序。", - "internet_search": "通过互联网实时搜索公开网页、新闻及财经资讯，作为数据库与知识库之外的信息兜底。", - "ai_chain_information_mcp": "获取产业链全景分析报告、核心数据指标、产业动态资讯、AI 领域热点聚合及企业画像查询。", - "generate_chart_tool": "根据给定数据调用 Python 生成柱状图、折线图、饼图等可视化图表，结果以图片形式直接展示。", - "report_export_mcp": "将 Markdown 格式的分析报告导出为 Word 文档，或将表格数据导出为 Excel 文件供下载。", + "query_database": "查询数据仓库中的行业指标与统计数值，支持自然语言提问直接获取精确数据。", + "db_query": "通过 DBHub 网关只读直连 MySQL/PostgreSQL/SQL Server/MariaDB/SQLite 等数据库，自动探查表结构并执行 SQL 取数。", + "retrieve_dataset_content": "从公有/私有知识库中语义检索政策文件、产业报告及用户上传文档，支持混合检索与重排序。", + "internet_search": "通过互联网实时搜索公开网页、新闻及财经资讯，作为数据库与知识库之外的信息兜底。", + "ai_chain_information_mcp": "获取产业链全景分析报告、核心数据指标、产业动态资讯、AI 领域热点聚合及企业画像查询。", + "generate_chart_tool": "根据给定数据调用 Python 生成柱状图、折线图、饼图等可视化图表，结果以图片形式直接展示。", + "report_export_mcp": "将 Markdown 格式的分析报告导出为 Word 文档，或将表格数据导出为 Excel 文件供下载。", # (Word capability migrated to the word-editing skill; no longer goes by an MCP tool name) # (Excel capability migrated to the excel-editing skill; no longer goes by an MCP tool name) # (PPT capability migrated to the ppt-design skill; no longer goes by an MCP tool name) # (PDF capability migrated to the pdf-editing skill; no longer goes by an MCP tool name) - "web_fetch": "抓取指定网页 URL 的内容，提取正文文本或 Markdown，支持搜索引擎结果页解析。", - "batch_runner": "对一组对象（Excel 行/多份文档/文本枚举）批量执行同一个任务；先生成可确认的计划，用户审阅模板后再逐条执行。", + "web_fetch": "抓取指定网页 URL 的内容，提取正文文本或 Markdown，支持搜索引擎结果页解析。", + "batch_runner": "对一组对象（Excel 行/多份文档/文本枚举）批量执行同一个任务；先生成可确认的计划，用户审阅模板后再逐条执行。", } # ── Tool function-level names ──────────────────────────────────────────────── @@ -49,29 +51,29 @@ # Tool function name -> Chinese display name (used for chat tool cards + streaming events) TOOL_DISPLAY_NAMES: Dict[str, str] = { # MCP tools - "publish_site": "发布站点", - "query_database": "数据库查询", - "execute_sql": "执行 SQL 查询", - "search_objects": "探查库表结构", - "retrieve_dataset_content": "公有知识库检索", - "retrieve_local_kb": "私有知识库检索", - "list_datasets": "查看知识库列表", - "internet_search": "互联网搜索", - "get_chain_information": "产业链分析", - "get_industry_news": "产业资讯", - "get_latest_ai_news": "AI 热点聚合", + "publish_site": "发布站点", + "query_database": "数据库查询", + "execute_sql": "执行 SQL 查询", + "search_objects": "探查库表结构", + "retrieve_dataset_content": "公有知识库检索", + "retrieve_local_kb": "私有知识库检索", + "list_datasets": "查看知识库列表", + "internet_search": "互联网搜索", + "get_chain_information": "产业链分析", + "get_industry_news": "产业资讯", + "get_latest_ai_news": "AI 热点聚合", "get_industry_hot_companies": "领域热门企业榜", - "get_industry_hot_products": "领域热门产品榜", - "get_company_hot_events": "企业舆情事件", - "get_product_detail": "产品情报详情", - "search_company": "企业搜索", - "get_company_base_info": "企业基本信息", + "get_industry_hot_products": "领域热门产品榜", + "get_company_hot_events": "企业舆情事件", + "get_product_detail": "产品情报详情", + "search_company": "企业搜索", + "get_company_base_info": "企业基本信息", "get_company_business_analysis": "企业经营分析", - "get_company_tech_insight": "企业技术洞察", - "get_company_funding": "企业资金穿透", - "get_company_risk_warning": "企业风险预警", - "generate_chart_tool": "数据可视化", - "export_table_to_excel": "导出 Excel 表格", + "get_company_tech_insight": "企业技术洞察", + "get_company_funding": "企业资金穿透", + "get_company_risk_warning": "企业风险预警", + "generate_chart_tool": "数据可视化", + "export_table_to_excel": "导出 Excel 表格", # (Word capability migrated to the word-editing skill, see src/backend/skill_bundles/word-editing/) # The MCP layer no longer exposes word_mcp; the scripts/*.py CLIs inside the skill are the single entry point. # (Excel capability migrated to the excel-editing skill, see src/backend/skill_bundles/excel-editing/) @@ -81,45 +83,44 @@ # (PDF capability migrated to the pdf-editing skill, see src/backend/skill_bundles/pdf-editing/) # The MCP layer no longer exposes pdf_mcp; the skill's scripts/pdf-cli is the single entry point. # Industry-chain bundle sub-tools (legacy workflow.py mapping) - "get_ai_chain_information": "区块链信息查询", - "get_latest_ai_chain_info": "最新区块链动态", - "get_ai_chain_news": "区块链新闻搜索", + "get_ai_chain_information": "区块链信息查询", + "get_latest_ai_chain_info": "最新区块链动态", + "get_ai_chain_news": "区块链新闻搜索", # Batch execution - "batch_plan": "批量执行计划", + "batch_plan": "批量执行计划", # Built-in tools - "get_skills": "查询可用技能", - "get_agents": "查询可用智能体", - "get_mcp_tools": "查询 MCP 工具列表", - "search_knowledge_base": "知识库搜索", + "get_skills": "查询可用技能", + "get_agents": "查询可用智能体", + "get_mcp_tools": "查询 MCP 工具列表", + "search_knowledge_base": "知识库搜索", # Sub-agent dispatch - "call_subagent": "调用子智能体", + "call_subagent": "调用子智能体", # Skill system - "view_text_file": "读取文件", - "web_fetch": "网页抓取", + "view_text_file": "读取文件", + "web_fetch": "网页抓取", # Cross-turn file access - "read_artifact": "读取文件内容", + "read_artifact": "读取文件内容", # Workspace file visibility - "pin_to_workspace": "固定到工作区", + "pin_to_workspace": "固定到工作区", # Code-execution Lab tools - "bash": "执行 Shell 命令", - "Bash": "执行 Shell 命令", # Title-cased alias for models that follow the Read/Edit/Write naming family - "Read": "读取文件", - "Edit": "编辑文件", - "Write": "写入文件", - "Glob": "查找文件", - "Grep": "搜索内容", - "sandbox_put_artifact": "上传文件到沙箱", - "sandbox_get_artifact": "从沙箱保存文件", + "bash": "执行 Shell 命令", + "Bash": "执行 Shell 命令", # Title-cased alias for models that follow the Read/Edit/Write naming family + "Read": "读取文件", + "Edit": "编辑文件", + "Write": "写入文件", + "Glob": "查找文件", + "Grep": "搜索内容", + "sandbox_put_artifact": "上传文件到沙箱", + "sandbox_get_artifact": "从沙箱保存文件", # Deprecated (kept for fallback display) - "execute_code": "代码执行（已废弃）", - "run_command": "执行命令（已废弃）", + "execute_code": "代码执行（已废弃）", + "run_command": "执行命令（已废弃）", # Deprecated (kept for fallback display) - "use_skill": "加载技能（已废弃）", + "use_skill": "加载技能（已废弃）", # My Space access tools (code-execution mode) - "list_myspace_files": "浏览我的空间", - "stage_myspace_file": "导入文件到工作区", - "list_favorite_chats": "浏览收藏会话", - "get_chat_messages": "读取会话记录", - "list_team_files": "浏览团队文件夹", - "stage_team_file": "导入团队文件到工作区", + "list_myspace_files": "浏览我的空间", + "stage_myspace_file": "导入文件到工作区", + "list_favorite_chats": "浏览收藏会话", + "get_chat_messages": "读取会话记录", + **edition_tool_display_names(), } diff --git a/src/backend/core/config/edition_display_names.py b/src/backend/core/config/edition_display_names.py new file mode 100644 index 00000000..fe6e1122 --- /dev/null +++ b/src/backend/core/config/edition_display_names.py @@ -0,0 +1,8 @@ +"""Community Edition has no additional commercial tool names.""" + + +def edition_tool_display_names() -> dict[str, str]: + return {} + + +__all__ = ["edition_tool_display_names"] diff --git a/src/backend/core/config/settings.py b/src/backend/core/config/settings.py index 8b0adabc..28fff0d0 100644 --- a/src/backend/core/config/settings.py +++ b/src/backend/core/config/settings.py @@ -15,7 +15,6 @@ from dotenv import dotenv_values - _REPO_ROOT = Path(__file__).resolve().parents[4] @@ -31,11 +30,15 @@ def _load_env_files() -> None: base_values = dotenv_values(base_env_path) if base_env_path.exists() else {} resolved_env = ( - os.getenv("ENV") - or os.getenv("ENVIRONMENT") - or str(base_values.get("ENV") or "") - or str(base_values.get("ENVIRONMENT") or "") - ).strip().lower() + ( + os.getenv("ENV") + or os.getenv("ENVIRONMENT") + or str(base_values.get("ENV") or "") + or str(base_values.get("ENVIRONMENT") or "") + ) + .strip() + .lower() + ) candidate_paths = [base_env_path] if resolved_env: @@ -93,8 +96,12 @@ class AuthSettings: # Local user system (self-managed accounts + invite codes + teams) local_enabled: bool = field(default_factory=lambda: _bool(_env("LOCAL_AUTH_ENABLED", "true"))) - password_min_length: int = field(default_factory=lambda: _int(_env("PASSWORD_MIN_LENGTH", "8"), 8)) - invite_code_default_ttl_hours: int = field(default_factory=lambda: _int(_env("INVITE_CODE_DEFAULT_TTL_HOURS", "168"), 168)) + password_min_length: int = field( + default_factory=lambda: _int(_env("PASSWORD_MIN_LENGTH", "8"), 8) + ) + invite_code_default_ttl_hours: int = field( + default_factory=lambda: _int(_env("INVITE_CODE_DEFAULT_TTL_HOURS", "168"), 168) + ) @dataclass(frozen=True) @@ -106,7 +113,9 @@ class SSOSettings: ticket_exchange_url: str = field(default_factory=lambda: _env("SSO_TICKET_EXCHANGE_URL", "")) login_provider_url: str = field(default_factory=lambda: _env("SSO_LOGIN_PROVIDER_URL", "")) logout_url: str = field(default_factory=lambda: _env("SSO_LOGOUT_URL", "")) - callback_param: str = field(default_factory=lambda: _env("SSO_CALLBACK_PARAM", "ticket").strip().lower() or "ticket") + callback_param: str = field( + default_factory=lambda: _env("SSO_CALLBACK_PARAM", "ticket").strip().lower() or "ticket" + ) timeout: int = field(default_factory=lambda: _int(_env("SSO_TIMEOUT_SECONDS", "5"), 5)) @property @@ -141,10 +150,16 @@ def effective_login_url(self) -> str: @dataclass(frozen=True) class SessionSettings: cookie_name: str = field(default_factory=lambda: _env("SESSION_COOKIE_NAME", "jx_session")) - cookie_secure: bool = field(default_factory=lambda: _bool(_env("SESSION_COOKIE_SECURE", "false"))) + cookie_secure: bool = field( + default_factory=lambda: _bool(_env("SESSION_COOKIE_SECURE", "false")) + ) cookie_samesite: str = field(default_factory=lambda: _env("SESSION_COOKIE_SAMESITE", "lax")) - cookie_domain: Optional[str] = field(default_factory=lambda: _env("SESSION_COOKIE_DOMAIN", "") or None) - cookie_httponly: bool = field(default_factory=lambda: _bool(_env("SESSION_COOKIE_HTTPONLY", "false"))) + cookie_domain: Optional[str] = field( + default_factory=lambda: _env("SESSION_COOKIE_DOMAIN", "") or None + ) + cookie_httponly: bool = field( + default_factory=lambda: _bool(_env("SESSION_COOKIE_HTTPONLY", "false")) + ) ttl_hours: float = field(default_factory=lambda: float(_env("SESSION_TTL_HOURS", "8"))) store_type: str = field(default_factory=lambda: _env("SESSION_STORE", "memory").lower().strip()) @@ -167,13 +182,21 @@ class OASsoSettings: # HMAC shared secret: when configured, signature verification is enforced (recommended); if empty, verification is skipped (intranet integration testing only, logs a warning) sign_secret: str = field(default_factory=lambda: _env("OA_SSO_SIGN_SECRET", "")) # Timestamp tolerance (seconds) — requests outside the window are rejected as replays - sign_ttl_seconds: int = field(default_factory=lambda: _int(_env("OA_SSO_SIGN_TTL_SECONDS", "300"), 300)) + sign_ttl_seconds: int = field( + default_factory=lambda: _int(_env("OA_SSO_SIGN_TTL_SECONDS", "300"), 300) + ) # Default role for new users joining the organization team - default_role: str = field(default_factory=lambda: _env("OA_SSO_DEFAULT_ROLE", "member").strip() or "member") + default_role: str = field( + default_factory=lambda: _env("OA_SSO_DEFAULT_ROLE", "member").strip() or "member" + ) # One-time login ticket TTL (seconds) — used for the browser redirect-to-session exchange; shorter is safer - ticket_ttl_seconds: int = field(default_factory=lambda: _int(_env("OA_SSO_TICKET_TTL_SECONDS", "60"), 60)) + ticket_ttl_seconds: int = field( + default_factory=lambda: _int(_env("OA_SSO_TICKET_TTL_SECONDS", "60"), 60) + ) # Page path the callback 302s to after establishing the session - redirect_path: str = field(default_factory=lambda: _env("OA_SSO_REDIRECT_PATH", "/").strip() or "/") + redirect_path: str = field( + default_factory=lambda: _env("OA_SSO_REDIRECT_PATH", "/").strip() or "/" + ) @dataclass(frozen=True) @@ -181,11 +204,21 @@ class DatabaseSettings: # Default/fallback SQLite DBs go in the system temp dir (absolute path) — a # relative path follows the process CWD and leaves stray .db files in the # repo root / src/backend - url: str = field(default_factory=lambda: _env("DATABASE_URL", f"sqlite:///{tempfile.gettempdir()}/hugagent.db")) - sqlite_fallback_url: str = field(default_factory=lambda: _env("SQLITE_FALLBACK_URL", f"sqlite:///{tempfile.gettempdir()}/hugagent_dev.db")) + url: str = field( + default_factory=lambda: _env( + "DATABASE_URL", f"sqlite:///{tempfile.gettempdir()}/hugagent.db" + ) + ) + sqlite_fallback_url: str = field( + default_factory=lambda: _env( + "SQLITE_FALLBACK_URL", f"sqlite:///{tempfile.gettempdir()}/hugagent_dev.db" + ) + ) echo: bool = field(default_factory=lambda: _bool(_env("DB_ECHO", "false"))) pool_size: int = field(default_factory=lambda: _int(_env("DB_POOL_SIZE", "20"), 20)) - pool_max_overflow: int = field(default_factory=lambda: _int(_env("DB_POOL_MAX_OVERFLOW", "10"), 10)) + pool_max_overflow: int = field( + default_factory=lambda: _int(_env("DB_POOL_MAX_OVERFLOW", "10"), 10) + ) pool_timeout: int = field(default_factory=lambda: _int(_env("DB_POOL_TIMEOUT", "30"), 30)) @@ -195,7 +228,9 @@ class LLMSettings: api_key: str = field(default_factory=lambda: _env("API_KEY", "")) base_model_name: str = field(default_factory=lambda: _env("BASE_MODEL_NAME", "")) enable_summary: bool = field(default_factory=lambda: _bool(_env("ENABLE_SUMMARY", "true"))) - summary_max_rounds: int = field(default_factory=lambda: _int(_env("SUMMARY_MAX_ROUNDS", "3"), 3)) + summary_max_rounds: int = field( + default_factory=lambda: _int(_env("SUMMARY_MAX_ROUNDS", "3"), 3) + ) @dataclass(frozen=True) @@ -206,26 +241,48 @@ class MemorySettings: embed_model: str = field(default_factory=lambda: _env("MEM0_EMBED_MODEL", "qwen3_embedding_8b")) embed_api_key: str = field(default_factory=lambda: _env("MEM0_EMBED_API_KEY", "sk-placeholder")) embed_dims: int = field(default_factory=lambda: _int(_env("MEM0_EMBED_DIMS", "1024"), 1024)) - model_name: str = field(default_factory=lambda: _env("MEMORY_MODEL_NAME", _env("BASE_MODEL_NAME", "deepseek-chat"))) + model_name: str = field( + default_factory=lambda: _env("MEMORY_MODEL_NAME", _env("BASE_MODEL_NAME", "deepseek-chat")) + ) model_url: str = field(default_factory=lambda: _env("MEMORY_MODEL_URL", _env("MODEL_URL", ""))) - api_key: str = field(default_factory=lambda: _env("MEMORY_API_KEY", _env("API_KEY", "sk-placeholder"))) + api_key: str = field( + default_factory=lambda: _env("MEMORY_API_KEY", _env("API_KEY", "sk-placeholder")) + ) milvus_url: str = field(default_factory=lambda: _env("MILVUS_URL", "http://milvus:19530")) milvus_token: str = field(default_factory=lambda: _env("MILVUS_TOKEN", "")) neo4j_url: str = field(default_factory=lambda: _env("NEO4J_URL", "bolt://neo4j:7687")) neo4j_username: str = field(default_factory=lambda: _env("NEO4J_USERNAME", "neo4j")) - neo4j_password: str = field(default_factory=lambda: _env("NEO4J_PASSWORD", "hugagent_neo4j_2026")) + neo4j_password: str = field( + default_factory=lambda: _env("NEO4J_PASSWORD", "hugagent_neo4j_2026") + ) # ── Layered memory additions ───────────────────────────────── - layered_enabled: bool = field(default_factory=lambda: _bool(_env("MEMORY_LAYERED_ENABLED", "true"))) + layered_enabled: bool = field( + default_factory=lambda: _bool(_env("MEMORY_LAYERED_ENABLED", "true")) + ) audit_enabled: bool = field(default_factory=lambda: _bool(_env("MEMORY_AUDIT_ENABLED", "true"))) - retrieval_budget_ms: int = field(default_factory=lambda: _int(_env("MEMORY_RETRIEVAL_BUDGET_MS", "600"), 600)) - bg_max_concurrency: int = field(default_factory=lambda: _int(_env("MEMORY_BG_MAX_CONCURRENCY", "8"), 8)) - extract_timeout_s: int = field(default_factory=lambda: _int(_env("MEMORY_EXTRACT_TIMEOUT_S", "30"), 30)) - profile_max_chars: int = field(default_factory=lambda: _int(_env("MEMORY_PROFILE_MAX_CHARS", "1500"), 1500)) - fact_default_ttl_days: int = field(default_factory=lambda: _int(_env("MEMORY_FACT_DEFAULT_TTL_DAYS", "180"), 180)) + retrieval_budget_ms: int = field( + default_factory=lambda: _int(_env("MEMORY_RETRIEVAL_BUDGET_MS", "600"), 600) + ) + bg_max_concurrency: int = field( + default_factory=lambda: _int(_env("MEMORY_BG_MAX_CONCURRENCY", "8"), 8) + ) + extract_timeout_s: int = field( + default_factory=lambda: _int(_env("MEMORY_EXTRACT_TIMEOUT_S", "30"), 30) + ) + profile_max_chars: int = field( + default_factory=lambda: _int(_env("MEMORY_PROFILE_MAX_CHARS", "1500"), 1500) + ) + fact_default_ttl_days: int = field( + default_factory=lambda: _int(_env("MEMORY_FACT_DEFAULT_TTL_DAYS", "180"), 180) + ) frozen_topk: int = field(default_factory=lambda: _int(_env("MEMORY_FROZEN_TOPK", "5"), 5)) - breaker_threshold: int = field(default_factory=lambda: _int(_env("MEMORY_BREAKER_THRESHOLD", "3"), 3)) - breaker_cooldown_s: int = field(default_factory=lambda: _int(_env("MEMORY_BREAKER_COOLDOWN_S", "60"), 60)) + breaker_threshold: int = field( + default_factory=lambda: _int(_env("MEMORY_BREAKER_THRESHOLD", "3"), 3) + ) + breaker_cooldown_s: int = field( + default_factory=lambda: _int(_env("MEMORY_BREAKER_COOLDOWN_S", "60"), 60) + ) @dataclass(frozen=True) @@ -246,10 +303,9 @@ def root(self) -> Path: @dataclass(frozen=True) class KnowledgeBaseSettings: backend: str = field(default_factory=lambda: (_env("KNOWLEDGE_BASE") or "").strip().lower()) - dify_url: str = field(default_factory=lambda: _env("DIFY_URL") or _env("DIFY_BASE_URL") or "") - dify_api_key: str = field(default_factory=lambda: _env("DIFY_API_KEY") or _env("DIFY_AUTH_TOKEN") or "") - dify_allowed_dataset_ids: str = field(default_factory=lambda: (_env("DIFY_ALLOWED_DATASET_IDS") or "").strip()) - detail_content_max_chars: int = field(default_factory=lambda: _int(_env("KB_DETAIL_CONTENT_MAX_CHARS", "50000"), 50000)) + detail_content_max_chars: int = field( + default_factory=lambda: _int(_env("KB_DETAIL_CONTENT_MAX_CHARS", "50000"), 50000) + ) reranker_url: str = field(default_factory=lambda: _env("RERANKER_URL", "").rstrip("/")) reranker_model: str = field(default_factory=lambda: _env("RERANKER_MODEL", "")) reranker_api_key: str = field(default_factory=lambda: _env("RERANKER_API_KEY", "")) @@ -264,19 +320,33 @@ class RedisSettings: # 5s, so the default would fire at the exact instant the 5s XREAD returns # its nil reply → spurious "Timeout reading from redis" on every idle # window. 30s gives a 25s safety margin while still catching dead sockets. - socket_timeout: int = field(default_factory=lambda: _int(_env("REDIS_SOCKET_TIMEOUT", "30"), 30)) + socket_timeout: int = field( + default_factory=lambda: _int(_env("REDIS_SOCKET_TIMEOUT", "30"), 30) + ) @dataclass(frozen=True) class ServerSettings: env: str = field(default_factory=lambda: _env("ENV", "dev").lower()) - port: int = field(default_factory=lambda: _int(_env("PORT", _env("BACKEND_PORT", "3001")), 3001)) + port: int = field( + default_factory=lambda: _int(_env("PORT", _env("BACKEND_PORT", "3001")), 3001) + ) cors_origins: str = field(default_factory=lambda: _env("CORS_ORIGINS", "")) - max_request_size: int = field(default_factory=lambda: _int(_env("MAX_REQUEST_SIZE", str(50 * 1024 * 1024)), 50 * 1024 * 1024)) + max_request_size: int = field( + default_factory=lambda: _int( + _env("MAX_REQUEST_SIZE", str(50 * 1024 * 1024)), 50 * 1024 * 1024 + ) + ) log_level: str = field(default_factory=lambda: _env("LOG_LEVEL", "INFO").upper()) - log_file_path: str = field(default_factory=lambda: (_env("LOG_FILE_PATH") or "/app/logs/backend.log").strip()) - log_file_max_bytes: int = field(default_factory=lambda: _int((_env("LOG_FILE_MAX_BYTES") or "10485760").strip(), 10485760)) - log_file_backup_count: int = field(default_factory=lambda: _int((_env("LOG_FILE_BACKUP_COUNT") or "5").strip(), 5)) + log_file_path: str = field( + default_factory=lambda: (_env("LOG_FILE_PATH") or "/app/logs/backend.log").strip() + ) + log_file_max_bytes: int = field( + default_factory=lambda: _int((_env("LOG_FILE_MAX_BYTES") or "10485760").strip(), 10485760) + ) + log_file_backup_count: int = field( + default_factory=lambda: _int((_env("LOG_FILE_BACKUP_COUNT") or "5").strip(), 5) + ) # Hostname of the dedicated `mcp` container — every MCP server is # reached at ``http://:/mcp/``. Defaults to the docker # service name; override with MCP_HOST=127.0.0.1 for local debugging @@ -292,17 +362,31 @@ def is_prod(self) -> bool: class RateLimitSettings: enabled: bool = field(default_factory=lambda: _bool(_env("RATE_LIMIT_ENABLED", "true"))) storage: str = field(default_factory=lambda: _env("RATE_LIMIT_STORAGE", "memory://")) - cb_user_center_threshold: int = field(default_factory=lambda: _int(_env("CB_USER_CENTER_THRESHOLD", "5"), 5)) - cb_user_center_timeout: int = field(default_factory=lambda: _int(_env("CB_USER_CENTER_TIMEOUT", "60"), 60)) - cb_model_api_threshold: int = field(default_factory=lambda: _int(_env("CB_MODEL_API_THRESHOLD", "10"), 10)) - cb_model_api_timeout: int = field(default_factory=lambda: _int(_env("CB_MODEL_API_TIMEOUT", "30"), 30)) - cb_storage_threshold: int = field(default_factory=lambda: _int(_env("CB_STORAGE_THRESHOLD", "5"), 5)) - cb_storage_timeout: int = field(default_factory=lambda: _int(_env("CB_STORAGE_TIMEOUT", "60"), 60)) + cb_user_center_threshold: int = field( + default_factory=lambda: _int(_env("CB_USER_CENTER_THRESHOLD", "5"), 5) + ) + cb_user_center_timeout: int = field( + default_factory=lambda: _int(_env("CB_USER_CENTER_TIMEOUT", "60"), 60) + ) + cb_model_api_threshold: int = field( + default_factory=lambda: _int(_env("CB_MODEL_API_THRESHOLD", "10"), 10) + ) + cb_model_api_timeout: int = field( + default_factory=lambda: _int(_env("CB_MODEL_API_TIMEOUT", "30"), 30) + ) + cb_storage_threshold: int = field( + default_factory=lambda: _int(_env("CB_STORAGE_THRESHOLD", "5"), 5) + ) + cb_storage_timeout: int = field( + default_factory=lambda: _int(_env("CB_STORAGE_TIMEOUT", "60"), 60) + ) @dataclass(frozen=True) class RoutingSettings: - strategy: str = field(default_factory=lambda: (_env("ROUTER_STRATEGY") or "main_only").strip().lower()) + strategy: str = field( + default_factory=lambda: (_env("ROUTER_STRATEGY") or "main_only").strip().lower() + ) followup_enabled: bool = field(default_factory=lambda: _bool(_env("FOLLOWUP_ENABLED", "true"))) @@ -349,7 +433,9 @@ class CompactionSettings: @dataclass(frozen=True) class PromptSettings: - provider: str = field(default_factory=lambda: (_env("PROMPT_PROVIDER") or "filesystem").strip().lower()) + provider: str = field( + default_factory=lambda: (_env("PROMPT_PROVIDER") or "filesystem").strip().lower() + ) dir: str = field(default_factory=lambda: _env("PROMPT_DIR", "")) inline_template: str = field(default_factory=lambda: _env("PROMPT_INLINE_TEMPLATE", "")) config_path: str = field(default_factory=lambda: _env("JX_PROMPT_CONFIG", "")) @@ -371,21 +457,39 @@ class SandboxSettings: - ``cube``: Tencent CubeSandbox (external E2B-compatible MicroVM node) """ - provider: str = field(default_factory=lambda: _env("SANDBOX_PROVIDER", "script_runner").strip().lower()) + provider: str = field( + default_factory=lambda: _env("SANDBOX_PROVIDER", "script_runner").strip().lower() + ) # script_runner sidecar call parameters. Defaults to the compose service name (not the container name), decoupled from container renames. - runner_url: str = field(default_factory=lambda: _env("SANDBOX_RUNNER_URL", "http://script-runner:8900")) + runner_url: str = field( + default_factory=lambda: _env("SANDBOX_RUNNER_URL", "http://script-runner:8900") + ) enabled: bool = field(default_factory=lambda: _bool(_env("SANDBOX_TOOLS_ENABLED", "false"))) - default_timeout: int = field(default_factory=lambda: _int(_env("SANDBOX_TOOLS_TIMEOUT", "30"), 30)) - max_timeout: int = field(default_factory=lambda: _int(_env("SANDBOX_TOOLS_MAX_TIMEOUT", "120"), 120)) + default_timeout: int = field( + default_factory=lambda: _int(_env("SANDBOX_TOOLS_TIMEOUT", "30"), 30) + ) + max_timeout: int = field( + default_factory=lambda: _int(_env("SANDBOX_TOOLS_MAX_TIMEOUT", "120"), 120) + ) # opensandbox - opensandbox_domain: str = field(default_factory=lambda: _env("OPENSANDBOX_DOMAIN", "http://opensandbox:8080")) + opensandbox_domain: str = field( + default_factory=lambda: _env("OPENSANDBOX_DOMAIN", "http://opensandbox:8080") + ) opensandbox_api_key: str = field(default_factory=lambda: _env("OPENSANDBOX_API_KEY", "")) - opensandbox_image: str = field(default_factory=lambda: _env("OPENSANDBOX_IMAGE", "opensandbox/code-interpreter:v1.0.2")) - opensandbox_default_timeout_s: int = field(default_factory=lambda: _int(_env("OPENSANDBOX_DEFAULT_TIMEOUT_S", "1800"), 1800)) - opensandbox_ready_timeout_s: int = field(default_factory=lambda: _int(_env("OPENSANDBOX_READY_TIMEOUT_S", "90"), 90)) - opensandbox_request_timeout_s: int = field(default_factory=lambda: _int(_env("OPENSANDBOX_REQUEST_TIMEOUT_S", "120"), 120)) + opensandbox_image: str = field( + default_factory=lambda: _env("OPENSANDBOX_IMAGE", "opensandbox/code-interpreter:v1.0.2") + ) + opensandbox_default_timeout_s: int = field( + default_factory=lambda: _int(_env("OPENSANDBOX_DEFAULT_TIMEOUT_S", "1800"), 1800) + ) + opensandbox_ready_timeout_s: int = field( + default_factory=lambda: _int(_env("OPENSANDBOX_READY_TIMEOUT_S", "90"), 90) + ) + opensandbox_request_timeout_s: int = field( + default_factory=lambda: _int(_env("OPENSANDBOX_REQUEST_TIMEOUT_S", "120"), 120) + ) # Direct execd connection: bypasses the OpenSandbox server's proxy # forwarding (measured ~3s extra buffering overhead per request — file # read/write 3s→<5ms, commands 4s→1s). Connects directly to the sandbox @@ -393,37 +497,57 @@ class SandboxSettings: # be reachable on the same docker network (true for this project's compose # topology). A one-time reachability probe runs at creation; if # unreachable, it auto-falls back to the server proxy with zero feature loss. - opensandbox_direct_execd_enabled: bool = field(default_factory=lambda: _bool(_env("OPENSANDBOX_DIRECT_EXECD", "true"))) + opensandbox_direct_execd_enabled: bool = field( + default_factory=lambda: _bool(_env("OPENSANDBOX_DIRECT_EXECD", "true")) + ) # endpoint fastpath: skips the server's GET /endpoints/{port} (measured # fixed ~3.3s hard server-side latency; Sandbox.create calls it once each # for execd+egress → saves ~6.6s cold start). The returned proxy endpoint # is a deterministic string constructible locally. Enabled only in insecure # mode (empty api_key): with an api_key the server may stuff auth/routing # headers into the endpoint that cannot be replicated locally. - opensandbox_endpoint_fastpath_enabled: bool = field(default_factory=lambda: _bool(_env("OPENSANDBOX_ENDPOINT_FASTPATH", "true"))) + opensandbox_endpoint_fastpath_enabled: bool = field( + default_factory=lambda: _bool(_env("OPENSANDBOX_ENDPOINT_FASTPATH", "true")) + ) # Warm pool: fill each bucket to min_idle right after process startup so the first user gets a warm sandbox - opensandbox_pool_jupyter_min_idle: int = field(default_factory=lambda: _int(_env("OPENSANDBOX_POOL_JUPYTER_MIN_IDLE", "2"), 2)) - opensandbox_pool_jupyter_max_idle: int = field(default_factory=lambda: _int(_env("OPENSANDBOX_POOL_JUPYTER_MAX_IDLE", "3"), 3)) - opensandbox_pool_light_min_idle: int = field(default_factory=lambda: _int(_env("OPENSANDBOX_POOL_LIGHT_MIN_IDLE", "2"), 2)) - opensandbox_pool_light_max_idle: int = field(default_factory=lambda: _int(_env("OPENSANDBOX_POOL_LIGHT_MAX_IDLE", "5"), 5)) - opensandbox_pool_max_total: int = field(default_factory=lambda: _int(_env("OPENSANDBOX_POOL_MAX_TOTAL", "20"), 20)) + opensandbox_pool_jupyter_min_idle: int = field( + default_factory=lambda: _int(_env("OPENSANDBOX_POOL_JUPYTER_MIN_IDLE", "2"), 2) + ) + opensandbox_pool_jupyter_max_idle: int = field( + default_factory=lambda: _int(_env("OPENSANDBOX_POOL_JUPYTER_MAX_IDLE", "3"), 3) + ) + opensandbox_pool_light_min_idle: int = field( + default_factory=lambda: _int(_env("OPENSANDBOX_POOL_LIGHT_MIN_IDLE", "2"), 2) + ) + opensandbox_pool_light_max_idle: int = field( + default_factory=lambda: _int(_env("OPENSANDBOX_POOL_LIGHT_MAX_IDLE", "5"), 5) + ) + opensandbox_pool_max_total: int = field( + default_factory=lambda: _int(_env("OPENSANDBOX_POOL_MAX_TOTAL", "20"), 20) + ) # Liveness probe (GET /sandboxes/{id}) timeout (seconds) before taking an # idle sandbox out of the pool. Unreachable/timeout does not delete it — # left for acquire / the next round to handle. See # _OpenSandboxSessionMixin._probe_pooled_sandbox_alive. - opensandbox_pool_liveness_probe_timeout_s: int = field(default_factory=lambda: _int(_env("OPENSANDBOX_POOL_LIVENESS_PROBE_TIMEOUT_S", "5"), 5)) + opensandbox_pool_liveness_probe_timeout_s: int = field( + default_factory=lambda: _int(_env("OPENSANDBOX_POOL_LIVENESS_PROBE_TIMEOUT_S", "5"), 5) + ) # Active idle-reap threshold for persistent sessions (seconds). A chat-level # persistent sandbox with no business requests beyond this value is # destroyed by a background task instead of waiting for the server-side # 30min TTL. <=0 disables active reaping. A hard prerequisite for keeping # the pool from being saturated by idle sessions once file tools are # enabled in all modes (see docs/code-execution-merge-proposal §4.2). - opensandbox_idle_reap_threshold_s: int = field(default_factory=lambda: _int(_env("OPENSANDBOX_IDLE_REAP_S", "600"), 600)) + opensandbox_idle_reap_threshold_s: int = field( + default_factory=lambda: _int(_env("OPENSANDBOX_IDLE_REAP_S", "600"), 600) + ) # ─── Snapshot persistence (see internal design docs) ───────────── # Master switch: true → enable snapshot park/restore + background worker; # false → fall back to the status quo (idle sandboxes are lost when reaped; # on reconnect, bash returns 404 if the sandbox is already dead). - opensandbox_snapshot_enabled: bool = field(default_factory=lambda: _bool(_env("OPENSANDBOX_SNAPSHOT_ENABLED", "true"))) + opensandbox_snapshot_enabled: bool = field( + default_factory=lambda: _bool(_env("OPENSANDBOX_SNAPSHOT_ENABLED", "true")) + ) # When a session is idle beyond this value (seconds), the background worker # proactively snapshots + kills it to free resources. Default was 300s # (5min). Must be < opensandbox_default_timeout_s (1800), otherwise GC gets @@ -433,11 +557,17 @@ class SandboxSettings: # value, a typical user reading a response for 5 minutes got repeatedly # snapshot+killed and rebuilt (measured ~21s restore) — very poor UX. The # new value gives the Q2 idle pool ample time to handle short-term reuse. - opensandbox_idle_snapshot_threshold_s: int = field(default_factory=lambda: _int(_env("OPENSANDBOX_IDLE_SNAPSHOT_THRESHOLD_S", "1500"), 1500)) + opensandbox_idle_snapshot_threshold_s: int = field( + default_factory=lambda: _int(_env("OPENSANDBOX_IDLE_SNAPSHOT_THRESHOLD_S", "1500"), 1500) + ) # Snapshot retention in the DB (days); expired ones are deleted by the GC worker (DB row + opensandbox side). - opensandbox_snapshot_retention_days: int = field(default_factory=lambda: _int(_env("OPENSANDBOX_SNAPSHOT_RETENTION_DAYS", "7"), 7)) + opensandbox_snapshot_retention_days: int = field( + default_factory=lambda: _int(_env("OPENSANDBOX_SNAPSHOT_RETENTION_DAYS", "7"), 7) + ) # Max polling time (seconds) waiting for snapshot accept→Ready. Measured ~60s; 120s gives a 1× safety margin. - opensandbox_snapshot_wait_timeout_s: int = field(default_factory=lambda: _int(_env("OPENSANDBOX_SNAPSHOT_WAIT_TIMEOUT_S", "120"), 120)) + opensandbox_snapshot_wait_timeout_s: int = field( + default_factory=lambda: _int(_env("OPENSANDBOX_SNAPSHOT_WAIT_TIMEOUT_S", "120"), 120) + ) # Skill dependencies come only from the image bake (docker/Dockerfile.opensandbox); new dependencies go through the admin-panel rebuild. # The all-modes code-execution capability switch has moved out of settings: @@ -450,7 +580,9 @@ class SandboxSettings: # outright. Ops can set false to disable (trusted-path policy soft # constraint). With code_capability enabled this is the key protection # against the main agent accidentally modifying the user's private drive. - myspace_write_confirm: bool = field(default_factory=lambda: _bool(_env("MYSPACE_WRITE_CONFIRM", "true"))) + myspace_write_confirm: bool = field( + default_factory=lambda: _bool(_env("MYSPACE_WRITE_CONFIRM", "true")) + ) # User confirmation for automation (cron task) changes (borrows the §13 # MySpace write-confirm suspend gate). true (default): when the Agent @@ -459,7 +591,9 @@ class SandboxSettings: # interactive chats; channel (IM bot) runs and non-interactive modes # (batch/sub-agent/plan execution/scheduler-triggered) skip the prompt and # pass directly. Set false to disable. - automation_write_confirm: bool = field(default_factory=lambda: _bool(_env("AUTOMATION_WRITE_CONFIRM", "true"))) + automation_write_confirm: bool = field( + default_factory=lambda: _bool(_env("AUTOMATION_WRITE_CONFIRM", "true")) + ) # ─── Plan F: direct myspace_cache bind-mount ─────────────────────────── # true (default): use an OpenSandbox host Volume to bind the backend's @@ -535,24 +669,36 @@ class SandboxSettings: ) # cube (Tencent CubeSandbox, E2B-compatible MicroVM; external node, no local sidecar needed) - cube_api_url: str = field(default_factory=lambda: _env("CUBE_API_URL", "http://cube-node:38473")) + cube_api_url: str = field( + default_factory=lambda: _env("CUBE_API_URL", "http://cube-node:38473") + ) cube_api_key: str = field(default_factory=lambda: _env("CUBE_API_KEY", "")) # Data-plane sandbox domain, may include a port (when cube-proxy is not on 443); the SDK builds https://{port}-{id}.{domain} from it - cube_api_sandbox_domain: str = field(default_factory=lambda: _env("CUBE_API_SANDBOX_DOMAIN", "cube.app:38573")) + cube_api_sandbox_domain: str = field( + default_factory=lambda: _env("CUBE_API_SANDBOX_DOMAIN", "cube.app:38573") + ) # Required: sandbox template id (CubeSandbox requires it when creating a sandbox) cube_template: str = field(default_factory=lambda: _env("CUBE_TEMPLATE", "").strip()) # Sandbox TTL (seconds); CubeMaster does not yet support set_timeout renewal, so this is the at-creation upper bound - cube_default_timeout_s: int = field(default_factory=lambda: _int(_env("CUBE_DEFAULT_TIMEOUT_S", "1800"), 1800)) - cube_request_timeout_s: int = field(default_factory=lambda: _int(_env("CUBE_REQUEST_TIMEOUT_S", "120"), 120)) + cube_default_timeout_s: int = field( + default_factory=lambda: _int(_env("CUBE_DEFAULT_TIMEOUT_S", "1800"), 1800) + ) + cube_request_timeout_s: int = field( + default_factory=lambda: _int(_env("CUBE_REQUEST_TIMEOUT_S", "120"), 120) + ) # mkcert rootCA bundle (in-container path); when non-empty, injects SSL_CERT_FILE so the SDK trusts the self-signed cert cube_ca_bundle: str = field(default_factory=lambda: _env("CUBE_CA_BUNDLE", "").strip()) # Active idle-reap threshold for session sandboxes (seconds); <=0 disables active reaping (rely on CubeSandbox's built-in TTL) - cube_idle_reap_threshold_s: int = field(default_factory=lambda: _int(_env("CUBE_IDLE_REAP_S", "600"), 600)) + cube_idle_reap_threshold_s: int = field( + default_factory=lambda: _int(_env("CUBE_IDLE_REAP_S", "600"), 600) + ) # Warm-pool target idle count: refilled in the background to this value on # startup / after each take, so a new session's first run gets a warm # sandbox, skipping AsyncSandbox.create's MicroVM cold start (~10s). <=0 # disables the warm pool. - cube_pool_min_idle: int = field(default_factory=lambda: _int(_env("CUBE_POOL_MIN_IDLE", "2"), 2)) + cube_pool_min_idle: int = field( + default_factory=lambda: _int(_env("CUBE_POOL_MIN_IDLE", "2"), 2) + ) # Sandbox owner tag: written into metadata["hugagent-owner"], used so the # startup orphan sweep only recognizes this environment's sandboxes. # CubeMaster (MVP) does not honor the sandbox TTL; a backend restart loses @@ -574,9 +720,15 @@ class SandboxSettings: # pre-pushed (oversized skills like ppt-master remain on-demand, paying one # upload only when actually used — avoids wasting tens of MB of pushes on # text-only sessions). - cube_skill_prepush: bool = field(default_factory=lambda: _bool(_env("CUBE_SKILL_PREPUSH", "true"))) - cube_skill_prepush_max_mb: int = field(default_factory=lambda: _int(_env("CUBE_SKILL_PREPUSH_MAX_MB", "20"), 20)) - cube_skill_prepush_concurrency: int = field(default_factory=lambda: _int(_env("CUBE_SKILL_PREPUSH_CONCURRENCY", "3"), 3)) + cube_skill_prepush: bool = field( + default_factory=lambda: _bool(_env("CUBE_SKILL_PREPUSH", "true")) + ) + cube_skill_prepush_max_mb: int = field( + default_factory=lambda: _int(_env("CUBE_SKILL_PREPUSH_MAX_MB", "20"), 20) + ) + cube_skill_prepush_concurrency: int = field( + default_factory=lambda: _int(_env("CUBE_SKILL_PREPUSH_CONCURRENCY", "3"), 3) + ) # ─── Cube remote template rebuild (the admin "Sandbox deps → App deps" path when provider=cube) ── # The backend does not build locally; instead it scps the aggregated @@ -586,34 +738,63 @@ class SandboxSettings: # .env. An empty host means "remote rebuild not configured": the rebuild # endpoint errors out for cube and directs users to the manual docs path. # See core/services/cube_template_builder.py for details. - cube_node_ssh_host: str = field(default_factory=lambda: _env("CUBE_NODE_SSH_HOST", _env("CUBE_NODE_IP", "")).strip()) - cube_node_ssh_port: int = field(default_factory=lambda: _int(_env("CUBE_NODE_SSH_PORT", "22"), 22)) - cube_node_ssh_user: str = field(default_factory=lambda: _env("CUBE_NODE_SSH_USER", "root").strip()) + cube_node_ssh_host: str = field( + default_factory=lambda: _env("CUBE_NODE_SSH_HOST", _env("CUBE_NODE_IP", "")).strip() + ) + cube_node_ssh_port: int = field( + default_factory=lambda: _int(_env("CUBE_NODE_SSH_PORT", "22"), 22) + ) + cube_node_ssh_user: str = field( + default_factory=lambda: _env("CUBE_NODE_SSH_USER", "root").strip() + ) # In-container private key path (docker-compose mounts the host key read-only); empty uses the default ssh key chain cube_node_ssh_key: str = field(default_factory=lambda: _env("CUBE_NODE_SSH_KEY", "").strip()) # Build context directory on the node (Dockerfile.cube-sandbox and dependency manifests are scp'd flat into it; the src tree is rsync'd on first deployment) - cube_build_ctx_dir: str = field(default_factory=lambda: _env("CUBE_BUILD_CTX_DIR", "/opt/cube-build").strip()) - cube_build_image_tag: str = field(default_factory=lambda: _env("CUBE_BUILD_IMAGE_TAG", "hugagent-cube-sandbox:latest").strip()) - cube_build_registry: str = field(default_factory=lambda: _env("CUBE_BUILD_REGISTRY", "127.0.0.1:5000").strip()) + cube_build_ctx_dir: str = field( + default_factory=lambda: _env("CUBE_BUILD_CTX_DIR", "/opt/cube-build").strip() + ) + cube_build_image_tag: str = field( + default_factory=lambda: _env("CUBE_BUILD_IMAGE_TAG", "hugagent-cube-sandbox:latest").strip() + ) + cube_build_registry: str = field( + default_factory=lambda: _env("CUBE_BUILD_REGISTRY", "127.0.0.1:5000").strip() + ) # create-from-image resource/port parameters (aligned with the existing READY template) - cube_build_writable_layer: str = field(default_factory=lambda: _env("CUBE_BUILD_WRITABLE_LAYER", "8Gi").strip()) + cube_build_writable_layer: str = field( + default_factory=lambda: _env("CUBE_BUILD_WRITABLE_LAYER", "8Gi").strip() + ) cube_build_cpu: int = field(default_factory=lambda: _int(_env("CUBE_BUILD_CPU", "2000"), 2000)) - cube_build_memory: int = field(default_factory=lambda: _int(_env("CUBE_BUILD_MEMORY", "4000"), 4000)) + cube_build_memory: int = field( + default_factory=lambda: _int(_env("CUBE_BUILD_MEMORY", "4000"), 4000) + ) # Comma-separated exposed ports; probe port + path - cube_build_expose_ports: str = field(default_factory=lambda: _env("CUBE_BUILD_EXPOSE_PORTS", "49983,49999").strip()) - cube_build_probe_port: int = field(default_factory=lambda: _int(_env("CUBE_BUILD_PROBE_PORT", "49999"), 49999)) - cube_build_probe_path: str = field(default_factory=lambda: _env("CUBE_BUILD_PROBE_PATH", "/health").strip()) + cube_build_expose_ports: str = field( + default_factory=lambda: _env("CUBE_BUILD_EXPOSE_PORTS", "49983,49999").strip() + ) + cube_build_probe_port: int = field( + default_factory=lambda: _int(_env("CUBE_BUILD_PROBE_PORT", "49999"), 49999) + ) + cube_build_probe_path: str = field( + default_factory=lambda: _env("CUBE_BUILD_PROBE_PATH", "/health").strip() + ) # Separate timeout ceilings for build / registration (seconds) - cube_build_timeout_s: int = field(default_factory=lambda: _int(_env("CUBE_BUILD_TIMEOUT_S", "1800"), 1800)) - cube_build_register_timeout_s: int = field(default_factory=lambda: _int(_env("CUBE_BUILD_REGISTER_TIMEOUT_S", "900"), 900)) + cube_build_timeout_s: int = field( + default_factory=lambda: _int(_env("CUBE_BUILD_TIMEOUT_S", "1800"), 1800) + ) + cube_build_register_timeout_s: int = field( + default_factory=lambda: _int(_env("CUBE_BUILD_REGISTER_TIMEOUT_S", "900"), 900) + ) # General - max_concurrent: int = field(default_factory=lambda: _int(_env("SANDBOX_MAX_CONCURRENT", "4"), 4)) + max_concurrent: int = field( + default_factory=lambda: _int(_env("SANDBOX_MAX_CONCURRENT", "4"), 4) + ) @dataclass(frozen=True) class EditionSettings: """Edition facade (CE/EE). Main repo defaults to ee; the CE derived tree sets JX_EDITION=ce via .env.""" + edition: str = field(default_factory=lambda: _env("JX_EDITION", "ee").strip().lower() or "ee") @property @@ -621,24 +802,6 @@ def is_ee(self) -> bool: return self.edition == "ee" -@dataclass(frozen=True) -class LicenseSettings: - """Offline license (Ed25519-signed file, verified in-process — no license service, no online dependency). - - - ``license_key_path``: path to the license file; when unset and - ``required=False``, runs fully featured as an "internal deployment" - (compatible with the current fully-managed / test-machine setups). - - ``required=True``: private-delivery mode — without a valid license all EE - capability bits are disabled. - - ``grace_days``: post-expiry grace period (days); features are retained - during grace while probes raise alerts. - """ - license_key_path: str = field(default_factory=lambda: _env("LICENSE_KEY_PATH", "").strip()) - public_key: str = field(default_factory=lambda: _env("LICENSE_PUBLIC_KEY", "").strip()) - required: bool = field(default_factory=lambda: _bool(_env("JX_LICENSE_REQUIRED", "false"))) - grace_days: int = field(default_factory=lambda: _int(_env("LICENSE_GRACE_DAYS", "14"), 14)) - - @dataclass(frozen=True) class GatewaySettings: """External model gateway (LiteLLM Proxy data plane). @@ -650,8 +813,11 @@ class GatewaySettings: is never sent to the frontend. Without ``master_key`` configured, the control-plane endpoints return errors. """ + enabled: bool = field(default_factory=lambda: _bool(_env("MODEL_GATEWAY_ENABLED", "false"))) - admin_url: str = field(default_factory=lambda: _env("LITELLM_ADMIN_URL", "http://litellm:4000").strip().rstrip("/")) + admin_url: str = field( + default_factory=lambda: _env("LITELLM_ADMIN_URL", "http://litellm:4000").strip().rstrip("/") + ) master_key: str = field(default_factory=lambda: _env("LITELLM_MASTER_KEY", "").strip()) timeout: int = field(default_factory=lambda: _int(_env("MODEL_GATEWAY_TIMEOUT", "15"), 15)) @@ -659,9 +825,14 @@ class GatewaySettings: @dataclass(frozen=True) class BrandingSettings: """Single source of brand defaults — the in-code fallback stays neutral; deployment branding comes from env / the content_blocks DB seed.""" - product_name: str = field(default_factory=lambda: _env("BRAND_PRODUCT_NAME", "智能体平台").strip()) + + product_name: str = field( + default_factory=lambda: _env("BRAND_PRODUCT_NAME", "智能体平台").strip() + ) org_name: str = field(default_factory=lambda: _env("BRAND_ORG_NAME", "").strip()) - powered_by_visible: bool = field(default_factory=lambda: _bool(_env("BRAND_POWERED_BY", "true"))) + powered_by_visible: bool = field( + default_factory=lambda: _bool(_env("BRAND_POWERED_BY", "true")) + ) @dataclass(frozen=True) @@ -678,6 +849,7 @@ class DeploySettings: behaviors like "in-process hosted sub-services + frontend static mounting"; each capability is still explicitly driven by its own env (written by the CLI). """ + profile: str = field(default_factory=lambda: _env("DEPLOY_PROFILE", "").strip().lower()) @property @@ -688,6 +860,7 @@ def is_local(self) -> bool: @dataclass(frozen=True) class AppSettings: """Top-level settings container — one read from env at startup.""" + deploy: DeploySettings = field(default_factory=DeploySettings) auth: AuthSettings = field(default_factory=AuthSettings) sso: SSOSettings = field(default_factory=SSOSettings) @@ -707,7 +880,6 @@ class AppSettings: industry: IndustrySettings = field(default_factory=IndustrySettings) sandbox: SandboxSettings = field(default_factory=SandboxSettings) edition: EditionSettings = field(default_factory=EditionSettings) - license: LicenseSettings = field(default_factory=LicenseSettings) gateway: GatewaySettings = field(default_factory=GatewaySettings) branding: BrandingSettings = field(default_factory=BrandingSettings) diff --git a/src/backend/core/db/edition_tables.py b/src/backend/core/db/edition_tables.py index 4cf77a66..d4c12554 100644 --- a/src/backend/core/db/edition_tables.py +++ b/src/backend/core/db/edition_tables.py @@ -1,103 +1,26 @@ -"""CE/EE table-creation boundary (single source of truth for migration baseline D1). - -The ``core.db.models`` package is shared by both editions (defining the EE table -classes is harmless in itself), but CE must not create empty EE-only tables. This -module provides the set of EE-only table names, filtered by both table-creation -entry points: - -- the CE branch of ``core.db.engine.init_db`` (SQLite startup fallback) -- the CE overlay migration baseline ``alembic/versions/ce_0001_initial.py`` - -EE (``JX_EDITION=ee``, covering all license states including internal/licensed) -always creates the full set of tables, matching historical behavior; filtering -happens only when ``JX_EDITION=ce``. - -Maintenance rule: when adding an EE-only model, add its table name to -``EE_ONLY_TABLES`` at the same time. Every name in the set must actually exist in -the metadata (``ce_create_all`` asserts this), so a renamed model that is not -updated here cannot silently degrade into full-table creation. - -Note a few tables that "look EE but are actually needed by CE" — do NOT add them -to the set: -``admin_prompt_parts`` (read at runtime by prompts/prompt_runtime), -``memory_sanitizer_rules`` (queried unconditionally by core/memory/sanitizer), -``admin_skills``/``admin_mcp_servers`` (personal self-service skills/MCP, owner-isolated), -``marketplace_submissions`` (CE marketplace.py keeps the submission endpoint). -""" - -EE_ONLY_TABLES: frozenset[str] = frozenset({ - # Multi-tenant / SSO / invitations - "teams", - "team_members", - "team_folders", - "invite_codes", - # Role/permission system (named capability bundles + assignment to teams/users) — - # CE is single-tenant with no roles; resolution degrades to "personal → system" - "roles", - "role_assignments", - # KB permission grants (per-user/team authorization; CE public KBs are always - # visible to everyone, so this table is not created) - "kb_grants", - # Marketplace item visibility grants (scoped whitelist; CE has no admin console, - # marketplace is always visible to everyone) - "marketplace_visibility_grants", - # Audit (user-facing audit + memory audit — CE's memory audit is a stub that - # writes no table; /v1/memories/audit short-circuits under the - # audit_enabled=False default) - "audit_logs", - "memory_audit", - # Billing - "model_pricing", - # Data sources / metadata governance (DB connections, table/column metadata, Golden SQL) - "data_sources", - "ds_table_meta", - "ds_column_meta", - "ds_golden_sql", - # External model gateway: virtual-key mirror (control plane, EE-only capability model_gateway) - "gateway_virtual_keys", - # Persistent sandbox rebuild / skill distillation (admin-console drafts + distillation run records) - "sandbox_rebuilds", - "admin_skill_drafts", - "distillation_runs", -}) +"""CE table creation without enterprise model-name knowledge.""" def ce_create_all(bind) -> list[str]: - """Create all non-EE tables under CE and return the list of created table names (idempotent, dialect-aware). - - Creates tables on a cloned MetaData: cross-boundary FK constraints in CE tables - that point at EE tables (projects/artifacts → teams/team_folders, plan D3 - "keep the column, always NULL") would make PostgreSQL fail table creation - because the referenced tables don't exist — so those constraints are stripped - on the clone (the columns themselves remain), the original metadata is - untouched, and ORM mappings are unaffected. - """ - import core.db.models # noqa: F401 registers all ORM tables into metadata - from sqlalchemy import MetaData - + """Create the CE metadata and remove foreign keys to omitted edition tables.""" + import core.db.models # noqa: F401 from core.db.engine import Base + from sqlalchemy import MetaData - metadata = Base.metadata - missing = EE_ONLY_TABLES - set(metadata.tables) - if missing: - raise RuntimeError( - f"EE_ONLY_TABLES 含 metadata 中不存在的表（模型改名漏同步？）: {sorted(missing)}" - ) clone = MetaData() - for name, table in metadata.tables.items(): - if name not in EE_ONLY_TABLES: - table.to_metadata(clone) + for table in Base.metadata.tables.values(): + table.to_metadata(clone) + + present = set(clone.tables) for table in clone.tables.values(): - for fkc in list(table.foreign_key_constraints): - referred = { - elem.target_fullname.rsplit(".", 1)[0] for elem in fkc.elements - } - if referred & EE_ONLY_TABLES: - # Both the constraints set and the column-level ForeignKey elements - # must be removed — foreign_key_constraints / DDL ordering read the latter - table.constraints.discard(fkc) - for elem in fkc.elements: - elem.parent.foreign_keys.discard(elem) - table.foreign_keys.discard(elem) + for constraint in list(table.foreign_key_constraints): + targets = {element.target_fullname.rsplit(".", 1)[0] for element in constraint.elements} + if targets <= present: + continue + table.constraints.discard(constraint) + for element in constraint.elements: + element.parent.foreign_keys.discard(element) + table.foreign_keys.discard(element) + clone.create_all(bind=bind) return sorted(clone.tables) diff --git a/src/backend/core/db/model_extensions.py b/src/backend/core/db/model_extensions.py new file mode 100644 index 00000000..74ce8399 --- /dev/null +++ b/src/backend/core/db/model_extensions.py @@ -0,0 +1,61 @@ +"""Community ORM extensions: personal-resource schema only.""" + +from sqlalchemy import CheckConstraint + + +class ProjectEditionFields: + pass + + +def project_edition_table_args() -> tuple: + return (CheckConstraint("kind = 'personal'", name="ck_projects_kind_personal"),) + + +class ArtifactEditionFields: + pass + + +def artifact_edition_table_args() -> tuple: + return () + + +class UserAgentEditionFields: + pass + + +def user_agent_edition_table_args() -> tuple: + return ( + CheckConstraint( + "owner_type IN ('admin', 'user')", + name="user_agents_owner_type_check", + ), + ) + + +class ChatSessionEditionFields: + pass + + +def chat_session_edition_table_args() -> tuple: + return () + + +class MarketplaceListingEditionFields: + pass + + +EDITION_MODEL_EXPORTS = {} + + +__all__ = [ + "ArtifactEditionFields", + "ChatSessionEditionFields", + "EDITION_MODEL_EXPORTS", + "MarketplaceListingEditionFields", + "ProjectEditionFields", + "UserAgentEditionFields", + "artifact_edition_table_args", + "chat_session_edition_table_args", + "project_edition_table_args", + "user_agent_edition_table_args", +] diff --git a/src/backend/core/db/models/__init__.py b/src/backend/core/db/models/__init__.py index 40b61c27..989e5ec3 100644 --- a/src/backend/core/db/models/__init__.py +++ b/src/backend/core/db/models/__init__.py @@ -1,29 +1,48 @@ -"""SQLAlchemy ORM models. - -Split into submodules by domain; this __init__ re-exports all model classes verbatim (plus Base/JSONType/INETType), -so `from core.db.models import X` and `from core.db.models import *` both stay unchanged. -""" +"""Community-edition ORM model exports.""" from core.db.engine import Base -from sqlalchemy import JSON, String -from sqlalchemy.dialects.postgresql import JSONB, INET - -JSONType = JSON().with_variant(JSONB(), "postgresql") -INETType = String(45).with_variant(INET(), "postgresql") - -from core.db.models.identity import UserShadow, LocalUser, InviteCode, Team, TeamMember, TeamFolder, UserFolder, UserApiKey, DingTalkConnection, LarkConnection, EmailConnection, ChannelConnection, Role, RoleAssignment -from core.db.models.project import Project, ProjectFavorite -from core.db.models.chat import ChatSession, ChatSessionUserState, ChatMessage, ChatRun, MessageFeedback, ChatSandboxSnapshot -from core.db.models.knowledge import KBSpace, KBDocument, KBChunk, CatalogOverride, KBGrant +from core.db.models.admin import ( + AdminMcpServer, + AdminPromptPart, + AdminSkill, + InstalledPlugin, + MarketplaceListingState, + MarketplaceSubmission, + PluginMarketPackage, + PluginMarketSkillExclusion, + SkillDependencyRequest, +) +from core.db.models.agent import ( + AgentLoop, + AgentMarketSubmission, + LoopIteration, + Plan, + PlanStep, + UserAgent, +) from core.db.models.artifact import Artifact, ContentBlock -from core.db.models.config import ModelProvider, SystemConfig, ModelRoleAssignment, ModelPricing, GatewayVirtualKey -from core.db.models.admin import AdminSkill, SandboxRebuild, SkillDependencyRequest, AdminPromptPart, AdminMcpServer, AdminSkillDraft, MarketplaceSubmission, InstalledPlugin, PluginMarketPackage, PluginMarketSkillExclusion, MarketplaceListingState, MarketplaceVisibilityGrant -from core.db.models.agent import UserAgent, AgentMarketSubmission, Plan, PlanStep, AgentLoop, LoopIteration -from core.db.models.automation import ScheduledTask, ScheduledTaskRun, DistillationRun, PersonaDistillJob, BatchPlan -from core.db.models.logs import ToolCallLog, SubAgentCallLog, SkillCallLog, AuditLog -from core.db.models.memory import ProfileMemory, MemoryAudit, MemorySanitizerRule -from core.db.models.datasource import DataSource, DsTableMeta, DsColumnMeta, DsGoldenSql -from core.db.models.site import Site, SiteKV, SiteSubmission +from core.db.models.automation import BatchPlan, PersonaDistillJob, ScheduledTask, ScheduledTaskRun +from core.db.models.chat import ( + ChatMessage, + ChatRun, + ChatSandboxSnapshot, + ChatSession, + MessageFeedback, +) +from core.db.models.config import ModelProvider, ModelRoleAssignment, SystemConfig +from core.db.models.identity import ( + ChannelConnection, + DingTalkConnection, + EmailConnection, + LarkConnection, + LocalUser, + UserApiKey, + UserFolder, + UserShadow, +) +from core.db.models.knowledge import CatalogOverride, KBChunk, KBDocument, KBSpace +from core.db.models.logs import SkillCallLog, SubAgentCallLog, ToolCallLog +from core.db.models.memory import MemorySanitizerRule, ProfileMemory from core.db.models.ontology import ( OntologyDraft, OntologyEnforcementEvent, @@ -31,85 +50,12 @@ OntologyPackVersion, OntologyReviewRun, ) +from core.db.models.project import Project, ProjectFavorite +from core.db.models.site import Site, SiteKV, SiteSubmission +from sqlalchemy import JSON, String +from sqlalchemy.dialects.postgresql import INET, JSONB + +JSONType = JSON().with_variant(JSONB(), "postgresql") +INETType = String(45).with_variant(INET(), "postgresql") -__all__ = [ - "Site", - "SiteKV", - "SiteSubmission", - "DataSource", - "DsTableMeta", - "DsColumnMeta", - "DsGoldenSql", - "Base", - "JSONType", - "INETType", - "UserShadow", - "UserApiKey", - "DingTalkConnection", - "LarkConnection", - "EmailConnection", - "ChannelConnection", - "LocalUser", - "InviteCode", - "Team", - "TeamMember", - "TeamFolder", - "Role", - "RoleAssignment", - "UserFolder", - "Project", - "ProjectFavorite", - "ChatSession", - "ChatSessionUserState", - "ChatMessage", - "ChatRun", - "MessageFeedback", - "ChatSandboxSnapshot", - "KBSpace", - "KBDocument", - "KBChunk", - "CatalogOverride", - "KBGrant", - "Artifact", - "ContentBlock", - "ModelProvider", - "SystemConfig", - "ModelRoleAssignment", - "ModelPricing", - "GatewayVirtualKey", - "AdminSkill", - "SandboxRebuild", - "SkillDependencyRequest", - "AdminPromptPart", - "AdminMcpServer", - "AdminSkillDraft", - "MarketplaceSubmission", - "InstalledPlugin", - "PluginMarketPackage", - "PluginMarketSkillExclusion", - "MarketplaceListingState", - "MarketplaceVisibilityGrant", - "UserAgent", - "AgentMarketSubmission", - "Plan", - "PlanStep", - "AgentLoop", - "LoopIteration", - "ScheduledTask", - "ScheduledTaskRun", - "DistillationRun", - "PersonaDistillJob", - "BatchPlan", - "ToolCallLog", - "SubAgentCallLog", - "SkillCallLog", - "AuditLog", - "ProfileMemory", - "MemoryAudit", - "MemorySanitizerRule", - "OntologyPack", - "OntologyPackVersion", - "OntologyEnforcementEvent", - "OntologyReviewRun", - "OntologyDraft", -] +__all__ = [name for name in globals() if not name.startswith("_")] diff --git a/src/backend/core/db/models/admin.py b/src/backend/core/db/models/admin.py index a23290f8..395dcb52 100644 --- a/src/backend/core/db/models/admin.py +++ b/src/backend/core/db/models/admin.py @@ -1,13 +1,25 @@ """SQLAlchemy ORM models — admin console.""" from datetime import datetime, timezone + +from core.db.engine import Base +from core.db.model_extensions import MarketplaceListingEditionFields from sqlalchemy import ( - Column, String, Integer, Boolean, Text, TIMESTAMP, - ForeignKey, CheckConstraint, UniqueConstraint, Index, Numeric, JSON + JSON, + TIMESTAMP, + Boolean, + CheckConstraint, + Column, + ForeignKey, + Index, + Integer, + Numeric, + String, + Text, + UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import JSONB, INET -from sqlalchemy.orm import relationship, mapped_column -from core.db.engine import Base +from sqlalchemy.dialects.postgresql import INET, JSONB +from sqlalchemy.orm import mapped_column, relationship JSONType = JSON().with_variant(JSONB(), "postgresql") INETType = String(45).with_variant(INET(), "postgresql") @@ -15,24 +27,29 @@ class AdminSkill(Base): """Admin-managed skills stored in DB (replaces filesystem storage).""" + __tablename__ = "admin_skills" - skill_id = Column(String(100), primary_key=True) - skill_content = Column(Text, nullable=False) # full SKILL.md source text - display_name = Column(String(255), nullable=False) # denormalized field, avoids re-parsing - description = Column(Text, nullable=False) - user_intro = Column(Text, nullable=True) # user-facing intro shown in the capability center (Markdown) - version = Column(String(50), nullable=False, default="1.0.0") - tags = Column(JSONType, default=list) + skill_id = Column(String(100), primary_key=True) + skill_content = Column(Text, nullable=False) # full SKILL.md source text + display_name = Column(String(255), nullable=False) # denormalized field, avoids re-parsing + description = Column(Text, nullable=False) + user_intro = Column( + Text, nullable=True + ) # user-facing intro shown in the capability center (Markdown) + version = Column(String(50), nullable=False, default="1.0.0") + tags = Column(JSONType, default=list) allowed_tools = Column(JSONType, default=list) - extra_files = Column(JSONType, default=dict) # {filename: content} - dependencies = Column(JSONType, default=dict) # {"pip":[...], "npm":[...], "apt":[...], "warnings":[...]} - is_enabled = Column(Boolean, nullable=False, default=True) + extra_files = Column(JSONType, default=dict) # {filename: content} + dependencies = Column( + JSONType, default=dict + ) # {"pip":[...], "npm":[...], "apt":[...], "warnings":[...]} + is_enabled = Column(Boolean, nullable=False, default=True) # Dependency-readiness status: 'ready' = usable; 'installing' = missing packages detected, # admin notified, waiting for a sandbox rebuild to install them (soft-disabled: excluded from # runtime loading, but still shown in the skill list with a status label). Automatically # returns to 'ready' after the admin's sandbox rebuild succeeds. - dep_status = Column(String(20), nullable=False, default="ready") + dep_status = Column(String(20), nullable=False, default="ready") # NULL = global skill (created by admin, visible to all users); non-null = a private skill # self-uploaded by a user, visible and usable only to that user. owner_user_id = Column(String(64), nullable=True) @@ -40,9 +57,9 @@ class AdminSkill(Base): # it is deleted precisely by this, without harming user-created skills. # See internal design docs. source_plugin = Column(String(100), nullable=True) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - created_by = Column(String(64)) + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) + updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) + created_by = Column(String(64)) __table_args__ = ( Index("idx_admin_skills_is_enabled", "is_enabled"), @@ -52,34 +69,6 @@ class AdminSkill(Base): ) -class SandboxRebuild(Base): - """Tracks an admin-triggered rebuild of sandbox images (script-runner / opensandbox). - - Used to apply aggregated skill dependencies. Each row represents one - asyncio-backed background job; status drives the admin progress UI. - """ - __tablename__ = "sandbox_rebuilds" - - run_id = Column(String(64), primary_key=True) - status = Column(String(20), nullable=False, default="pending") - targets = Column(JSONType, default=list) # ["script-runner", "opensandbox"] - deps_hash = Column(String(64)) # SHA256 of aggregated manifests at trigger time - started_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, nullable=False) - completed_at = Column(TIMESTAMP(timezone=True)) - log = Column(Text, default="") # docker compose stdout/stderr (truncated to 256 KB) - error_message = Column(Text) - created_by = Column(String(64)) - - __table_args__ = ( - CheckConstraint( - "status IN ('pending', 'running', 'completed', 'failed', 'cancelled')", - name="sandbox_rebuilds_status_check", - ), - Index("idx_sandbox_rebuilds_started_at", "started_at"), - Index("idx_sandbox_rebuilds_status", "status"), - ) - - class SkillDependencyRequest(Base): """A "skill missing dependencies, pending admin installation" record. @@ -92,19 +81,20 @@ class SkillDependencyRequest(Base): skill stays soft-disabled with ``dep_status='rejected'``, and the reason is surfaced to the user). """ + __tablename__ = "skill_dependency_requests" - request_id = Column(String(64), primary_key=True) - skill_id = Column(String(100), nullable=False, index=True) - user_id = Column(String(64), index=True) # user who triggered the import (maps to a person) - missing = Column(JSONType, default=dict) # {"pip":[...], "npm":[...], "apt":[...]} - status = Column(String(16), nullable=False, default="pending") # pending | satisfied | rejected - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, nullable=False) + request_id = Column(String(64), primary_key=True) + skill_id = Column(String(100), nullable=False, index=True) + user_id = Column(String(64), index=True) # user who triggered the import (maps to a person) + missing = Column(JSONType, default=dict) # {"pip":[...], "npm":[...], "apt":[...]} + status = Column(String(16), nullable=False, default="pending") # pending | satisfied | rejected + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, nullable=False) satisfied_at = Column(TIMESTAMP(timezone=True)) - satisfied_by_run_id = Column(String(64)) # the sandbox_rebuild run_id that fulfilled it - reason = Column(Text) # rejection reason (optional), surfaced to the user - rejected_at = Column(TIMESTAMP(timezone=True)) - rejected_by = Column(String(64)) # identifier of the rejecting admin + satisfied_by_run_id = Column(String(64)) # the sandbox_rebuild run_id that fulfilled it + reason = Column(Text) # rejection reason (optional), surfaced to the user + rejected_at = Column(TIMESTAMP(timezone=True)) + rejected_by = Column(String(64)) # identifier of the rejecting admin __table_args__ = ( CheckConstraint( @@ -117,16 +107,17 @@ class SkillDependencyRequest(Base): class AdminPromptPart(Base): """Admin-managed prompt parts stored in DB (overrides filesystem prompts).""" + __tablename__ = "admin_prompt_parts" - part_id = Column(String(100), primary_key=True) # e.g. "system/00_role" - content = Column(Text, nullable=False) - display_name = Column(String(255), nullable=False) - sort_order = Column(Integer, nullable=False, default=0) - is_enabled = Column(Boolean, nullable=False, default=True) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - created_by = Column(String(64)) + part_id = Column(String(100), primary_key=True) # e.g. "system/00_role" + content = Column(Text, nullable=False) + display_name = Column(String(255), nullable=False) + sort_order = Column(Integer, nullable=False, default=0) + is_enabled = Column(Boolean, nullable=False, default=True) + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) + updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) + created_by = Column(String(64)) __table_args__ = ( Index("idx_admin_prompt_parts_sort_order", "sort_order"), @@ -136,33 +127,36 @@ class AdminPromptPart(Base): class AdminMcpServer(Base): """Admin-managed MCP server configurations stored in DB.""" + __tablename__ = "admin_mcp_servers" - server_id = Column(String(100), primary_key=True) + server_id = Column(String(100), primary_key=True) display_name = Column(String(255), nullable=False) - description = Column(Text, nullable=False, default="") - user_intro = Column(Text, nullable=True) # user-facing intro shown in the capability center (Markdown) - transport = Column(String(20), nullable=False, default="stdio") - command = Column(String(500)) - args = Column(JSONType, default=list) - url = Column(Text) - env_vars = Column(JSONType, default=dict) - env_inherit = Column(JSONType, default=list) - headers = Column(JSONType, default=dict) - is_stable = Column(Boolean, nullable=False, default=True) - is_enabled = Column(Boolean, nullable=False, default=True) - sort_order = Column(Integer, nullable=False, default=0) + description = Column(Text, nullable=False, default="") + user_intro = Column( + Text, nullable=True + ) # user-facing intro shown in the capability center (Markdown) + transport = Column(String(20), nullable=False, default="stdio") + command = Column(String(500)) + args = Column(JSONType, default=list) + url = Column(Text) + env_vars = Column(JSONType, default=dict) + env_inherit = Column(JSONType, default=list) + headers = Column(JSONType, default=dict) + is_stable = Column(Boolean, nullable=False, default=True) + is_enabled = Column(Boolean, nullable=False, default=True) + sort_order = Column(Integer, nullable=False, default=0) extra_config = Column(JSONType, default=dict) - tools_json = Column(JSONType, default=list) # cached tool list from discovery - icon = Column(String(500)) # optional icon URL (library path or uploaded asset) + tools_json = Column(JSONType, default=list) # cached tool list from discovery + icon = Column(String(500)) # optional icon URL (library path or uploaded asset) # NULL = global MCP (created by admin, visible to all users); non-null = a private remote MCP # self-added by a user, visible and usable only to that user. owner_user_id = Column(String(64), nullable=True) # Non-null = this MCP was installed/imported by a plugin (plugin slug); deleted precisely by this on plugin uninstall. source_plugin = Column(String(100), nullable=True) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - created_by = Column(String(64)) + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) + updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) + created_by = Column(String(64)) __table_args__ = ( CheckConstraint( @@ -187,31 +181,34 @@ class MarketplaceSubmission(Base): (source=community) and can be installed by everyone; the admin can reject at any time (including delisting already-listed items). """ + __tablename__ = "marketplace_submissions" - submission_id = Column(String(64), primary_key=True) + submission_id = Column(String(64), primary_key=True) # Marketplace slug / entry_name after listing (derived at submission time, # globally unique, does not clash with the preset marketplace catalog) - slug = Column(String(128), nullable=False, unique=True) - skill_id = Column(String(100), nullable=False) # source AdminSkill.skill_id - owner_user_id = Column(String(64), nullable=False) # applicant - submitter_name = Column(String(255), default="") # applicant display name (denormalized, for display) - - display_name = Column(String(255), nullable=False) - summary = Column(Text, default="") - category = Column(String(64), default="社区共享") - tags = Column(JSONType, default=list) - version = Column(String(50), default="1.0.0") - note = Column(Text, default="") # application note (for the admin) - - skill_content = Column(Text, nullable=False) # SKILL.md snapshot - extra_files = Column(JSONType, default=dict) # attachment snapshot {path: content} - - status = Column(String(16), nullable=False, default="pending") - review_note = Column(Text) # rejection reason / review note - reviewed_at = Column(TIMESTAMP(timezone=True)) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) + slug = Column(String(128), nullable=False, unique=True) + skill_id = Column(String(100), nullable=False) # source AdminSkill.skill_id + owner_user_id = Column(String(64), nullable=False) # applicant + submitter_name = Column( + String(255), default="" + ) # applicant display name (denormalized, for display) + + display_name = Column(String(255), nullable=False) + summary = Column(Text, default="") + category = Column(String(64), default="社区共享") + tags = Column(JSONType, default=list) + version = Column(String(50), default="1.0.0") + note = Column(Text, default="") # application note (for the admin) + + skill_content = Column(Text, nullable=False) # SKILL.md snapshot + extra_files = Column(JSONType, default=dict) # attachment snapshot {path: content} + + status = Column(String(16), nullable=False, default="pending") + review_note = Column(Text) # rejection reason / review note + reviewed_at = Column(TIMESTAMP(timezone=True)) + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) + updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) __table_args__ = ( CheckConstraint( @@ -223,54 +220,6 @@ class MarketplaceSubmission(Base): ) -class AdminSkillDraft(Base): - """Skill distillation draft — a candidate skill automatically distilled from a successful trajectory, pending admin review.""" - __tablename__ = "admin_skill_drafts" - - draft_id = Column(String(64), primary_key=True) - proposed_skill_id = Column(String(128), nullable=False) - decision = Column(String(16), nullable=False) # new_skill | patch - patch_target_id = Column(String(128)) - display_name = Column(String(255)) - description = Column(Text) - tags = Column(JSONType, default=list) - allowed_tools = Column(JSONType, default=list) - version = Column(String(32), default="0.1.0") - skill_content = Column(Text, nullable=False) - extra_files = Column(JSONType, default=dict) - - source_chat_id = Column(String(64), nullable=False) - source_user_id = Column(String(64), nullable=False) - source_trace_ids = Column(JSONType, default=list) - trajectory_digest = Column(Text) - distillation_cost_usd = Column(Numeric(8, 4)) - # auto = nightly single-session auto distillation; colleague = background persona-level colleague distillation (mirrored from persona_distill_jobs) - draft_kind = Column(String(16), nullable=False, default="auto") - source_job_id = Column(String(64)) # colleague drafts link back to persona_distill_jobs.job_id - - review_status = Column(String(16), nullable=False, default="pending") - reviewer_id = Column(String(64)) - reviewer_comment = Column(Text) - rejected_reason = Column(Text) - reviewed_at = Column(TIMESTAMP(timezone=True)) - - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - - __table_args__ = ( - CheckConstraint( - "decision IN ('new_skill', 'patch')", - name="admin_skill_drafts_decision_check", - ), - CheckConstraint( - "review_status IN ('pending', 'approved', 'rejected')", - name="admin_skill_drafts_status_check", - ), - Index("idx_admin_skill_drafts_status", "review_status", "created_at"), - Index("idx_admin_skill_drafts_source", "source_chat_id"), - ) - - class InstalledPlugin(Base): """An installed plugin (bundle) — the single source of truth for uninstall/upgrade. @@ -285,27 +234,28 @@ class InstalledPlugin(Base): plugins, and imported Codex plugins. See ``internal design docs``. """ + __tablename__ = "installed_plugins" # f"{slug}@{owner_user_id or 'global'}" — the same plugin can be installed privately by multiple users - install_id = Column(String(160), primary_key=True) - slug = Column(String(100), nullable=False) - name = Column(String(255), nullable=False) # display name - version = Column(String(50), nullable=False, default="1.0.0") - description = Column(Text, default="") - category = Column(String(64), default="") - icon = Column(String(500)) + install_id = Column(String(160), primary_key=True) + slug = Column(String(100), nullable=False) + name = Column(String(255), nullable=False) # display name + version = Column(String(50), nullable=False, default="1.0.0") + description = Column(Text, default="") + category = Column(String(64), default="") + icon = Column(String(500)) # NULL = global plugin (installed by admin, visible to all users); non-null = a user's private install. owner_user_id = Column(String(64), nullable=True) # builtin (built-in package) / imported_claude (imported CC plugin) / imported_codex (imported Codex plugin) - source = Column(String(24), nullable=False, default="builtin") + source = Column(String(24), nullable=False, default="builtin") # Component ids actually persisted: {"skills":[...], "mcp":[...], "prompts":[...]} component_ids = Column(JSONType, default=dict) # Import report (imported / adapted / dropped), for front-end display import_report = Column(JSONType, default=dict) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - created_by = Column(String(64)) + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) + updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) + created_by = Column(String(64)) __table_args__ = ( CheckConstraint( @@ -333,27 +283,26 @@ class PluginMarketPackage(Base): fallback**. Install records still go into ``InstalledPlugin``; this table is only an "installable source". """ + __tablename__ = "plugin_market_packages" - slug = Column(String(100), primary_key=True) - name = Column(String(255), nullable=False) - version = Column(String(50), nullable=False, default="1.0.0") - description = Column(Text, default="") - category = Column(String(64), default="") - icon = Column(String(500)) + slug = Column(String(100), primary_key=True) + name = Column(String(255), nullable=False) + version = Column(String(50), nullable=False, default="1.0.0") + description = Column(Text, default="") + category = Column(String(64), default="") + icon = Column(String(500)) # Package kind: native / claude / codex (determined by normalize), display only - kind = Column(String(16), nullable=False, default="native") - skills_count = Column(Integer, nullable=False, default=0) + kind = Column(String(16), nullable=False, default="native") + skills_count = Column(Integer, nullable=False, default=0) required_secrets = Column(JSONType, default=list) has_admin_config = Column(Boolean, nullable=False, default=False) - package_b64 = Column(Text, nullable=False) # base64 of the originally uploaded zip - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - created_by = Column(String(64)) + package_b64 = Column(Text, nullable=False) # base64 of the originally uploaded zip + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) + updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) + created_by = Column(String(64)) - __table_args__ = ( - Index("idx_plugin_market_packages_category", "category"), - ) + __table_args__ = (Index("idx_plugin_market_packages_category", "category"),) class PluginMarketSkillExclusion(Base): @@ -368,15 +317,16 @@ class PluginMarketSkillExclusion(Base): Installed instances are unaffected (those are install snapshots, managed separately via enable/disable and uninstall). """ + __tablename__ = "plugin_market_skill_exclusions" - slug = Column(String(100), primary_key=True) + slug = Column(String(100), primary_key=True) skill_name = Column(String(100), primary_key=True) created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) created_by = Column(String(64)) -class MarketplaceListingState(Base): +class MarketplaceListingState(MarketplaceListingEditionFields, Base): """Marketplace listing switch: controls whether a plugin/skill is shown in the (plugin/skill) marketplace. ``kind`` = ``plugin`` | ``skill``; ``item_id`` = plugin slug or skill @@ -388,46 +338,12 @@ class MarketplaceListingState(Base): uploaded DB items) still goes through the respective delete endpoints; this table only governs "listing visibility". - ``visibility``: visibility scope. ``public`` (default, missing row is - synonymous) = visible to everyone; ``scoped`` = visible only to - users/teams/roles granted in ``marketplace_visibility_grants``. Governs - only marketplace browsing and installation; does not retroactively - affect installed instances. """ + __tablename__ = "marketplace_listing_states" - kind = Column(String(16), primary_key=True) # plugin | skill - item_id = Column(String(160), primary_key=True) # plugin slug / skill marketplace slug - enabled = Column(Boolean, nullable=False, default=True) - visibility = Column(String(16), nullable=False, default="public") # public | scoped + kind = Column(String(16), primary_key=True) # plugin | skill + item_id = Column(String(160), primary_key=True) # plugin slug / skill marketplace slug + enabled = Column(Boolean, nullable=False, default=True) updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) updated_by = Column(String(64)) - - -class MarketplaceVisibilityGrant(Base): - """Marketplace item visibility-scope grant (the whitelist when ``visibility='scoped'``). - - ``kind``/``item_id`` mirror ``MarketplaceListingState``; - ``principal_type`` = ``user`` | ``team`` | ``role``, and matching any of - the three principal types grants visibility (union, no precedence). - EE-only table (configured server-side via the admin marketplace routes); - CE has no admin console and is always visible to everyone — the resolver - wraps queries against this table in try/except as a fallback (see - core/auth/marketplace_visibility.py). - """ - __tablename__ = "marketplace_visibility_grants" - - kind = Column(String(16), primary_key=True) # plugin | skill | agent - item_id = Column(String(160), primary_key=True) - principal_type = Column(String(16), primary_key=True) # user | team | role - principal_id = Column(String(64), primary_key=True) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - created_by = Column(String(64)) - - __table_args__ = ( - CheckConstraint( - "principal_type IN ('user', 'team', 'role')", - name="marketplace_visibility_grants_principal_type_check", - ), - Index("idx_mkt_vis_grants_principal", "principal_type", "principal_id"), - ) diff --git a/src/backend/core/db/models/agent.py b/src/backend/core/db/models/agent.py index 45003baf..434cc42c 100644 --- a/src/backend/core/db/models/agent.py +++ b/src/backend/core/db/models/agent.py @@ -1,58 +1,72 @@ """SQLAlchemy ORM models — user agents / plans.""" from datetime import datetime, timezone + +from core.db.engine import Base +from core.db.model_extensions import UserAgentEditionFields, user_agent_edition_table_args from sqlalchemy import ( - Column, String, Integer, BigInteger, Boolean, Text, TIMESTAMP, - ForeignKey, CheckConstraint, UniqueConstraint, Index, Numeric, JSON + JSON, + TIMESTAMP, + BigInteger, + Boolean, + CheckConstraint, + Column, + ForeignKey, + Index, + Integer, + Numeric, + String, + Text, + UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import JSONB, INET -from sqlalchemy.orm import relationship, mapped_column -from core.db.engine import Base +from sqlalchemy.dialects.postgresql import INET, JSONB +from sqlalchemy.orm import mapped_column, relationship JSONType = JSON().with_variant(JSONB(), "postgresql") INETType = String(45).with_variant(INET(), "postgresql") -class UserAgent(Base): +class UserAgent(UserAgentEditionFields, Base): """Custom sub-agent (admin-created or user-created).""" + __tablename__ = "user_agents" - agent_id = Column(String(64), primary_key=True) - owner_type = Column(String(10), nullable=False) # "admin" | "user" | "team" - user_id = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE")) - # Team sub-agent: when owner_type='team' it hangs under a team, visible and usable by all members, managed by the team owner/admin. - team_id = Column(String(64), ForeignKey("teams.team_id", ondelete="CASCADE")) - name = Column(String(255), nullable=False) - avatar = Column(Text) - description = Column(Text, default="") + agent_id = Column(String(64), primary_key=True) + owner_type = Column(String(10), nullable=False) + user_id = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE")) + name = Column(String(255), nullable=False) + avatar = Column(Text) + description = Column(Text, default="") # Core config - system_prompt = Column(Text, nullable=False, default="") - welcome_message = Column(Text, default="") + system_prompt = Column(Text, nullable=False, default="") + welcome_message = Column(Text, default="") suggested_questions = Column(JSONType, default=list) # Capability bindings - mcp_server_ids = Column(JSONType, default=list) - skill_ids = Column(JSONType, default=list) - kb_ids = Column(JSONType, default=list) + mcp_server_ids = Column(JSONType, default=list) + skill_ids = Column(JSONType, default=list) + kb_ids = Column(JSONType, default=list) # Bound plugins (install_id list): a plugin is a "skills + MCP" bundle unit, expanded # at runtime into its component skills/tools. Complementary to skill_ids/mcp_server_ids — # those two only store "loose" non-plugin capabilities. - plugin_ids = Column(JSONType, default=list) + plugin_ids = Column(JSONType, default=list) # Model config - model_provider_id = Column(String(64), ForeignKey("model_providers.provider_id", ondelete="SET NULL")) - temperature = Column(Numeric(3, 2)) - max_tokens = Column(Integer) + model_provider_id = Column( + String(64), ForeignKey("model_providers.provider_id", ondelete="SET NULL") + ) + temperature = Column(Numeric(3, 2)) + max_tokens = Column(Integer) # Runtime controls - max_iters = Column(Integer, default=10) - timeout = Column(Integer, default=120) - is_enabled = Column(Boolean, default=True) - sort_order = Column(Integer, default=0) + max_iters = Column(Integer, default=10) + timeout = Column(Integer, default=120) + is_enabled = Column(Boolean, default=True) + sort_order = Column(Integer, default=0) # Advanced config - extra_config = Column(JSONType, default=dict) + extra_config = Column(JSONType, default=dict) # Sub-agent marketplace origin: non-empty means this agent was "install-cloned" from a # marketplace listing (value = marketplace slug). Used for the "installed" badge in the @@ -60,24 +74,18 @@ class UserAgent(Base): source_market_slug = Column(String(128)) # Metadata - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - created_by = Column(String(64)) + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) + updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) + created_by = Column(String(64)) # Relationships - user = relationship("UserShadow", foreign_keys=[user_id], back_populates="user_agents") - model_provider = relationship("ModelProvider") + user = relationship("UserShadow", foreign_keys=[user_id], back_populates="user_agents") + model_provider = relationship("ModelProvider") __table_args__ = ( - CheckConstraint("owner_type IN ('admin', 'user', 'team')", name="user_agents_owner_type_check"), - # team_id exists only under the team scope; must be empty for all other scopes - CheckConstraint( - "(owner_type <> 'team' AND team_id IS NULL) OR (owner_type = 'team' AND team_id IS NOT NULL)", - name="user_agents_team_scope_check", - ), + *user_agent_edition_table_args(), Index("idx_user_agents_owner_type", "owner_type"), Index("idx_user_agents_user_id", "user_id"), - Index("idx_user_agents_team_id", "team_id"), Index("idx_user_agents_is_enabled", "is_enabled"), Index("idx_user_agents_sort_order", "sort_order"), Index("idx_user_agents_updated_at", "updated_at"), @@ -99,39 +107,42 @@ class AgentMarketSubmission(Base): admin uploads to the marketplace are distinguished by the sentinel owner ``__admin_upload__``. """ + __tablename__ = "agent_market_submissions" - submission_id = Column(String(64), primary_key=True) + submission_id = Column(String(64), primary_key=True) # Marketplace slug once listed (derived from name at submission, globally unique, never colliding with the preset marketplace catalog) - slug = Column(String(128), nullable=False, unique=True) - agent_id = Column(String(64), nullable=False) # source UserAgent.agent_id - owner_user_id = Column(String(64), nullable=False) # applicant (or __admin_upload__) - submitter_name = Column(String(255), default="") # applicant display name (denormalized for display) + slug = Column(String(128), nullable=False, unique=True) + agent_id = Column(String(64), nullable=False) # source UserAgent.agent_id + owner_user_id = Column(String(64), nullable=False) # applicant (or __admin_upload__) + submitter_name = Column( + String(255), default="" + ) # applicant display name (denormalized for display) # Marketplace display metadata - name = Column(String(255), nullable=False) - avatar = Column(Text) - description = Column(Text, default="") - summary = Column(Text, default="") - category = Column(String(64), default="通用助手") - tags = Column(JSONType, default=list) - version = Column(String(50), default="1.0.0") - note = Column(Text, default="") # application note (for the admin) + name = Column(String(255), nullable=False) + avatar = Column(Text) + description = Column(Text, default="") + summary = Column(Text, default="") + category = Column(String(64), default="通用助手") + tags = Column(JSONType, default=list) + version = Column(String(50), default="1.0.0") + note = Column(Text, default="") # application note (for the admin) # Sub-agent content snapshot (decoupled from the source UserAgent) - system_prompt = Column(Text, default="") - welcome_message = Column(Text, default="") - suggested_questions = Column(JSONType, default=list) + system_prompt = Column(Text, default="") + welcome_message = Column(Text, default="") + suggested_questions = Column(JSONType, default=list) # {temperature, max_tokens, max_iters, timeout} model_config_snapshot = Column(JSONType, default=dict) # {skill_ids, mcp_server_ids, plugin_ids, kb_ids} - bindings_snapshot = Column(JSONType, default=dict) + bindings_snapshot = Column(JSONType, default=dict) - status = Column(String(16), nullable=False, default="pending") - review_note = Column(Text) # rejection reason / review note - reviewed_at = Column(TIMESTAMP(timezone=True)) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) + status = Column(String(16), nullable=False, default="pending") + review_note = Column(Text) # rejection reason / review note + reviewed_at = Column(TIMESTAMP(timezone=True)) + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) + updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) __table_args__ = ( CheckConstraint( @@ -145,23 +156,30 @@ class AgentMarketSubmission(Base): class Plan(Base): """Plan mode - plans table""" + __tablename__ = "plans" - plan_id = Column(String(64), primary_key=True) - user_id = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False) - title = Column(String(500), nullable=False) - description = Column(Text, default="") - task_input = Column(Text, nullable=False) - status = Column(String(20), nullable=False, default="draft") - total_steps = Column(Integer, default=0) + plan_id = Column(String(64), primary_key=True) + user_id = Column( + String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False + ) + title = Column(String(500), nullable=False) + description = Column(Text, default="") + task_input = Column(Text, nullable=False) + status = Column(String(20), nullable=False, default="draft") + total_steps = Column(Integer, default=0) completed_steps = Column(Integer, default=0) - result_summary = Column(Text) - extra_data = Column("metadata", JSONType, default={}) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - - steps = relationship("PlanStep", back_populates="plan", - cascade="all, delete-orphan", order_by="PlanStep.step_order") + result_summary = Column(Text) + extra_data = Column("metadata", JSONType, default={}) + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) + updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) + + steps = relationship( + "PlanStep", + back_populates="plan", + cascade="all, delete-orphan", + order_by="PlanStep.step_order", + ) __table_args__ = ( CheckConstraint( @@ -175,23 +193,24 @@ class Plan(Base): class PlanStep(Base): """Plan mode - steps table""" + __tablename__ = "plan_steps" - step_id = Column(String(64), primary_key=True) - plan_id = Column(String(64), ForeignKey("plans.plan_id", ondelete="CASCADE"), nullable=False) - step_order = Column(Integer, nullable=False) - title = Column(String(500), nullable=False) - description = Column(Text, default="") - expected_tools = Column(JSONType, default=list) + step_id = Column(String(64), primary_key=True) + plan_id = Column(String(64), ForeignKey("plans.plan_id", ondelete="CASCADE"), nullable=False) + step_order = Column(Integer, nullable=False) + title = Column(String(500), nullable=False) + description = Column(Text, default="") + expected_tools = Column(JSONType, default=list) expected_skills = Column(JSONType, default=list) expected_agents = Column(JSONType, default=list) - status = Column(String(20), nullable=False, default="pending") - result_summary = Column(Text) - tool_calls_log = Column(JSONType, default=list) - ai_output = Column(Text) - error_message = Column(Text) - started_at = Column(TIMESTAMP(timezone=True)) - completed_at = Column(TIMESTAMP(timezone=True)) + status = Column(String(20), nullable=False, default="pending") + result_summary = Column(Text) + tool_calls_log = Column(JSONType, default=list) + ai_output = Column(Text) + error_message = Column(Text) + started_at = Column(TIMESTAMP(timezone=True)) + completed_at = Column(TIMESTAMP(timezone=True)) plan = relationship("Plan", back_populates="steps") @@ -211,28 +230,35 @@ class AgentLoop(Base): sandbox; the DB stores lineage/budget/audit indexes. See internal design docs (§3.1). CE table (not in EE_ONLY_TABLES). """ + __tablename__ = "agent_loops" - loop_id = Column(String(64), primary_key=True) - user_id = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False) - chat_id = Column(String(64)) - title = Column(String(500), default="") + loop_id = Column(String(64), primary_key=True) + user_id = Column( + String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False + ) + chat_id = Column(String(64)) + title = Column(String(500), default="") # goal_spec: {objective, acceptance_criteria[], verify_cmd, score_regex, target_score, maximize} - goal_spec = Column(JSONType, default=dict) + goal_spec = Column(JSONType, default=dict) # budget: {max_iters, max_wall_clock_s, max_tokens, max_subagents} - budget = Column(JSONType, default=dict) + budget = Column(JSONType, default=dict) workspace_session = Column(String(128)) - status = Column(String(24), nullable=False, default="created") - iteration_count = Column(Integer, default=0) - tokens_spent = Column(BigInteger, default=0) - final_score = Column(Numeric) - result_summary = Column(Text) - extra_data = Column("metadata", JSONType, default=dict) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - - iterations = relationship("LoopIteration", back_populates="loop", - cascade="all, delete-orphan", order_by="LoopIteration.seq") + status = Column(String(24), nullable=False, default="created") + iteration_count = Column(Integer, default=0) + tokens_spent = Column(BigInteger, default=0) + final_score = Column(Numeric) + result_summary = Column(Text) + extra_data = Column("metadata", JSONType, default=dict) + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) + updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) + + iterations = relationship( + "LoopIteration", + back_populates="loop", + cascade="all, delete-orphan", + order_by="LoopIteration.seq", + ) __table_args__ = ( CheckConstraint( @@ -247,24 +273,25 @@ class AgentLoop(Base): class LoopIteration(Base): """Per-iteration audit trail of an autonomous loop (decision log). See §3.2.""" + __tablename__ = "loop_iterations" - iteration_id = Column(String(64), primary_key=True) - loop_id = Column(String(64), ForeignKey("agent_loops.loop_id", ondelete="CASCADE"), nullable=False) - seq = Column(Integer, nullable=False) - run_id = Column(String(64)) - verdict = Column(String(20)) - score = Column(Numeric) - evidence = Column(Text) # environment evidence (verify output) - reasoning = Column(Text) # evaluator feedback / rationale - handoff_summary = Column(Text) - tool_calls = Column(Integer, default=0) - tokens = Column(Integer, default=0) - decided_by = Column(String(20)) # environment / llm / fallback - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) + iteration_id = Column(String(64), primary_key=True) + loop_id = Column( + String(64), ForeignKey("agent_loops.loop_id", ondelete="CASCADE"), nullable=False + ) + seq = Column(Integer, nullable=False) + run_id = Column(String(64)) + verdict = Column(String(20)) + score = Column(Numeric) + evidence = Column(Text) # environment evidence (verify output) + reasoning = Column(Text) # evaluator feedback / rationale + handoff_summary = Column(Text) + tool_calls = Column(Integer, default=0) + tokens = Column(Integer, default=0) + decided_by = Column(String(20)) # environment / llm / fallback + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) loop = relationship("AgentLoop", back_populates="iterations") - __table_args__ = ( - Index("idx_loop_iterations_loop_id", "loop_id"), - ) + __table_args__ = (Index("idx_loop_iterations_loop_id", "loop_id"),) diff --git a/src/backend/core/db/models/artifact.py b/src/backend/core/db/models/artifact.py index 1abb0efd..3865b53d 100644 --- a/src/backend/core/db/models/artifact.py +++ b/src/backend/core/db/models/artifact.py @@ -1,33 +1,41 @@ """SQLAlchemy ORM models — artifacts / content blocks.""" from datetime import datetime, timezone + +from core.db.engine import Base +from core.db.model_extensions import ArtifactEditionFields, artifact_edition_table_args from sqlalchemy import ( - Column, String, Integer, BigInteger, Boolean, Text, TIMESTAMP, - ForeignKey, CheckConstraint, UniqueConstraint, Index, Numeric, JSON + JSON, + TIMESTAMP, + BigInteger, + Boolean, + CheckConstraint, + Column, + ForeignKey, + Index, + Integer, + Numeric, + String, + Text, + UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import JSONB, INET -from sqlalchemy.orm import relationship, mapped_column -from core.db.engine import Base +from sqlalchemy.dialects.postgresql import INET, JSONB +from sqlalchemy.orm import mapped_column, relationship JSONType = JSON().with_variant(JSONB(), "postgresql") INETType = String(45).with_variant(INET(), "postgresql") -class Artifact(Base): +class Artifact(ArtifactEditionFields, Base): """Artifact table - AI-generated files (reports, charts, etc.).""" + __tablename__ = "artifacts" artifact_id = Column(String(64), primary_key=True) chat_id = Column(String(64), ForeignKey("chat_sessions.chat_id", ondelete="SET NULL")) - user_id = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False) - # Team ownership (NULL = personal file; non-NULL means a team file, which can be further located to a specific folder via team_folder_id) - team_id = Column(String(64), ForeignKey("teams.team_id", ondelete="SET NULL"), nullable=True) - team_folder_id = Column( - String(64), - ForeignKey("team_folders.folder_id", ondelete="SET NULL"), - nullable=True, + user_id = Column( + String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False ) - # Personal folder ownership (used only when team_id is NULL, i.e. personal files; mutually exclusive with team_folder_id, guaranteed by the service layer) user_folder_id = Column( String(64), ForeignKey("user_folders.folder_id", ondelete="SET NULL"), @@ -55,16 +63,19 @@ class Artifact(Base): session = relationship("ChatSession", back_populates="artifacts") __table_args__ = ( + *artifact_edition_table_args(), CheckConstraint("size_bytes > 0", name="artifacts_size_check"), - CheckConstraint("type IN ('report', 'chart', 'document', 'code', 'other')", name="artifacts_type_check"), + CheckConstraint( + "type IN ('report', 'chart', 'document', 'code', 'other')", name="artifacts_type_check" + ), Index("idx_artifacts_user_id", "user_id"), Index("idx_artifacts_chat_id", "chat_id"), Index("idx_artifacts_type", "type"), Index("idx_artifacts_created_at", "created_at"), Index("idx_artifacts_user_created", "user_id", "created_at"), - Index("idx_artifacts_deleted", "deleted_at", postgresql_where=Column("deleted_at").isnot(None)), - Index("idx_artifacts_team_folder", "team_id", "team_folder_id", "created_at"), - Index("idx_artifacts_team_folder_id", "team_folder_id"), + Index( + "idx_artifacts_deleted", "deleted_at", postgresql_where=Column("deleted_at").isnot(None) + ), Index("idx_artifacts_user_folder", "user_id", "user_folder_id", "created_at"), Index("idx_artifacts_user_folder_id", "user_folder_id"), ) @@ -72,9 +83,10 @@ class Artifact(Base): class ContentBlock(Base): """Content blocks for editable frontend sections (feature updates / capability center).""" + __tablename__ = "content_blocks" - id = Column(String(64), primary_key=True) # e.g. 'docs_updates', 'docs_capabilities' + id = Column(String(64), primary_key=True) # e.g. 'docs_updates', 'docs_capabilities' payload = Column(JSONType, nullable=False, default=[]) updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) updated_by = Column(String(64), nullable=True) diff --git a/src/backend/core/db/models/automation.py b/src/backend/core/db/models/automation.py index f525e0cd..e1585a8a 100644 --- a/src/backend/core/db/models/automation.py +++ b/src/backend/core/db/models/automation.py @@ -1,13 +1,25 @@ """SQLAlchemy ORM models — automation / batch / distillation.""" from datetime import datetime, timezone + +from core.db.engine import Base from sqlalchemy import ( - Column, String, Integer, BigInteger, Boolean, Text, TIMESTAMP, - ForeignKey, CheckConstraint, UniqueConstraint, Index, Numeric, JSON + JSON, + TIMESTAMP, + BigInteger, + Boolean, + CheckConstraint, + Column, + ForeignKey, + Index, + Integer, + Numeric, + String, + Text, + UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import JSONB, INET -from sqlalchemy.orm import relationship, mapped_column -from core.db.engine import Base +from sqlalchemy.dialects.postgresql import INET, JSONB +from sqlalchemy.orm import mapped_column, relationship JSONType = JSON().with_variant(JSONB(), "postgresql") INETType = String(45).with_variant(INET(), "postgresql") @@ -15,57 +27,67 @@ class ScheduledTask(Base): """Automation — scheduled tasks table""" + __tablename__ = "scheduled_tasks" - task_id = Column(String(64), primary_key=True) - user_id = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False) + task_id = Column(String(64), primary_key=True) + user_id = Column( + String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False + ) # Task content — either prompt or plan, one of the two - task_type = Column(String(20), nullable=False) # "prompt" | "plan" - prompt = Column(Text) - plan_id = Column(String(64), ForeignKey("plans.plan_id", ondelete="SET NULL")) + task_type = Column(String(20), nullable=False) # "prompt" | "plan" + prompt = Column(Text) + plan_id = Column(String(64), ForeignKey("plans.plan_id", ondelete="SET NULL")) # Scheduling config - cron_expression = Column(String(100), nullable=False) - recurring = Column(Boolean, nullable=False, default=True) - timezone = Column(String(50), nullable=False, default="Asia/Shanghai") - schedule_type = Column(String(20), nullable=False, default="recurring") # "recurring" | "once" | "manual" + cron_expression = Column(String(100), nullable=False) + recurring = Column(Boolean, nullable=False, default=True) + timezone = Column(String(50), nullable=False, default="Asia/Shanghai") + schedule_type = Column( + String(20), nullable=False, default="recurring" + ) # "recurring" | "once" | "manual" # Execution capability config - enabled_mcp_ids = Column(JSONType, default=list) + enabled_mcp_ids = Column(JSONType, default=list) enabled_skill_ids = Column(JSONType, default=list) - enabled_kb_ids = Column(JSONType, default=list) + enabled_kb_ids = Column(JSONType, default=list) enabled_agent_ids = Column(JSONType, default=list) # Status - status = Column(String(20), nullable=False, default="active") - next_run_at = Column(TIMESTAMP(timezone=True)) - last_run_at = Column(TIMESTAMP(timezone=True)) - run_count = Column(Integer, default=0) - max_runs = Column(Integer) + status = Column(String(20), nullable=False, default="active") + next_run_at = Column(TIMESTAMP(timezone=True)) + last_run_at = Column(TIMESTAMP(timezone=True)) + run_count = Column(Integer, default=0) + max_runs = Column(Integer) # Failure tracking consecutive_failures = Column(Integer, default=0) - max_failures = Column(Integer, default=3) - last_error = Column(Text) + max_failures = Column(Integer, default=3) + last_error = Column(Text) # Metadata - name = Column(String(200)) - description = Column(Text, default="") - extra_data = Column("metadata", JSONType, default={}) + name = Column(String(200)) + description = Column(Text, default="") + extra_data = Column("metadata", JSONType, default={}) sidebar_activated = Column(Boolean, default=False, nullable=False) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) + updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) # Relationships - user = relationship("UserShadow") - plan = relationship("Plan") - run_history = relationship("ScheduledTaskRun", back_populates="task", - cascade="all, delete-orphan", - order_by="ScheduledTaskRun.started_at.desc()") + user = relationship("UserShadow") + plan = relationship("Plan") + run_history = relationship( + "ScheduledTaskRun", + back_populates="task", + cascade="all, delete-orphan", + order_by="ScheduledTaskRun.started_at.desc()", + ) __table_args__ = ( - CheckConstraint("task_type IN ('prompt', 'plan', 'loop')", name="scheduled_tasks_type_check"), + CheckConstraint( + "task_type IN ('prompt', 'plan', 'loop')", name="scheduled_tasks_type_check" + ), CheckConstraint( "status IN ('active', 'paused', 'disabled', 'completed', 'expired')", name="scheduled_tasks_status_check", @@ -82,18 +104,21 @@ class ScheduledTask(Base): class ScheduledTaskRun(Base): """Automation — execution records table""" + __tablename__ = "scheduled_task_runs" - run_id = Column(String(64), primary_key=True) - task_id = Column(String(64), ForeignKey("scheduled_tasks.task_id", ondelete="CASCADE"), nullable=False) - status = Column(String(20), nullable=False, default="running") - chat_id = Column(String(64)) - result_summary = Column(Text) - error_message = Column(Text) - started_at = Column(TIMESTAMP(timezone=True), default=lambda: datetime.now(timezone.utc)) - completed_at = Column(TIMESTAMP(timezone=True)) - duration_ms = Column(Integer) - usage = Column(JSONType, default={}) + run_id = Column(String(64), primary_key=True) + task_id = Column( + String(64), ForeignKey("scheduled_tasks.task_id", ondelete="CASCADE"), nullable=False + ) + status = Column(String(20), nullable=False, default="running") + chat_id = Column(String(64)) + result_summary = Column(Text) + error_message = Column(Text) + started_at = Column(TIMESTAMP(timezone=True), default=lambda: datetime.now(timezone.utc)) + completed_at = Column(TIMESTAMP(timezone=True)) + duration_ms = Column(Integer) + usage = Column(JSONType, default={}) task = relationship("ScheduledTask", back_populates="run_history") @@ -107,37 +132,6 @@ class ScheduledTaskRun(Base): ) -class DistillationRun(Base): - """Distillation task queue + audit. At most one row per chat_id (UNIQUE).""" - __tablename__ = "distillation_runs" - - run_id = Column(String(64), primary_key=True) - chat_id = Column(String(64), nullable=False) - trigger = Column(String(16), nullable=False) # daily_cron | manual - status = Column(String(32), nullable=False, default="queued") - skip_reason = Column(String(64)) - draft_id = Column(String(64)) - cost_usd = Column(Numeric(8, 4), default=0) - error_message = Column(Text) - started_at = Column(TIMESTAMP(timezone=True)) - finished_at = Column(TIMESTAMP(timezone=True)) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - - __table_args__ = ( - UniqueConstraint("chat_id", name="distillation_runs_chat_id_unique"), - CheckConstraint( - "trigger IN ('daily_cron', 'manual')", - name="distillation_runs_trigger_check", - ), - CheckConstraint( - "status IN ('queued','running','skipped','completed_distilled','failed_parse','failed_budget','failed_model')", - name="distillation_runs_status_check", - ), - Index("idx_distillation_runs_status", "status", "created_at"), - Index("idx_distillation_runs_created_at", "created_at"), - ) - - class PersonaDistillJob(Base): """Persona-level distillation job (colleague skill / personal skill). @@ -148,26 +142,29 @@ class PersonaDistillJob(Base): review; in personal mode it lands directly as AdminSkill(owner=the user) after the user's own confirmation. """ + __tablename__ = "persona_distill_jobs" - job_id = Column(String(64), primary_key=True) # pdj_<16hex> - kind = Column(String(16), nullable=False) # colleague | personal - target_user_id = Column(String(64), nullable=False) # the user being distilled - requested_by = Column(String(64), nullable=False) # initiator (config_admin or user_id) - scope = Column(JSONType, default=dict) # chat_ids / date range / memory switch / hint - status = Column(String(16), nullable=False, default="queued") - progress_done = Column(Integer, nullable=False, default=0) # conversations completed in the map stage - progress_total = Column(Integer, nullable=False, default=0) - intermediate = Column(JSONType, default=list) # list of conversation summaries (map output) - result_skill_content = Column(Text) # full SKILL.md text (reduce output) - result_meta = Column(JSONType, default=dict) # proposed_skill_id/display_name/... - result_draft_id = Column(String(64)) # mirrored draft in colleague mode - saved_skill_id = Column(String(100)) # AdminSkill.skill_id after confirmed persistence - cost_usd = Column(Numeric(8, 4), default=0) - error = Column(Text) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - started_at = Column(TIMESTAMP(timezone=True)) - finished_at = Column(TIMESTAMP(timezone=True)) + job_id = Column(String(64), primary_key=True) # pdj_<16hex> + kind = Column(String(16), nullable=False) # colleague | personal + target_user_id = Column(String(64), nullable=False) # the user being distilled + requested_by = Column(String(64), nullable=False) # initiator (config_admin or user_id) + scope = Column(JSONType, default=dict) # chat_ids / date range / memory switch / hint + status = Column(String(16), nullable=False, default="queued") + progress_done = Column( + Integer, nullable=False, default=0 + ) # conversations completed in the map stage + progress_total = Column(Integer, nullable=False, default=0) + intermediate = Column(JSONType, default=list) # list of conversation summaries (map output) + result_skill_content = Column(Text) # full SKILL.md text (reduce output) + result_meta = Column(JSONType, default=dict) # proposed_skill_id/display_name/... + result_draft_id = Column(String(64)) # mirrored draft in colleague mode + saved_skill_id = Column(String(100)) # AdminSkill.skill_id after confirmed persistence + cost_usd = Column(Numeric(8, 4), default=0) + error = Column(Text) + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) + started_at = Column(TIMESTAMP(timezone=True)) + finished_at = Column(TIMESTAMP(timezone=True)) __table_args__ = ( CheckConstraint( @@ -188,22 +185,23 @@ class PersonaDistillJob(Base): class BatchPlan(Base): """Batch execution plan (generated by the batch_plan MCP tool; executed by BatchOrchestrator after user confirmation).""" + __tablename__ = "batch_plans" - plan_id = Column(String(64), primary_key=True) - user_id = Column(String(64), nullable=False) - chat_id = Column(String(64)) # optional associated conversation - source_type = Column(String(20), nullable=False) # xlsx | word_files | text_list - items = Column(JSONType, nullable=False, default=list) # list[dict] - placeholder_keys = Column(JSONType, default=list) # placeholders available to the template - instruction = Column(Text) # the user's original batch goal - prompt_template = Column(Text, nullable=False) - max_retries = Column(Integer, nullable=False, default=2) - status = Column(String(20), nullable=False, default="pending") - progress = Column(JSONType, default=dict) # {done, success, failed} - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - expires_at = Column(TIMESTAMP(timezone=True)) # +24h, cleaned up periodically + plan_id = Column(String(64), primary_key=True) + user_id = Column(String(64), nullable=False) + chat_id = Column(String(64)) # optional associated conversation + source_type = Column(String(20), nullable=False) # xlsx | word_files | text_list + items = Column(JSONType, nullable=False, default=list) # list[dict] + placeholder_keys = Column(JSONType, default=list) # placeholders available to the template + instruction = Column(Text) # the user's original batch goal + prompt_template = Column(Text, nullable=False) + max_retries = Column(Integer, nullable=False, default=2) + status = Column(String(20), nullable=False, default="pending") + progress = Column(JSONType, default=dict) # {done, success, failed} + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) + updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) + expires_at = Column(TIMESTAMP(timezone=True)) # +24h, cleaned up periodically __table_args__ = ( CheckConstraint( diff --git a/src/backend/core/db/models/chat.py b/src/backend/core/db/models/chat.py index f90502d7..8e63dcad 100644 --- a/src/backend/core/db/models/chat.py +++ b/src/backend/core/db/models/chat.py @@ -1,13 +1,26 @@ """SQLAlchemy ORM models — chat sessions/messages.""" from datetime import datetime, timezone + +from core.db.engine import Base +from core.db.model_extensions import ChatSessionEditionFields, chat_session_edition_table_args from sqlalchemy import ( - Column, String, Integer, BigInteger, Boolean, Text, TIMESTAMP, - ForeignKey, CheckConstraint, UniqueConstraint, Index, Numeric, JSON + JSON, + TIMESTAMP, + BigInteger, + Boolean, + CheckConstraint, + Column, + ForeignKey, + Index, + Integer, + Numeric, + String, + Text, + UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import JSONB, INET -from sqlalchemy.orm import relationship, mapped_column -from core.db.engine import Base +from sqlalchemy.dialects.postgresql import INET, JSONB +from sqlalchemy.orm import mapped_column, relationship JSONType = JSON().with_variant(JSONB(), "postgresql") INETType = String(45).with_variant(INET(), "postgresql") @@ -17,12 +30,15 @@ BigIntPK = BigInteger().with_variant(Integer(), "sqlite") -class ChatSession(Base): +class ChatSession(ChatSessionEditionFields, Base): """Chat session table.""" + __tablename__ = "chat_sessions" chat_id = Column(String(64), primary_key=True) - user_id = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False) + user_id = Column( + String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False + ) title = Column(String(500), nullable=False, default="新对话") message_count = Column(Integer, default=0) pinned = Column(Boolean, default=False) @@ -31,16 +47,8 @@ class ChatSession(Base): deleted_at = Column(TIMESTAMP(timezone=True)) extra_data = Column("metadata", JSONType, default={}) # Project mode: the chat is mounted on a specific project (NULL = ordinary chat) - project_id = Column(String(64), ForeignKey("projects.project_id", ondelete="SET NULL"), nullable=True) - # Sharing scope of a chat within a team project (only effective when - # project.kind='team'; personal-project / non-project chats are always treated - # as private): - # 'private' — visible only to the owner (default) - # 'team_read' — project members can read, but cannot send messages / rename / delete - # 'team_edit' — project members can keep sending messages and renaming; delete - # is still owner / project admin only - share_scope = Column( - String(16), nullable=False, default="private", server_default="private" + project_id = Column( + String(64), ForeignKey("projects.project_id", ondelete="SET NULL"), nullable=True ) # Inbound channel-bot origin (NULL = ordinary web chat). Channel messages upsert # by (channel_id, external_conversation_id) to reuse the same session, running @@ -60,77 +68,52 @@ class ChatSession(Base): artifacts = relationship("Artifact", back_populates="session") __table_args__ = ( + *chat_session_edition_table_args(), CheckConstraint("length(title) >= 1", name="chat_sessions_title_length"), CheckConstraint("message_count >= 0", name="chat_sessions_message_count_check"), - CheckConstraint( - "share_scope IN ('private','team_read','team_edit')", - name="ck_chat_sessions_share_scope", - ), Index("idx_chat_sessions_user_id", "user_id"), Index("idx_chat_sessions_updated_at", "updated_at"), Index("idx_chat_sessions_user_updated", "user_id", "updated_at"), - Index("idx_chat_sessions_pinned", "user_id", "pinned", "updated_at", postgresql_where=Column("pinned") == True), - Index("idx_chat_sessions_favorite", "user_id", "favorite", "updated_at", postgresql_where=Column("favorite") == True), - Index("idx_chat_sessions_deleted", "deleted_at", postgresql_where=Column("deleted_at").isnot(None)), - Index("idx_chat_sessions_metadata_gin", "metadata", postgresql_using="gin"), - Index("idx_chat_sessions_last_message_at", "last_message_at"), - # Inbound channel message → locate/reuse the session: (channel_id, external_conversation_id) Index( - "idx_chat_sessions_channel_conv", - "channel_id", "external_conversation_id", - postgresql_where=Column("channel_id").isnot(None), + "idx_chat_sessions_pinned", + "user_id", + "pinned", + "updated_at", + postgresql_where=Column("pinned") == True, ), - # Team-project shared chat listing query: composite index on (project_id, share_scope) Index( - "idx_chat_sessions_project_share", - "project_id", "share_scope", - postgresql_where=Column("project_id").isnot(None), + "idx_chat_sessions_favorite", + "user_id", + "favorite", + "updated_at", + postgresql_where=Column("favorite") == True, ), - ) - - -class ChatSessionUserState(Base): - """Per-user state of a chat × user pair (pin / favorite). - - Purpose: once a chat is shared within a team project, each member's pin / - favorite are independent of each other. - - Team-project chats: reads and writes always go through this table (the owner - does too). - - Non-project chats / personal-project chats: keep reading/writing the legacy - ``ChatSession.pinned/favorite`` fields; this table is unused, leaving - existing behavior untouched. - """ - __tablename__ = "chat_session_user_states" - - chat_id = Column(String(64), ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"), primary_key=True) - user_id = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), primary_key=True) - pinned = Column(Boolean, nullable=False, default=False, server_default="false") - favorite = Column(Boolean, nullable=False, default=False, server_default="false") - last_seen_at = Column(TIMESTAMP(timezone=True)) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - - __table_args__ = ( - Index("idx_csus_user", "user_id"), Index( - "idx_csus_user_pinned", - "user_id", "pinned", - postgresql_where=Column("pinned") == True, + "idx_chat_sessions_deleted", + "deleted_at", + postgresql_where=Column("deleted_at").isnot(None), ), + Index("idx_chat_sessions_metadata_gin", "metadata", postgresql_using="gin"), + Index("idx_chat_sessions_last_message_at", "last_message_at"), + # Inbound channel message → locate/reuse the session: (channel_id, external_conversation_id) Index( - "idx_csus_user_favorite", - "user_id", "favorite", - postgresql_where=Column("favorite") == True, + "idx_chat_sessions_channel_conv", + "channel_id", + "external_conversation_id", + postgresql_where=Column("channel_id").isnot(None), ), ) class ChatMessage(Base): """Chat message table.""" + __tablename__ = "chat_messages" message_id = Column(String(64), primary_key=True) - chat_id = Column(String(64), ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"), nullable=False) + chat_id = Column( + String(64), ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"), nullable=False + ) role = Column(String(20), nullable=False) content = Column(Text, nullable=False) model = Column(String(100)) @@ -144,13 +127,20 @@ class ChatMessage(Base): session = relationship("ChatSession", back_populates="messages") __table_args__ = ( - CheckConstraint("role IN ('user', 'assistant', 'system', 'tool')", name="chat_messages_role_check"), + CheckConstraint( + "role IN ('user', 'assistant', 'system', 'tool')", name="chat_messages_role_check" + ), CheckConstraint("length(content) <= 100000", name="chat_messages_content_length"), Index("idx_chat_messages_chat_id", "chat_id"), Index("idx_chat_messages_chat_created", "chat_id", "created_at"), Index("idx_chat_messages_role", "chat_id", "role"), Index("idx_chat_messages_created_at", "created_at"), - Index("idx_chat_messages_tool_calls_gin", "tool_calls", postgresql_using="gin", postgresql_where=Column("tool_calls").isnot(None)), + Index( + "idx_chat_messages_tool_calls_gin", + "tool_calls", + postgresql_using="gin", + postgresql_where=Column("tool_calls").isnot(None), + ), ) @@ -161,20 +151,27 @@ class ChatRun(Base): chunks are written to a Redis Stream, and the SSE endpoint pulls from the Stream. After a page refresh, playback resumes via follow_run + offset. """ + __tablename__ = "chat_runs" - run_id = Column(String(64), primary_key=True) - chat_id = Column(String(64), ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"), nullable=False) - user_id = Column(String(64), nullable=False) - message_id = Column(String(64), nullable=False) # pre-allocated assistant message id - status = Column(String(20), nullable=False, default="pending") - request_payload = Column(JSONType) # serialized ChatRequest (for the worker to rebuild the context) + run_id = Column(String(64), primary_key=True) + chat_id = Column( + String(64), ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"), nullable=False + ) + user_id = Column(String(64), nullable=False) + message_id = Column(String(64), nullable=False) # pre-allocated assistant message id + status = Column(String(20), nullable=False, default="pending") + request_payload = Column( + JSONType + ) # serialized ChatRequest (for the worker to rebuild the context) last_event_offset = Column(Integer, default=0, nullable=False) - error_message = Column(Text) - usage = Column(JSONType) - created_at = Column(TIMESTAMP(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False) - started_at = Column(TIMESTAMP(timezone=True)) - completed_at = Column(TIMESTAMP(timezone=True)) + error_message = Column(Text) + usage = Column(JSONType) + created_at = Column( + TIMESTAMP(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False + ) + started_at = Column(TIMESTAMP(timezone=True)) + completed_at = Column(TIMESTAMP(timezone=True)) __table_args__ = ( CheckConstraint( @@ -189,13 +186,20 @@ class ChatRun(Base): class MessageFeedback(Base): """Message feedback table - stores like/dislike ratings and optional comments.""" + __tablename__ = "message_feedback" feedback_id = Column(BigIntPK, primary_key=True, autoincrement=True) - message_id = Column(String(64), ForeignKey("chat_messages.message_id", ondelete="CASCADE"), nullable=False) - chat_id = Column(String(64), ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"), nullable=False) - user_id = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="SET NULL"), nullable=True) - rating = Column(String(10), nullable=False) # 'like' or 'dislike' + message_id = Column( + String(64), ForeignKey("chat_messages.message_id", ondelete="CASCADE"), nullable=False + ) + chat_id = Column( + String(64), ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"), nullable=False + ) + user_id = Column( + String(64), ForeignKey("users_shadow.user_id", ondelete="SET NULL"), nullable=True + ) + rating = Column(String(10), nullable=False) # 'like' or 'dislike' comment = Column(Text, nullable=True) created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) @@ -218,15 +222,22 @@ class ChatSandboxSnapshot(Base): expires_at < now and deletes remote + DB together. Design in internal design docs. """ + __tablename__ = "chat_sandbox_snapshots" - chat_id = Column(String(64), primary_key=True) + chat_id = Column(String(64), primary_key=True) snapshot_id = Column(String(64), nullable=False, unique=True) - sandbox_id = Column(String(64), nullable=False) # source sandbox id parked at the time, for debugging / reconciliation - created_at = Column(TIMESTAMP(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) - expires_at = Column(TIMESTAMP(timezone=True), nullable=False) # created_at + SNAPSHOT_RETENTION_DAYS - size_bytes = Column(BigInteger) # for metrics, nullable - extra = Column("metadata", JSONType, default=dict) # reserved: image uri / pool kind / notes + sandbox_id = Column( + String(64), nullable=False + ) # source sandbox id parked at the time, for debugging / reconciliation + created_at = Column( + TIMESTAMP(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) + ) + expires_at = Column( + TIMESTAMP(timezone=True), nullable=False + ) # created_at + SNAPSHOT_RETENTION_DAYS + size_bytes = Column(BigInteger) # for metrics, nullable + extra = Column("metadata", JSONType, default=dict) # reserved: image uri / pool kind / notes __table_args__ = ( Index("idx_chat_sandbox_snapshots_expires", "expires_at"), diff --git a/src/backend/core/db/models/config.py b/src/backend/core/db/models/config.py index 895051a3..41d5f333 100644 --- a/src/backend/core/db/models/config.py +++ b/src/backend/core/db/models/config.py @@ -1,13 +1,25 @@ """SQLAlchemy ORM models — model / system configuration.""" from datetime import datetime, timezone + +from core.db.engine import Base from sqlalchemy import ( - Column, String, Integer, BigInteger, Boolean, Text, TIMESTAMP, - ForeignKey, CheckConstraint, UniqueConstraint, Index, Numeric, JSON + JSON, + TIMESTAMP, + BigInteger, + Boolean, + CheckConstraint, + Column, + ForeignKey, + Index, + Integer, + Numeric, + String, + Text, + UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import JSONB, INET -from sqlalchemy.orm import relationship, mapped_column -from core.db.engine import Base +from sqlalchemy.dialects.postgresql import INET, JSONB +from sqlalchemy.orm import mapped_column, relationship JSONType = JSON().with_variant(JSONB(), "postgresql") INETType = String(45).with_variant(INET(), "postgresql") @@ -15,6 +27,7 @@ class ModelProvider(Base): """Model provider — an OpenAI-compatible model endpoint.""" + __tablename__ = "model_providers" provider_id = Column(String(64), primary_key=True) @@ -43,7 +56,9 @@ class ModelProvider(Base): updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) # Relationships - role_assignments = relationship("ModelRoleAssignment", back_populates="provider", cascade="all, delete-orphan") + role_assignments = relationship( + "ModelRoleAssignment", back_populates="provider", cascade="all, delete-orphan" + ) __table_args__ = ( CheckConstraint( @@ -63,6 +78,7 @@ class ModelProvider(Base): class SystemConfig(Base): """Key-value store for external service configurations (DB query, KB, industry, file parser).""" + __tablename__ = "system_configs" config_key = Column(String(100), primary_key=True) @@ -74,13 +90,12 @@ class SystemConfig(Base): updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) updated_by = Column(String(64)) - __table_args__ = ( - Index("idx_system_configs_group_key", "group_key"), - ) + __table_args__ = (Index("idx_system_configs_group_key", "group_key"),) class ModelRoleAssignment(Base): """Role → provider mapping. Each role_key can have at most one provider.""" + __tablename__ = "model_role_assignments" role_key = Column(String(50), primary_key=True) @@ -95,71 +110,4 @@ class ModelRoleAssignment(Base): # Relationships provider = relationship("ModelProvider", back_populates="role_assignments") - __table_args__ = ( - Index("idx_model_role_assignments_provider", "provider_id"), - ) - - -class ModelPricing(Base): - """Model pricing configuration for token billing.""" - __tablename__ = "model_pricing" - - pricing_id = Column(String(64), primary_key=True) - model_name = Column(String(255), nullable=False, unique=True) - display_name = Column(String(255)) - input_price = Column(Numeric(12, 6), nullable=False, default=0) - output_price = Column(Numeric(12, 6), nullable=False, default=0) - currency = Column(String(10), nullable=False, default="CNY") - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - - __table_args__ = ( - Index("idx_model_pricing_model_name", "model_name"), - Index("idx_model_pricing_active", "is_active"), - ) - - -class GatewayVirtualKey(Base): - """**Mirror** of the outbound model gateway's virtual keys (EE-only). - - The source of truth is LiteLLM Proxy's own DB (the key secret + budget/rate-limit - enforcement all live there); this table stores displayable/auditable metadata: - - ``litellm_token``: the hashed token returned by LiteLLM ``/key/generate``, used as the - management handle for subsequent block/unblock/delete/update (not plaintext, safe to persist). - - ``litellm_key_enc``: the **encrypted** ciphertext (Fernet) of the plaintext ``sk-...``, for - admins to copy the plaintext afterwards; decryption only via the ADMIN-authenticated reveal - endpoint (``core/infra/crypto.py``). The LiteLLM side stores only the hash and cannot recover - the plaintext, so this table carries the encryption. This column is empty for pre-existing legacy keys. - When the license lapses (DEAD_MODES), the control plane batch-calls LiteLLM ``/key/block`` per - this table to ban keys, unbanning on renewal — see ``core/services/litellm_gateway_service.py``. - """ - __tablename__ = "gateway_virtual_keys" - - key_id = Column(String(64), primary_key=True) - key_alias = Column(String(255), nullable=False, unique=True) - litellm_token = Column(Text) # hashed token management handle; may be empty on issuance failure / for historical data - # **Encrypted** ciphertext (Fernet, core/infra/crypto.py) of the plaintext sk- key. Stored only to let admins copy the plaintext afterwards; - # written on issuance, decrypted and returned on reveal. Empty for pre-existing legacy keys (no plaintext to copy, can only re-issue). - litellm_key_enc = Column(Text) - display_name = Column(String(255), nullable=False) - owner = Column(String(255)) # issuance target (third-party / external system name), free text - allowed_models = Column("allowed_models", JSONType, default=list) # model alias whitelist - max_budget = Column(Numeric(12, 4)) # budget cap (currency unit), NULL = unlimited - tpm_limit = Column(Integer) # tokens/min, NULL = unlimited - rpm_limit = Column(Integer) # requests/min, NULL = unlimited - # active / blocked / license_blocked (auto-banned on license lapse, distinct from manual blocked) - status = Column(String(20), nullable=False, default="active") - created_by = Column(String(64)) - deleted_at = Column(TIMESTAMP(timezone=True)) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - - __table_args__ = ( - CheckConstraint( - "status IN ('active', 'blocked', 'license_blocked')", - name="gateway_virtual_keys_status_check", - ), - Index("idx_gateway_virtual_keys_status", "status"), - Index("idx_gateway_virtual_keys_alias", "key_alias"), - ) + __table_args__ = (Index("idx_model_role_assignments_provider", "provider_id"),) diff --git a/src/backend/core/db/models/datasource.py b/src/backend/core/db/models/datasource.py deleted file mode 100644 index 2fd5b7bc..00000000 --- a/src/backend/core/db/models/datasource.py +++ /dev/null @@ -1,173 +0,0 @@ -"""SQLAlchemy ORM model — data sources (unified "database tools" configuration). - -One DataSource = one database connection managed by the "database tools" feature. -ds_type decides which MCP tool is exposed to the agent: - - - ``external_nl2sql`` → exposes the ``query_database`` tool (external NL2SQL HTTP service). - - other SQL types (mysql/postgresql/sqlserver/mariadb/sqlite) → expose the - ``execute_sql`` / ``search_objects`` tools via the DBHub sidecar. - -DSN/passwords are stored in plaintext (consistent with the repo's existing -system_configs / admin_mcp_servers); API responses are masked. -""" - -from datetime import datetime - -from sqlalchemy import ( - JSON, Boolean, CheckConstraint, Column, ForeignKey, Index, Integer, String, - TIMESTAMP, Text, UniqueConstraint, -) -from sqlalchemy.dialects.postgresql import JSONB - -from core.db.engine import Base - -JSONType = JSON().with_variant(JSONB(), "postgresql") - -# Supported data source types: -# external_nl2sql → the legacy query_database tool; elasticsearch → the es_query tool; other SQL → db_query (DBHub). -DS_TYPES = ( - "external_nl2sql", "mysql", "postgresql", "sqlserver", "mariadb", "sqlite", "elasticsearch", -) - - -class DataSource(Base): - """Data source configuration row for the "database tools" feature. - - Connection params prefer the structured fields (host/port/username/password/database), - from which the backend builds the DSN / ES_URL; ``dsn`` is only an advanced - "raw connection string override". external_nl2sql uses ``url``. - """ - - __tablename__ = "data_sources" - - id = Column(String(64), primary_key=True) # slug, also used as the dbhub source id - name = Column(String(255), nullable=False) - ds_type = Column(String(32), nullable=False, default="mysql") - # Structured connection params (stored plaintext; API masks password) - host = Column(String(255)) - port = Column(Integer) - username = Column(String(255)) - password = Column(Text) # plaintext, masked in the API - database = Column(String(255)) # database name / sqlite file path / ES index pattern - dsn = Column(Text) # advanced: raw DSN override (takes precedence when filled) - url = Column(Text) # external_nl2sql service address - description = Column(Text) - readonly = Column(Boolean, nullable=False, default=True) - is_enabled = Column(Boolean, nullable=False, default=True) - sort_order = Column(Integer, nullable=False, default=0) - extra = Column(JSONType, default=dict) # search_path / api_key / ssl_skip_verify / custom keys, etc. - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - - __table_args__ = ( - CheckConstraint( - "ds_type IN ('external_nl2sql','mysql','postgresql','sqlserver','mariadb','sqlite','elasticsearch')", - name="data_sources_ds_type_check", - ), - Index("idx_data_sources_enabled", "is_enabled"), - Index("idx_data_sources_sort", "sort_order"), - ) - - -# ── Metadata governance (improves direct-connection retrieval accuracy) ──────── -# -# A direct-connection data source (non external_nl2sql) can carry a set of metadata: -# table/column business semantics + enum dictionaries + golden Q→SQL exemplars. At -# retrieval time the built-in tool ``get_data_context`` recalls them on demand and -# feeds them to the model (**not into the system prompt**). The external NL2SQL black -# box (external_nl2sql) does text2sql internally and cannot be fed this metadata, so -# it is not annotated in this domain. See -# internal design docs. - - -class DsTableMeta(Base): - """Table-level metadata: business name, description, synonyms, whitelist / deprecation flags.""" - - __tablename__ = "ds_table_meta" - - id = Column(Integer, primary_key=True, autoincrement=True) - datasource_id = Column(String(64), ForeignKey("data_sources.id", ondelete="CASCADE"), nullable=False) - schema_name = Column(String(128), nullable=False, default="") # database/schema (PG multi-schema / ES) - table_name = Column(String(255), nullable=False) - display_name = Column(String(255)) # business name, e.g. "订单主表" (order master table) - description = Column(Text) # business-definition notes - synonyms = Column(JSONType, default=list) # synonyms/aliases (matching user phrasing) - lifecycle = Column(String(16)) # None / 'certified' / 'deprecated' (a "don't use" signal) - is_whitelisted = Column(Boolean, nullable=False, default=True) # whether it enters the data dictionary - row_estimate = Column(Integer) # probed magnitude (aids judgment) - sort_order = Column(Integer, nullable=False, default=0) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - - __table_args__ = ( - UniqueConstraint("datasource_id", "schema_name", "table_name", name="uq_ds_table_meta"), - Index("idx_ds_table_meta_ds", "datasource_id"), - ) - - -class DsColumnMeta(Base): - """Column-level metadata: business name, description, enum dictionary (status=1→approved), semantic role, foreign keys, etc. - - The two highest-ROI fields are ``value_map`` (the enum dictionary, directly attacking - "values are codes with no dictionary") and golden SQL (separate table). ``foreign_key`` - holds ``other_table.other_col``, supplementing relationship declarations missing in the - external database so the model doesn't write JOINs wrong or omit them. - """ - - __tablename__ = "ds_column_meta" - - id = Column(Integer, primary_key=True, autoincrement=True) - datasource_id = Column(String(64), ForeignKey("data_sources.id", ondelete="CASCADE"), nullable=False) - schema_name = Column(String(128), nullable=False, default="") - table_name = Column(String(255), nullable=False) - column_name = Column(String(255), nullable=False) - display_name = Column(String(255)) # business name, e.g. "订单状态" (order status) - description = Column(Text) - synonyms = Column(JSONType, default=list) - data_type = Column(String(64)) # probed physical type - semantic_role = Column(String(16)) # 'dimension' / 'measure' / 'time' - value_map = Column(JSONType, default=dict) # enum dictionary, e.g. {"1":"已审核","2":"已驳回"} - sample_values = Column(JSONType, default=list) # sample values (backfilled from probe sampling) - unit_format = Column(JSONType, default=dict) # {type: currency, currency_code: CNY}, etc. - is_pii = Column(Boolean, nullable=False, default=False) - lifecycle = Column(String(16)) # None / 'certified' / 'deprecated' - is_primary_key = Column(Boolean, nullable=False, default=False) - foreign_key = Column(String(512)) # 'other_table.other_col' - sort_order = Column(Integer, nullable=False, default=0) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - - __table_args__ = ( - UniqueConstraint("datasource_id", "schema_name", "table_name", "column_name", - name="uq_ds_column_meta"), - Index("idx_ds_column_meta_ds", "datasource_id"), - Index("idx_ds_column_meta_tbl", "datasource_id", "schema_name", "table_name"), - ) - - -class DsGoldenSql(Base): - """Golden Q→SQL pairs: human-verified correct "question → SQL" exemplars (the #1 accuracy lever). - - At retrieval time they are recalled by similarity (embedding from Phase 2; in - Phase 1 the small corpus is simply loaded in full) and injected into the model as - few-shot examples, turning business-side tribal knowledge into a governed asset. - """ - - __tablename__ = "ds_golden_sql" - - id = Column(Integer, primary_key=True, autoincrement=True) - datasource_id = Column(String(64), ForeignKey("data_sources.id", ondelete="CASCADE"), nullable=False) - question = Column(Text, nullable=False) - sql = Column(Text, nullable=False) - tables_used = Column(JSONType, default=list) # tables involved (eases per-table trimming/aggregation) - status = Column(String(16), nullable=False, default="candidate") # 'verified' / 'candidate' - hit_count = Column(Integer, nullable=False, default=0) - verified_by = Column(String(64)) - verified_at = Column(TIMESTAMP(timezone=True)) - sort_order = Column(Integer, nullable=False, default=0) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - - __table_args__ = ( - Index("idx_ds_golden_sql_ds", "datasource_id"), - ) diff --git a/src/backend/core/db/models/identity.py b/src/backend/core/db/models/identity.py index d5daabc8..70099c77 100644 --- a/src/backend/core/db/models/identity.py +++ b/src/backend/core/db/models/identity.py @@ -1,13 +1,25 @@ """SQLAlchemy ORM models — identity / teams.""" from datetime import datetime, timezone + +from core.db.engine import Base from sqlalchemy import ( - Column, String, Integer, BigInteger, Boolean, Text, TIMESTAMP, - ForeignKey, CheckConstraint, UniqueConstraint, Index, Numeric, JSON + JSON, + TIMESTAMP, + BigInteger, + Boolean, + CheckConstraint, + Column, + ForeignKey, + Index, + Integer, + Numeric, + String, + Text, + UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import JSONB, INET -from sqlalchemy.orm import relationship, mapped_column -from core.db.engine import Base +from sqlalchemy.dialects.postgresql import INET, JSONB +from sqlalchemy.orm import mapped_column, relationship JSONType = JSON().with_variant(JSONB(), "postgresql") INETType = String(45).with_variant(INET(), "postgresql") @@ -15,6 +27,7 @@ class UserShadow(Base): """User shadow table - synced from user center.""" + __tablename__ = "users_shadow" user_id = Column(String(64), primary_key=True) @@ -29,10 +42,14 @@ class UserShadow(Base): # Relationships chat_sessions = relationship("ChatSession", back_populates="user", cascade="all, delete-orphan") - catalog_overrides = relationship("CatalogOverride", back_populates="user", cascade="all, delete-orphan") + catalog_overrides = relationship( + "CatalogOverride", back_populates="user", cascade="all, delete-orphan" + ) kb_spaces = relationship("KBSpace", back_populates="user", cascade="all, delete-orphan") artifacts = relationship("Artifact", back_populates="user", cascade="all, delete-orphan") - user_agents = relationship("UserAgent", foreign_keys="[UserAgent.user_id]", back_populates="user") + user_agents = relationship( + "UserAgent", foreign_keys="[UserAgent.user_id]", back_populates="user" + ) __table_args__ = ( Index("idx_users_shadow_user_center_id", "user_center_id"), @@ -42,9 +59,12 @@ class UserShadow(Base): class LocalUser(Base): """Sensitive local-account info (password, status, contact details). 1:1 with users_shadow.""" + __tablename__ = "local_users" - user_id = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), primary_key=True) + user_id = Column( + String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), primary_key=True + ) password_hash = Column(String(255), nullable=False) nickname = Column(String(64)) real_name = Column(String(64)) @@ -65,212 +85,13 @@ class LocalUser(Base): ) -class InviteCode(Base): - """Invite code — generated by the Config platform, single-use, may pre-bind a team + role.""" - __tablename__ = "invite_codes" - - code = Column(String(32), primary_key=True) - created_by = Column(String(64)) - preset_team_id = Column(String(64), ForeignKey("teams.team_id", ondelete="SET NULL")) - preset_role = Column(String(16), nullable=False, default="member") - expires_at = Column(TIMESTAMP(timezone=True)) - used_by = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="SET NULL")) - used_at = Column(TIMESTAMP(timezone=True)) - revoked = Column(Boolean, nullable=False, default=False) - note = Column(String(255)) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - - __table_args__ = ( - CheckConstraint( - "preset_role IN ('admin', 'member')", - name="invite_codes_preset_role_check", - ), - Index("idx_invite_codes_used_by", "used_by"), - Index("idx_invite_codes_expires_at", "expires_at"), - Index("idx_invite_codes_revoked", "revoked", "used_by"), - ) - - -class Team(Base): - """Team table — created and maintained via the Config platform. Can also be auto-created from external SSO departments (source=sso_auto).""" - __tablename__ = "teams" - - team_id = Column(String(64), primary_key=True) - name = Column(String(128), nullable=False, unique=True) - description = Column(Text) - owner_user_id = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="SET NULL")) - - # External SSO department mapping: bridges the department field returned by external SSO - sso_department = Column(String(128)) # External department name (keeps the raw SSO string) - source = Column(String(16), nullable=False, default="manual") # manual | sso_auto - - # Team default permissions: configures capability bits for team members in one place - # (lab_enabled / can_use_api_key / can_add_skill / can_add_mcp / can_import_plugin / - # allowed_apps). Acts only as "defaults" — a member's own explicit settings take precedence; - # only the bits the team explicitly imposes (a subset) are stored, and unset bits do not - # affect members. Resolution logic lives in core/auth/capabilities.py. - default_permissions = Column(JSONType, nullable=True) - - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - - members = relationship("TeamMember", back_populates="team", cascade="all, delete-orphan") - - __table_args__ = ( - CheckConstraint("length(name) >= 1 AND length(name) <= 128", name="teams_name_length"), - CheckConstraint("source IN ('manual', 'sso_auto')", name="teams_source_check"), - Index("idx_teams_name", "name"), - Index("idx_teams_owner", "owner_user_id"), - Index("idx_teams_sso_department", "sso_department"), - ) - - -class TeamMember(Base): - """Team member (users N:M teams).""" - __tablename__ = "team_members" - - team_id = Column( - String(64), - ForeignKey("teams.team_id", ondelete="CASCADE"), - primary_key=True, - ) - user_id = Column( - String(64), - ForeignKey("users_shadow.user_id", ondelete="CASCADE"), - primary_key=True, - ) - role = Column(String(16), nullable=False, default="member") - # Team-folder file permission (only takes effect for role=member; owner/admin are always admin) - file_permission = Column(String(16), nullable=False, default="viewer") - joined_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - - team = relationship("Team", back_populates="members") - - __table_args__ = ( - CheckConstraint( - "role IN ('owner', 'admin', 'member')", - name="team_members_role_check", - ), - CheckConstraint( - "file_permission IN ('viewer', 'editor')", - name="team_members_file_permission_check", - ), - Index("idx_team_members_user_id", "user_id"), - Index("idx_team_members_team_role", "team_id", "role"), - ) - - -class Role(Base): - """Role — a reusable named capability bundle (maintained in the Config console's "Role Permissions" page). - - A role packages a set of capability bits into a "grant bundle" (``permissions``; after - normalization only granted boolean bits set to True + ``allowed_apps`` are stored), and is - assigned to a team (= department default role, inherited by members in real time) or to an - individual. Capability-bit resolution chain: individual explicit override → union of roles → - team defaults → system defaults (see core/auth/role_permissions.py and - core/auth/capabilities.py). - - Multiple roles are unioned (a grant from any role takes effect). ``is_system=True`` marks a - built-in role, which must not be deleted. - """ - __tablename__ = "roles" - - role_id = Column(String(64), primary_key=True) - name = Column(String(64), nullable=False, unique=True) - description = Column(Text) - # Normalized "grant" capability bundle: contains only granted boolean bits (True) + allowed_apps. - # Resolution: see core/auth/role_permissions.py::normalize_role_permissions. - permissions = Column(JSONType, nullable=False, default=dict) - is_system = Column(Boolean, nullable=False, default=False) # Built-in role, must not be deleted - # Default role for new teams: when checked, it is auto-assigned to teams created manually or - # via OA/SSO sync (members inherit it) - is_team_default = Column(Boolean, nullable=False, default=False) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - - __table_args__ = ( - CheckConstraint("length(name) >= 1 AND length(name) <= 64", name="roles_name_length"), - Index("idx_roles_name", "name"), - ) - - -class RoleAssignment(Base): - """Role assignment (roles N:M principals). - - The generalized ``principal_type`` is the single forward-compatibility point for a later - refactor into an organization tree: currently supports ``user`` (assigned directly to an - individual) / ``team`` (assigned to a team = department default role, inherited by members in - real time); when a Department(parent_id) tree lands later, we only need to add a - ``department`` type + walk up the parent-department chain in the resolver — no table - structure changes. - """ - __tablename__ = "role_assignments" - - role_id = Column( - String(64), - ForeignKey("roles.role_id", ondelete="CASCADE"), - primary_key=True, - ) - principal_type = Column(String(16), primary_key=True) # user | team (department reserved) - principal_id = Column(String(64), primary_key=True) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - - __table_args__ = ( - CheckConstraint( - "principal_type IN ('user', 'team', 'department')", - name="role_assignments_principal_type_check", - ), - Index("idx_role_assignments_principal", "principal_type", "principal_id"), - ) - - -class TeamFolder(Base): - """Team folder — tree structure; NULL parent means directly under the team root.""" - __tablename__ = "team_folders" - - folder_id = Column(String(64), primary_key=True) - team_id = Column( - String(64), - ForeignKey("teams.team_id", ondelete="CASCADE"), - nullable=False, - ) - parent_folder_id = Column( - String(64), - ForeignKey("team_folders.folder_id", ondelete="CASCADE"), - nullable=True, - ) - name = Column(String(255), nullable=False) - created_by = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="SET NULL")) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - deleted_at = Column(TIMESTAMP(timezone=True)) - - __table_args__ = ( - CheckConstraint( - "length(name) >= 1 AND length(name) <= 255", - name="team_folders_name_length", - ), - CheckConstraint( - "name NOT LIKE '%/%' AND name <> '.' AND name <> '..'", - name="team_folders_name_safe", - ), - Index("idx_team_folders_team_parent", "team_id", "parent_folder_id"), - Index("idx_team_folders_team_deleted", "team_id", "deleted_at"), - # Same-name uniqueness among siblings is enforced on PostgreSQL by the partial unique - # index ux_team_folders_name (see migration); in the SQLite dev environment, the - # duplicate check in TeamFolderService serves as the fallback. - ) - - class UserFolder(Base): """Personal folder — tree structure under My Space; NULL parent means the root directory. - Symmetric to TeamFolder: - - team_id dimension → user_id dimension - - Artifact.team_folder_id ↔ Artifact.user_folder_id Constraints such as tree depth and name validation are enforced by UserFolderService (see core/services/user_folder_service.py). """ + __tablename__ = "user_folders" folder_id = Column(String(64), primary_key=True) @@ -313,23 +134,28 @@ class UserApiKey(Base): decrypts on demand. Callers send ``Authorization: Bearer sk-jx-...``; the auth layer looks the key up by hash and inherits all of that user's capabilities. """ + __tablename__ = "user_api_keys" - id = Column(String(64), primary_key=True) - user_id = Column( + id = Column(String(64), primary_key=True) + user_id = Column( String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False, ) - name = Column(String(128), nullable=False, default="API Key") - key_prefix = Column(String(32), nullable=False) # Plaintext prefix (e.g. sk-jx-a1b2c3), for list display - key_hash = Column(String(128), nullable=False) # sha256(full key), unique - key_enc = Column(Text) # Reversible ciphertext of the full key (Fernet), supports copy-again; NULL for legacy keys - enabled = Column(Boolean, nullable=False, default=True) - expires_at = Column(TIMESTAMP(timezone=True)) # NULL = never expires + name = Column(String(128), nullable=False, default="API Key") + key_prefix = Column( + String(32), nullable=False + ) # Plaintext prefix (e.g. sk-jx-a1b2c3), for list display + key_hash = Column(String(128), nullable=False) # sha256(full key), unique + key_enc = Column( + Text + ) # Reversible ciphertext of the full key (Fernet), supports copy-again; NULL for legacy keys + enabled = Column(Boolean, nullable=False, default=True) + expires_at = Column(TIMESTAMP(timezone=True)) # NULL = never expires last_used_at = Column(TIMESTAMP(timezone=True)) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - revoked_at = Column(TIMESTAMP(timezone=True)) # Soft delete: revocation time + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) + revoked_at = Column(TIMESTAMP(timezone=True)) # Soft delete: revocation time __table_args__ = ( UniqueConstraint("key_hash", name="uq_user_api_keys_key_hash"), @@ -348,6 +174,7 @@ class DingTalkConnection(Base): (``auth_bundle``, for plan-B fallback/migration, stored after application-layer encryption). See internal design docs. """ + __tablename__ = "dingtalk_connections" user_id = Column( @@ -357,16 +184,26 @@ class DingTalkConnection(Base): ) # disconnected (default / disconnected) | pending (device-flow login in progress) | connected | error status = Column(String(16), nullable=False, default="disconnected") - dingtalk_user_id = Column(String(128)) # DingTalk userId (backfilled from get-self after a successful connect) - dingtalk_name = Column(String(255)) # DingTalk display name - corp_id = Column(String(128)) # DingTalk corp corpId - granted_scopes = Column(JSONType, default=list) # List of granted scopes (accumulated via PAT grants) + dingtalk_user_id = Column( + String(128) + ) # DingTalk userId (backfilled from get-self after a successful connect) + dingtalk_name = Column(String(255)) # DingTalk display name + corp_id = Column(String(128)) # DingTalk corp corpId + granted_scopes = Column( + JSONType, default=list + ) # List of granted scopes (accumulated via PAT grants) # Device-flow login state (echoed to the frontend while pending; cleared on success) - login_verification_url = Column(Text) # Plain verification URL (must be used together with user_code) - login_verification_url_complete = Column(Text) # Full URL with the code embedded → target of the QR code + login_verification_url = Column( + Text + ) # Plain verification URL (must be used together with user_code) + login_verification_url_complete = Column( + Text + ) # Full URL with the code embedded → target of the QR code login_user_code = Column(String(64)) login_started_at = Column(TIMESTAMP(timezone=True)) - auth_bundle = Column(Text) # Plan B: dws auth export --base64 (application-layer encrypted), nullable + auth_bundle = Column( + Text + ) # Plan B: dws auth export --base64 (application-layer encrypted), nullable last_verified_at = Column(TIMESTAMP(timezone=True)) last_error = Column(Text) created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) @@ -395,6 +232,7 @@ class LarkConnection(Base): (not injected as env vars, to avoid breaking --as user). See internal design docs. """ + __tablename__ = "lark_connections" user_id = Column( @@ -404,17 +242,29 @@ class LarkConnection(Base): ) # disconnected (default / disconnected) | pending (device-flow login in progress) | connected | error status = Column(String(16), nullable=False, default="disconnected") - lark_open_id = Column(String(128)) # Lark open_id / union_id (backfilled after a successful connect) - lark_name = Column(String(255)) # Lark display name - tenant_key = Column(String(128)) # Lark tenant tenant_key (counterpart of DingTalk corp_id) - granted_scopes = Column(JSONType, default=list) # List of granted scopes (accumulated via incremental grants) + lark_open_id = Column( + String(128) + ) # Lark open_id / union_id (backfilled after a successful connect) + lark_name = Column(String(255)) # Lark display name + tenant_key = Column(String(128)) # Lark tenant tenant_key (counterpart of DingTalk corp_id) + granted_scopes = Column( + JSONType, default=list + ) # List of granted scopes (accumulated via incremental grants) # Device-flow login state (echoed to the frontend while pending; cleared on success) - login_verification_url = Column(Text) # Plain verification URL (must be used together with user_code) - login_verification_url_complete = Column(Text) # Full URL with the code embedded → target of the QR code + login_verification_url = Column( + Text + ) # Plain verification URL (must be used together with user_code) + login_verification_url_complete = Column( + Text + ) # Full URL with the code embedded → target of the QR code login_user_code = Column(String(64)) - login_device_code = Column(Text) # device_code obtained via --no-wait, used when completing with --device-code + login_device_code = Column( + Text + ) # device_code obtained via --no-wait, used when completing with --device-code login_started_at = Column(TIMESTAMP(timezone=True)) - auth_bundle = Column(Text) # Portable credential bundle (for cube cross-machine use, application-layer encrypted), nullable + auth_bundle = Column( + Text + ) # Portable credential bundle (for cube cross-machine use, application-layer encrypted), nullable last_verified_at = Column(TIMESTAMP(timezone=True)) last_error = Column(Text) created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) @@ -448,6 +298,7 @@ class EmailConnection(Base): per-session when cube has no bind-mount, base64(config.toml)). See internal design docs. """ + __tablename__ = "email_connections" user_id = Column( @@ -457,20 +308,24 @@ class EmailConnection(Base): ) # disconnected (default / disconnected) | connected | error (no pending — binding is a synchronous check) status = Column(String(16), nullable=False, default="disconnected") - email_address = Column(String(320)) # Bound email address (= IMAP/SMTP login name) - display_name = Column(String(255)) # Sender display name - provider = Column(String(32)) # Auto-detected provider: gmail/outlook/netease/qq/qiye163/exmail/custom + email_address = Column(String(320)) # Bound email address (= IMAP/SMTP login name) + display_name = Column(String(255)) # Sender display name + provider = Column( + String(32) + ) # Auto-detected provider: gmail/outlook/netease/qq/qiye163/exmail/custom # Server settings (auto-detected + user-overridable) imap_host = Column(String(255)) imap_port = Column(Integer) - imap_security = Column(String(16)) # tls | starttls | none + imap_security = Column(String(16)) # tls | starttls | none smtp_host = Column(String(255)) smtp_port = Column(Integer) smtp_security = Column(String(16)) - secret_enc = Column(Text) # Authorization code (application-layer Fernet encryption, never stored in plaintext) - config_bundle = Column(Text) # base64(config.toml), for cube cross-session injection, nullable + secret_enc = Column( + Text + ) # Authorization code (application-layer Fernet encryption, never stored in plaintext) + config_bundle = Column(Text) # base64(config.toml), for cube cross-session injection, nullable last_verified_at = Column(TIMESTAMP(timezone=True)) last_error = Column(Text) created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) @@ -507,6 +362,7 @@ class ChannelConnection(Base): See internal design docs. """ + __tablename__ = "channel_connections" channel_id = Column(String(64), primary_key=True) @@ -515,11 +371,15 @@ class ChannelConnection(Base): ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False, ) - channel_type = Column(String(16), nullable=False) # 'lark' | 'dingtalk' | 'wecom' | ... + channel_type = Column(String(16), nullable=False) # 'lark' | 'dingtalk' | 'wecom' | ... display_name = Column(String(100), nullable=False, default="我的机器人") transport = Column(String(16), nullable=False, default="long_conn") # 'long_conn' | 'webhook' - app_id = Column(String(128), nullable=False) # Plaintext redundancy: for the unique constraint + establishing the long connection - config = Column(JSONType, nullable=False, default=dict) # Encrypted credentials {app_secret_enc, encrypt_key_enc, verification_token_enc} + app_id = Column( + String(128), nullable=False + ) # Plaintext redundancy: for the unique constraint + establishing the long connection + config = Column( + JSONType, nullable=False, default=dict + ) # Encrypted credentials {app_secret_enc, encrypt_key_enc, verification_token_enc} # Resource allowlist (for when the owner wants precise control over what the bot exposes): # {"kb_ids": [...], "skill_ids": [...]}; NULL = expose everything the owner has (default). # Retrieval/skill loading narrows down by this on top of the owner's permissions. diff --git a/src/backend/core/db/models/knowledge.py b/src/backend/core/db/models/knowledge.py index e93a23c5..73065f47 100644 --- a/src/backend/core/db/models/knowledge.py +++ b/src/backend/core/db/models/knowledge.py @@ -1,13 +1,25 @@ """SQLAlchemy ORM models — knowledge base / capability catalog.""" from datetime import datetime, timezone + +from core.db.engine import Base from sqlalchemy import ( - Column, String, Integer, BigInteger, Boolean, Text, TIMESTAMP, - ForeignKey, CheckConstraint, UniqueConstraint, Index, Numeric, JSON + JSON, + TIMESTAMP, + BigInteger, + Boolean, + CheckConstraint, + Column, + ForeignKey, + Index, + Integer, + Numeric, + String, + Text, + UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import JSONB, INET -from sqlalchemy.orm import relationship, mapped_column -from core.db.engine import Base +from sqlalchemy.dialects.postgresql import INET, JSONB +from sqlalchemy.orm import mapped_column, relationship JSONType = JSON().with_variant(JSONB(), "postgresql") INETType = String(45).with_variant(INET(), "postgresql") @@ -15,10 +27,13 @@ class KBSpace(Base): """Knowledge base space table.""" + __tablename__ = "kb_spaces" kb_id = Column(String(64), primary_key=True) - user_id = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False) + user_id = Column( + String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False + ) name = Column(String(255), nullable=False) description = Column(Text) document_count = Column(Integer, default=0) @@ -38,16 +53,21 @@ class KBSpace(Base): CheckConstraint("length(name) >= 1 AND length(name) <= 255", name="kb_spaces_name_length"), CheckConstraint("document_count >= 0", name="kb_spaces_document_count_check"), CheckConstraint("total_size_bytes >= 0", name="kb_spaces_total_size_check"), - CheckConstraint("visibility IN ('public', 'private', 'scoped')", name="kb_spaces_visibility_check"), + CheckConstraint( + "visibility IN ('public', 'private', 'scoped')", name="kb_spaces_visibility_check" + ), Index("idx_kb_spaces_user_id", "user_id"), Index("idx_kb_spaces_updated_at", "updated_at"), - Index("idx_kb_spaces_deleted", "deleted_at", postgresql_where=Column("deleted_at").isnot(None)), + Index( + "idx_kb_spaces_deleted", "deleted_at", postgresql_where=Column("deleted_at").isnot(None) + ), Index("idx_kb_spaces_visibility", "visibility"), ) class KBDocument(Base): """Knowledge base document table.""" + __tablename__ = "kb_documents" document_id = Column(String(64), primary_key=True) @@ -59,7 +79,9 @@ class KBDocument(Base): storage_key = Column(Text, nullable=False) storage_url = Column(Text) checksum = Column(String(64)) - indexing_status = Column(String(20), nullable=False, default="processing") # processing | completed | failed + indexing_status = Column( + String(20), nullable=False, default="processing" + ) # processing | completed | failed extra_data = Column("metadata", JSONType, default={}) uploaded_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) deleted_at = Column(TIMESTAMP(timezone=True)) @@ -74,7 +96,11 @@ class KBDocument(Base): Index("idx_kb_documents_kb_id", "kb_id"), Index("idx_kb_documents_uploaded_at", "uploaded_at"), Index("idx_kb_documents_kb_uploaded", "kb_id", "uploaded_at"), - Index("idx_kb_documents_deleted", "deleted_at", postgresql_where=Column("deleted_at").isnot(None)), + Index( + "idx_kb_documents_deleted", + "deleted_at", + postgresql_where=Column("deleted_at").isnot(None), + ), Index("idx_kb_documents_metadata_gin", "metadata", postgresql_using="gin"), ) @@ -86,15 +112,20 @@ class KBChunk(Base): (vectorised in Milvus hugagent_kb_private collection). Retrieval finds child chunks via vector search, then fetches the parent content from this table. """ + __tablename__ = "kb_chunks" chunk_id = Column(String(64), primary_key=True) kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False) - document_id = Column(String(64), ForeignKey("kb_documents.document_id", ondelete="CASCADE"), nullable=False) + document_id = Column( + String(64), ForeignKey("kb_documents.document_id", ondelete="CASCADE"), nullable=False + ) chunk_index = Column(Integer, nullable=False) - content = Column(Text, nullable=False) # parent chunk original text, returned to the LLM on retrieval hit - tags = Column(JSONType, default=list) # tag list ["数字化转型", "申报条件"] - questions = Column(JSONType, default=list) # associated question list (array of strings) + content = Column( + Text, nullable=False + ) # parent chunk original text, returned to the LLM on retrieval hit + tags = Column(JSONType, default=list) # tag list ["数字化转型", "申报条件"] + questions = Column(JSONType, default=list) # associated question list (array of strings) char_start = Column(Integer) char_end = Column(Integer) created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) @@ -112,10 +143,13 @@ class KBChunk(Base): class CatalogOverride(Base): """Catalog override table - user customizations for skills/agents/MCPs.""" + __tablename__ = "catalog_overrides" override_id = Column(Integer, primary_key=True, autoincrement=True) - user_id = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False) + user_id = Column( + String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False + ) kind = Column(String(20), nullable=False) item_id = Column(String(100), nullable=False) enabled = Column(Boolean, nullable=False, default=True) @@ -128,45 +162,9 @@ class CatalogOverride(Base): __table_args__ = ( CheckConstraint("kind IN ('skill', 'agent', 'mcp')", name="catalog_overrides_kind_check"), - UniqueConstraint("user_id", "kind", "item_id", name="catalog_overrides_unique_user_kind_item"), + UniqueConstraint( + "user_id", "kind", "item_id", name="catalog_overrides_unique_user_kind_item" + ), Index("idx_catalog_overrides_user_id", "user_id"), Index("idx_catalog_overrides_kind", "kind", "enabled"), ) - - -class KBGrant(Base): - """Knowledge-base grant table — assigns KB access per user/team (implicit authorization model). - - Uniformly carries both local shared bases and Dify datasets: ``resource_id`` - is the ``kb_id`` or Dify ``dataset_id``, distinguished by ``resource_type``. - ``level`` is a view/edit/admin tier (modeled on team folder permissions): - - view : visible in the capability catalog + retrievable by the agent (read-only) - - edit : on top of view, can upload documents - - admin : on top of edit, can manage (modify/delete/configure) - - Visibility is set explicitly on ``KBSpace.visibility`` (public = visible to - everyone / scoped = visible to designated). This table assigns the access - subjects of **scoped bases**. A public base is visible to everyone; a grant - in this table then elevates individual users/teams to edit/admin. Personal - grants take precedence over team grants. (Dify datasets have no visibility - column, treated as "restricted whenever a grant exists".) - """ - __tablename__ = "kb_grants" - - resource_id = Column(String(64), primary_key=True) - resource_type = Column(String(8), primary_key=True) # 'local' | 'dify' - principal_type = Column(String(8), primary_key=True) # 'user' | 'team' - principal_id = Column(String(64), primary_key=True) # user_id | team_id - level = Column(String(8), nullable=False, default="view") # view | edit | admin - granted_by = Column(String(64)) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - - __table_args__ = ( - CheckConstraint("resource_type IN ('local', 'dify')", name="kb_grants_resource_type_check"), - CheckConstraint("principal_type IN ('user', 'team')", name="kb_grants_principal_type_check"), - CheckConstraint("level IN ('view', 'edit', 'admin')", name="kb_grants_level_check"), - # Reverse lookup "which bases a given user/team can access" (resolver hot path) - Index("idx_kb_grants_principal", "principal_type", "principal_id"), - # Forward lookup "who a given base is granted to" (admin console display) - Index("idx_kb_grants_resource", "resource_id", "resource_type"), - ) diff --git a/src/backend/core/db/models/logs.py b/src/backend/core/db/models/logs.py index f5357a28..fcf89fcf 100644 --- a/src/backend/core/db/models/logs.py +++ b/src/backend/core/db/models/logs.py @@ -1,13 +1,25 @@ """SQLAlchemy ORM models — call logs / audit.""" from datetime import datetime, timezone + +from core.db.engine import Base from sqlalchemy import ( - Column, String, Integer, BigInteger, Boolean, Text, TIMESTAMP, - ForeignKey, CheckConstraint, UniqueConstraint, Index, Numeric, JSON + JSON, + TIMESTAMP, + BigInteger, + Boolean, + CheckConstraint, + Column, + ForeignKey, + Index, + Integer, + Numeric, + String, + Text, + UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import JSONB, INET -from sqlalchemy.orm import relationship, mapped_column -from core.db.engine import Base +from sqlalchemy.dialects.postgresql import INET, JSONB +from sqlalchemy.orm import mapped_column, relationship JSONType = JSON().with_variant(JSONB(), "postgresql") INETType = String(45).with_variant(INET(), "postgresql") @@ -19,32 +31,33 @@ class ToolCallLog(Base): """Tool call log — one row per MCP / built-in tool execution.""" + __tablename__ = "tool_call_logs" - id = Column(String(64), primary_key=True) - trace_id = Column(String(64)) - chat_id = Column(String(64), index=True) - message_id = Column(String(64)) - user_id = Column(String(64), index=True) - user_name = Column(String(255)) - tool_name = Column(String(128), nullable=False) - tool_display_name= Column(String(255)) - tool_call_id = Column(String(64)) - mcp_server = Column(String(64)) + id = Column(String(64), primary_key=True) + trace_id = Column(String(64)) + chat_id = Column(String(64), index=True) + message_id = Column(String(64)) + user_id = Column(String(64), index=True) + user_name = Column(String(255)) + tool_name = Column(String(128), nullable=False) + tool_display_name = Column(String(255)) + tool_call_id = Column(String(64)) + mcp_server = Column(String(64)) # Sandbox instance id (only set for sandbox tools bash / sandbox_put_artifact / sandbox_get_artifact): # links "which tool call → which sandbox instance → which user" together, supporting audit filtering by sandbox. - sandbox_id = Column(String(128), index=True) - tool_args = Column(JSONType) - tool_result = Column(JSONType) + sandbox_id = Column(String(128), index=True) + tool_args = Column(JSONType) + tool_result = Column(JSONType) result_truncated = Column(Boolean, default=False, nullable=False) - status = Column(String(20), nullable=False, default="success") - error_message = Column(Text) - duration_ms = Column(Integer) - source = Column(String(20), nullable=False, default="main_agent") - subagent_log_id = Column(String(64), index=True) - skill_log_id = Column(String(64), index=True) - started_at = Column(TIMESTAMP(timezone=True)) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, nullable=False) + status = Column(String(20), nullable=False, default="success") + error_message = Column(Text) + duration_ms = Column(Integer) + source = Column(String(20), nullable=False, default="main_agent") + subagent_log_id = Column(String(64), index=True) + skill_log_id = Column(String(64), index=True) + started_at = Column(TIMESTAMP(timezone=True)) + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, nullable=False) __table_args__ = ( CheckConstraint( @@ -67,35 +80,36 @@ class ToolCallLog(Base): class SubAgentCallLog(Base): """Sub-agent call log — a full execution record of one sub-agent / plan step.""" + __tablename__ = "subagent_call_logs" - id = Column(String(64), primary_key=True) - trace_id = Column(String(64)) - chat_id = Column(String(64), index=True) - message_id = Column(String(64)) - user_id = Column(String(64), index=True) - user_name = Column(String(255)) - subagent_id = Column(String(64)) - subagent_name = Column(String(128), nullable=False) - subagent_type = Column(String(32)) # plan_mode / report_generator / user_agent ... - plan_id = Column(String(64)) - step_id = Column(String(64)) - step_index = Column(Integer) - step_title = Column(String(500)) - model = Column(String(128)) - input_messages = Column(JSONType) - output_content = Column(Text) - intermediate_steps = Column(JSONType) - token_usage = Column(JSONType) - tool_calls_count = Column(Integer, default=0) - skill_calls_count = Column(Integer, default=0) - status = Column(String(20), nullable=False, default="running") - error_message = Column(Text) - duration_ms = Column(Integer) + id = Column(String(64), primary_key=True) + trace_id = Column(String(64)) + chat_id = Column(String(64), index=True) + message_id = Column(String(64)) + user_id = Column(String(64), index=True) + user_name = Column(String(255)) + subagent_id = Column(String(64)) + subagent_name = Column(String(128), nullable=False) + subagent_type = Column(String(32)) # plan_mode / report_generator / user_agent ... + plan_id = Column(String(64)) + step_id = Column(String(64)) + step_index = Column(Integer) + step_title = Column(String(500)) + model = Column(String(128)) + input_messages = Column(JSONType) + output_content = Column(Text) + intermediate_steps = Column(JSONType) + token_usage = Column(JSONType) + tool_calls_count = Column(Integer, default=0) + skill_calls_count = Column(Integer, default=0) + status = Column(String(20), nullable=False, default="running") + error_message = Column(Text) + duration_ms = Column(Integer) parent_subagent_log_id = Column(String(64), index=True) - started_at = Column(TIMESTAMP(timezone=True)) - completed_at = Column(TIMESTAMP(timezone=True)) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, nullable=False) + started_at = Column(TIMESTAMP(timezone=True)) + completed_at = Column(TIMESTAMP(timezone=True)) + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, nullable=False) __table_args__ = ( CheckConstraint( @@ -114,34 +128,35 @@ class SubAgentCallLog(Base): class SkillCallLog(Base): """Skill call log — records all three trigger types: view / run_script / auto_load.""" + __tablename__ = "skill_call_logs" - id = Column(String(64), primary_key=True) - trace_id = Column(String(64)) - chat_id = Column(String(64), index=True) - message_id = Column(String(64)) - user_id = Column(String(64), index=True) - user_name = Column(String(255)) - skill_id = Column(String(128), nullable=False) - skill_name = Column(String(255)) - skill_version = Column(String(50)) - skill_source = Column(String(20)) # filesystem / database - invocation_type = Column(String(20), nullable=False, default="auto_load") - script_name = Column(String(255)) - script_language = Column(String(32)) - script_args = Column(JSONType) - script_stdin = Column(Text) - script_stdout = Column(Text) - script_stderr = Column(Text) - output_truncated = Column(Boolean, default=False, nullable=False) - exit_code = Column(Integer) - status = Column(String(20), nullable=False, default="success") - error_message = Column(Text) - duration_ms = Column(Integer) - source = Column(String(20), nullable=False, default="main_agent") - subagent_log_id = Column(String(64), index=True) - started_at = Column(TIMESTAMP(timezone=True)) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, nullable=False) + id = Column(String(64), primary_key=True) + trace_id = Column(String(64)) + chat_id = Column(String(64), index=True) + message_id = Column(String(64)) + user_id = Column(String(64), index=True) + user_name = Column(String(255)) + skill_id = Column(String(128), nullable=False) + skill_name = Column(String(255)) + skill_version = Column(String(50)) + skill_source = Column(String(20)) # filesystem / database + invocation_type = Column(String(20), nullable=False, default="auto_load") + script_name = Column(String(255)) + script_language = Column(String(32)) + script_args = Column(JSONType) + script_stdin = Column(Text) + script_stdout = Column(Text) + script_stderr = Column(Text) + output_truncated = Column(Boolean, default=False, nullable=False) + exit_code = Column(Integer) + status = Column(String(20), nullable=False, default="success") + error_message = Column(Text) + duration_ms = Column(Integer) + source = Column(String(20), nullable=False, default="main_agent") + subagent_log_id = Column(String(64), index=True) + started_at = Column(TIMESTAMP(timezone=True)) + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, nullable=False) __table_args__ = ( CheckConstraint( @@ -164,36 +179,3 @@ class SkillCallLog(Base): Index("idx_skill_call_logs_status", "status", "created_at"), Index("idx_skill_call_logs_trace_id", "trace_id"), ) - - -class AuditLog(Base): - """Audit log table - record all critical operations.""" - __tablename__ = "audit_logs" - - log_id = Column(BigIntPK, primary_key=True, autoincrement=True) - user_id = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="SET NULL")) - action = Column(String(100), nullable=False) - resource_type = Column(String(50)) - resource_id = Column(String(64)) - # Sandbox instance id (only set for sandbox operations sandbox.bash.exec / sandbox.artifact.*): - # lets the security management → audit log distinguish and filter by "sandbox instance", i.e. "who ran what in which sandbox". - sandbox_id = Column(String(128), index=True) - details = Column(JSONType, default={}) - ip_address = Column(INETType) - user_agent = Column(Text) - trace_id = Column(String(64)) - status = Column(String(20), default="success") - error_code = Column(Integer) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - - __table_args__ = ( - CheckConstraint("status IN ('success', 'failure', 'error')", name="audit_logs_status_check"), - Index("idx_audit_logs_user_id", "user_id"), - Index("idx_audit_logs_action", "action"), - Index("idx_audit_logs_resource", "resource_type", "resource_id"), - Index("idx_audit_logs_sandbox_created", "sandbox_id", "created_at"), - Index("idx_audit_logs_created_at", "created_at"), - Index("idx_audit_logs_user_created", "user_id", "created_at"), - Index("idx_audit_logs_trace_id", "trace_id"), - Index("idx_audit_logs_status", "status", "created_at", postgresql_where=Column("status") != "success"), - ) diff --git a/src/backend/core/db/models/memory.py b/src/backend/core/db/models/memory.py index fdc31658..9fdb517f 100644 --- a/src/backend/core/db/models/memory.py +++ b/src/backend/core/db/models/memory.py @@ -1,13 +1,25 @@ """SQLAlchemy ORM models — memory.""" from datetime import datetime, timezone + +from core.db.engine import Base from sqlalchemy import ( - Column, String, Integer, BigInteger, Boolean, Text, TIMESTAMP, - ForeignKey, CheckConstraint, UniqueConstraint, Index, Numeric, JSON + JSON, + TIMESTAMP, + BigInteger, + Boolean, + CheckConstraint, + Column, + ForeignKey, + Index, + Integer, + Numeric, + String, + Text, + UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import JSONB, INET -from sqlalchemy.orm import relationship, mapped_column -from core.db.engine import Base +from sqlalchemy.dialects.postgresql import INET, JSONB +from sqlalchemy.orm import mapped_column, relationship JSONType = JSON().with_variant(JSONB(), "postgresql") INETType = String(45).with_variant(INET(), "postgresql") @@ -22,66 +34,33 @@ class ProfileMemory(Base): Primary key is (user_id, workspace_id) — the same natural person has isolated memory across workspaces. """ + __tablename__ = "profile_memory" - user_id = Column(String(64), primary_key=True) - workspace_id = Column(String(64), primary_key=True, default="default") - content_md = Column(Text, nullable=False, default="") + user_id = Column(String(64), primary_key=True) + workspace_id = Column(String(64), primary_key=True, default="default") + content_md = Column(Text, nullable=False, default="") last_compacted_at = Column(TIMESTAMP(timezone=True)) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - - __table_args__ = ( - Index("idx_profile_memory_updated_at", "updated_at"), - ) - + updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) -class MemoryAudit(Base): - """Audit trail for all memory read / write / update / delete / write-rejected operations. - - content_hash stores a SHA256; the original text is never persisted. - """ - __tablename__ = "memory_audit" - - id = Column(BigIntPK, primary_key=True, autoincrement=True) - ts = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, nullable=False) - actor = Column(String(64), nullable=False) - action = Column(String(16), nullable=False) - layer = Column(String(16), nullable=False) - memory_id = Column(String(128)) - workspace_id = Column(String(64), default="default") - user_id = Column(String(64)) - chat_id = Column(String(64)) - confidentiality = Column(String(16)) - content_hash = Column(String(64)) - reason = Column(Text) - - __table_args__ = ( - CheckConstraint( - "action IN ('read','write','update','delete','write_rejected','forget')", - name="memory_audit_action_check", - ), - CheckConstraint( - "layer IN ('L1','L2','L3','session','batch')", - name="memory_audit_layer_check", - ), - Index("idx_memory_audit_user_ts", "user_id", "ts"), - Index("idx_memory_audit_workspace_ts", "workspace_id", "ts"), - Index("idx_memory_audit_ts", "ts"), - ) + __table_args__ = (Index("idx_profile_memory_updated_at", "updated_at"),) class MemorySanitizerRule(Base): """Sensitive-word rules appended / disabled at runtime (defaults are hardcoded in memory_sanitizer.py).""" + __tablename__ = "memory_sanitizer_rules" - id = Column(BigIntPK, primary_key=True, autoincrement=True) - rule_type = Column(String(32), nullable=False) # redact | classified | disable_redact | disable_classified - name = Column(String(64)) # redact rule name; used as target name when disable_redact - pattern = Column(Text, nullable=False) # redact regex or classified word + id = Column(BigIntPK, primary_key=True, autoincrement=True) + rule_type = Column( + String(32), nullable=False + ) # redact | classified | disable_redact | disable_classified + name = Column(String(64)) # redact rule name; used as target name when disable_redact + pattern = Column(Text, nullable=False) # redact regex or classified word description = Column(Text) - enabled = Column(Boolean, default=True, nullable=False) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - created_by = Column(String(64)) + enabled = Column(Boolean, default=True, nullable=False) + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) + created_by = Column(String(64)) __table_args__ = ( CheckConstraint( diff --git a/src/backend/core/db/models/project.py b/src/backend/core/db/models/project.py index 2b09b01a..9e58cdf3 100644 --- a/src/backend/core/db/models/project.py +++ b/src/backend/core/db/models/project.py @@ -1,73 +1,77 @@ """SQLAlchemy ORM models — Projects.""" from datetime import datetime, timezone + +from core.db.engine import Base +from core.db.model_extensions import ProjectEditionFields, project_edition_table_args from sqlalchemy import ( - Column, String, Integer, BigInteger, Boolean, Text, TIMESTAMP, - ForeignKey, CheckConstraint, UniqueConstraint, Index, Numeric, JSON + JSON, + TIMESTAMP, + BigInteger, + Boolean, + CheckConstraint, + Column, + ForeignKey, + Index, + Integer, + Numeric, + String, + Text, + UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import JSONB, INET -from sqlalchemy.orm import relationship, mapped_column -from core.db.engine import Base +from sqlalchemy.dialects.postgresql import INET, JSONB +from sqlalchemy.orm import mapped_column, relationship JSONType = JSON().with_variant(JSONB(), "postgresql") INETType = String(45).with_variant(INET(), "postgresql") -class Project(Base): - """Project (personal / team) — Claude-style workspace. +class Project(ProjectEditionFields, Base): + """Project workspace; edition extensions add optional organization scope.""" - Personal project: ``kind='personal'`` + ``owner_user_id`` is the owner. - Team project: ``kind='team'`` + ``team_id`` is the owning team; ``owner_user_id`` - records the creator. Team project visibility / write permission follows - ``TeamMember.role`` + ``file_permission`` (owner/admin are always admin; member is - two-tiered editor/viewer per file_permission). - """ __tablename__ = "projects" - project_id = Column(String(64), primary_key=True) - name = Column(String(120), nullable=False) - description = Column(Text) - kind = Column(String(16), nullable=False) # 'personal' | 'team' - owner_user_id = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False) - team_id = Column(String(64), ForeignKey("teams.team_id", ondelete="CASCADE"), nullable=True) - # Project ↔ folder strict 1:1 linkage (mutually exclusive): - # - personal project: linked_folder_id points to user_folders (personal folder) - # - team project: linked_team_folder_id points to team_folders (team folder) - # The service layer guarantees only one is non-null on write, and that it matches kind. - linked_folder_id = Column(String(64), ForeignKey("user_folders.folder_id", ondelete="SET NULL"), nullable=True) - linked_team_folder_id = Column(String(64), ForeignKey("team_folders.folder_id", ondelete="SET NULL"), nullable=True) - instructions = Column(Text) - icon_color = Column(String(20)) - pinned = Column(Boolean, nullable=False, default=False) - extra_data = Column("metadata", JSONType, default={}) - created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) + project_id = Column(String(64), primary_key=True) + name = Column(String(120), nullable=False) + description = Column(Text) + kind = Column(String(16), nullable=False) + owner_user_id = Column( + String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False + ) + linked_folder_id = Column( + String(64), ForeignKey("user_folders.folder_id", ondelete="SET NULL"), nullable=True + ) + instructions = Column(Text) + icon_color = Column(String(20)) + pinned = Column(Boolean, nullable=False, default=False) + extra_data = Column("metadata", JSONType, default={}) + created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) + updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) last_activity_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - deleted_at = Column(TIMESTAMP(timezone=True)) + deleted_at = Column(TIMESTAMP(timezone=True)) __table_args__ = ( + *project_edition_table_args(), CheckConstraint( - "(kind = 'personal' AND team_id IS NULL) OR (kind = 'team' AND team_id IS NOT NULL)", - name="ck_projects_kind_team", + "length(name) >= 1 AND length(name) <= 120", name="ck_projects_name_length" ), - CheckConstraint("kind IN ('personal','team')", name="ck_projects_kind_enum"), - CheckConstraint("length(name) >= 1 AND length(name) <= 120", name="ck_projects_name_length"), Index("idx_projects_owner", "owner_user_id"), - Index("idx_projects_team", "team_id"), Index("idx_projects_last_activity", "last_activity_at"), Index("idx_projects_linked_user_folder", "linked_folder_id"), - Index("idx_projects_linked_team_folder", "linked_team_folder_id"), ) class ProjectFavorite(Base): """Per-user independent star (does not affect others' view of the project).""" + __tablename__ = "project_favorites" - project_id = Column(String(64), ForeignKey("projects.project_id", ondelete="CASCADE"), primary_key=True) - user_id = Column(String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), primary_key=True) + project_id = Column( + String(64), ForeignKey("projects.project_id", ondelete="CASCADE"), primary_key=True + ) + user_id = Column( + String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), primary_key=True + ) created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - __table_args__ = ( - Index("idx_project_favorites_user", "user_id"), - ) + __table_args__ = (Index("idx_project_favorites_user", "user_id"),) diff --git a/src/backend/core/db/models/site.py b/src/backend/core/db/models/site.py index dfa13633..3c6106ba 100644 --- a/src/backend/core/db/models/site.py +++ b/src/backend/core/db/models/site.py @@ -2,18 +2,27 @@ from datetime import datetime +from core.db.engine import Base +from core.db.models.site_scope import SiteScopeMixin, site_scope_table_args from sqlalchemy import ( - Column, String, Integer, BigInteger, Text, TIMESTAMP, - ForeignKey, CheckConstraint, Index, JSON, PrimaryKeyConstraint, + JSON, + TIMESTAMP, + BigInteger, + CheckConstraint, + Column, + ForeignKey, + Index, + Integer, + PrimaryKeyConstraint, + String, + Text, ) from sqlalchemy.dialects.postgresql import JSONB -from core.db.engine import Base - JSONType = JSON().with_variant(JSONB(), "postgresql") -class Site(Base): +class Site(SiteScopeMixin, Base): """User site — a static website generated in chat, files stored under ``sites//v/``. slug is globally unique (hosting URL is ``/site//``); on soft delete the @@ -36,12 +45,6 @@ class Site(Base): ForeignKey("chat_sessions.chat_id", ondelete="SET NULL"), nullable=True, ) - # Authorization scope when visibility=team: members of this team can access - team_id = Column( - String(64), - ForeignKey("teams.team_id", ondelete="SET NULL"), - nullable=True, - ) # The source-code project (personal project) this site corresponds to. Non-null → the # site can be "re-edited": building/editing both happen inside this project folder, and # publish_site takes files from the project folder. Old sites are empty (source only in @@ -54,8 +57,6 @@ class Site(Base): ) title = Column(String(200), nullable=False) description = Column(Text) - # public=anyone with the link; private=owner only; team=members of team_id (all validated via session cookie) - visibility = Column(String(16), nullable=False, default="public") entry_file = Column(String(200), nullable=False, default="index.html") current_version = Column(Integer, nullable=False, default=1) file_count = Column(Integer, nullable=False, default=0) @@ -64,18 +65,13 @@ class Site(Base): view_count = Column(BigInteger, nullable=False, default=0) extra_data = Column("metadata", JSONType, default={}) created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - updated_at = Column( - TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow - ) + updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) deleted_at = Column(TIMESTAMP(timezone=True)) __table_args__ = ( - CheckConstraint( - "visibility IN ('public', 'private', 'team')", name="sites_visibility_check" - ), + *site_scope_table_args(), CheckConstraint("current_version >= 1", name="sites_version_check"), Index("idx_sites_user_id", "user_id"), - Index("idx_sites_team_id", "team_id"), Index("idx_sites_project_id", "project_id"), Index("idx_sites_updated_at", "updated_at"), ) @@ -86,14 +82,10 @@ class SiteKV(Base): __tablename__ = "site_kv" - site_id = Column( - String(64), ForeignKey("sites.site_id", ondelete="CASCADE"), nullable=False - ) + site_id = Column(String(64), ForeignKey("sites.site_id", ondelete="CASCADE"), nullable=False) k = Column(String(64), nullable=False) v = Column(Text, nullable=False) - updated_at = Column( - TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow - ) + updated_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) __table_args__ = (PrimaryKeyConstraint("site_id", "k"),) @@ -104,14 +96,10 @@ class SiteSubmission(Base): __tablename__ = "site_submissions" id = Column(String(64), primary_key=True) - site_id = Column( - String(64), ForeignKey("sites.site_id", ondelete="CASCADE"), nullable=False - ) + site_id = Column(String(64), ForeignKey("sites.site_id", ondelete="CASCADE"), nullable=False) form_key = Column(String(64), nullable=False) payload = Column(JSONType, nullable=False) client_ip = Column(String(45)) created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow) - __table_args__ = ( - Index("idx_site_submissions_site_created", "site_id", "created_at"), - ) + __table_args__ = (Index("idx_site_submissions_site_created", "site_id", "created_at"),) diff --git a/src/backend/core/db/models/site_scope.py b/src/backend/core/db/models/site_scope.py new file mode 100644 index 00000000..e242578f --- /dev/null +++ b/src/backend/core/db/models/site_scope.py @@ -0,0 +1,19 @@ +"""Community-edition site visibility ORM fields.""" + +from sqlalchemy import CheckConstraint, Column, String +from sqlalchemy.orm import declared_attr + + +class SiteScopeMixin: + @declared_attr + def visibility(cls): + return Column(String(16), nullable=False, default="public") + + +def site_scope_table_args() -> tuple: + return ( + CheckConstraint( + "visibility IN ('public', 'private')", + name="sites_visibility_check", + ), + ) diff --git a/src/backend/core/db/repository/__init__.py b/src/backend/core/db/repository/__init__.py index c1a48809..32654911 100644 --- a/src/backend/core/db/repository/__init__.py +++ b/src/backend/core/db/repository/__init__.py @@ -1,33 +1,22 @@ -"""Data access layer — Repository pattern. +"""Community-edition repository exports.""" -Repositories are organised into domain submodules; this package re-exports -every public class so existing imports (``from core.db.repository import -XxxRepository``) keep working. -""" - -from core.db.repository.user import UserRepository, LocalUserRepository, DingTalkConnectionRepository, LarkConnectionRepository, EmailConnectionRepository -from core.db.repository.chat import ChatSessionRepository, ChatMessageRepository -from core.db.repository.catalog import CatalogRepository -from core.db.repository.kb import KBRepository, KBGrantRepository -from core.db.repository.artifact import ArtifactRepository, ROOT_FOLDER_SENTINEL -from core.db.repository.audit import AuditLogRepository from core.db.repository.agent import UserAgentRepository -from core.db.repository.team import TeamRepository, InviteCodeRepository -from core.db.repository.role import RoleRepository +from core.db.repository.artifact import ROOT_FOLDER_SENTINEL, ArtifactRepository +from core.db.repository.audit import AuditLogRepository +from core.db.repository.catalog import CatalogRepository from core.db.repository.channel import ChannelConnectionRepository -from core.db.repository.site import SiteRepository +from core.db.repository.chat import ChatMessageRepository, ChatSessionRepository +from core.db.repository.kb import KBRepository from core.db.repository.ontology import OntologyRepository +from core.db.repository.site import SiteRepository +from core.db.repository.user import ( + DingTalkConnectionRepository, + EmailConnectionRepository, + LarkConnectionRepository, + LocalUserRepository, + UserRepository, +) __all__ = [ - "UserRepository", "LocalUserRepository", "DingTalkConnectionRepository", - "LarkConnectionRepository", "EmailConnectionRepository", - "ChatSessionRepository", "ChatMessageRepository", - "CatalogRepository", "KBRepository", "KBGrantRepository", - "ArtifactRepository", "ROOT_FOLDER_SENTINEL", - "AuditLogRepository", "UserAgentRepository", - "TeamRepository", "InviteCodeRepository", - "RoleRepository", - "ChannelConnectionRepository", - "SiteRepository", - "OntologyRepository", + name for name in globals() if name.endswith("Repository") or name == "ROOT_FOLDER_SENTINEL" ] diff --git a/src/backend/core/db/repository/agent.py b/src/backend/core/db/repository/agent.py index 2ffbe2ce..ab65a7fa 100644 --- a/src/backend/core/db/repository/agent.py +++ b/src/backend/core/db/repository/agent.py @@ -1,39 +1,14 @@ -"""Data access layer — user agent repositories. +"""Community repository for personal and administrator sub-agents.""" -Split out of the former monolithic ``core/db/repository.py``. The package -``__init__`` re-exports every repository class, so ``from core.db.repository -import XxxRepository`` keeps working unchanged. -""" - -import logging from datetime import datetime from typing import Any, Dict, List, Optional -import sqlalchemy as sa -from core.config.settings import settings -from core.db.models import ( - Artifact, - AuditLog, - CatalogOverride, - ChatMessage, - ChatSession, - InviteCode, - KBDocument, - KBSpace, - LocalUser, - Team, - TeamFolder, - TeamMember, - UserAgent, - UserShadow, -) -from sqlalchemy import and_, desc, func, or_, select +from core.db.models import UserAgent +from sqlalchemy import and_, func, or_ from sqlalchemy.orm import Session class UserAgentRepository: - """Repository for user agent (sub-agent) operations.""" - def __init__(self, db: Session): self.db = db @@ -41,67 +16,33 @@ def get_by_id(self, agent_id: str) -> Optional[UserAgent]: return self.db.query(UserAgent).filter(UserAgent.agent_id == agent_id).first() def list_for_user(self, user_id: str) -> List[UserAgent]: - """Return all agents visible to a user: enabled admin agents + user's own agents - + team agents — every member sees *enabled* team agents; team owner/admin also - see *disabled* ones (so they can re-enable them; there is no separate admin view).""" - visibility_filters = [ - and_(UserAgent.owner_type == "admin", UserAgent.is_enabled == True), - and_(UserAgent.owner_type == "user", UserAgent.user_id == user_id), - ] - if settings.edition.is_ee: - member_team_ids = self.db.query(TeamMember.team_id).filter( - TeamMember.user_id == user_id - ) - manager_team_ids = self.db.query(TeamMember.team_id).filter( - TeamMember.user_id == user_id, - TeamMember.role.in_(("owner", "admin")), - ) - visibility_filters.extend( - [ - and_( - UserAgent.owner_type == "team", - UserAgent.is_enabled == True, - UserAgent.team_id.in_(member_team_ids), - ), - and_( - UserAgent.owner_type == "team", - UserAgent.team_id.in_(manager_team_ids), - ), - ] - ) return ( self.db.query(UserAgent) - .filter(or_(*visibility_filters)) + .filter( + or_( + and_(UserAgent.owner_type == "admin", UserAgent.is_enabled.is_(True)), + and_(UserAgent.owner_type == "user", UserAgent.user_id == user_id), + ) + ) .order_by(UserAgent.owner_type.desc(), UserAgent.sort_order, UserAgent.created_at) .all() ) def list_admin(self) -> List[UserAgent]: - """Return all admin-owned agents.""" - return self.db.query(UserAgent).filter( - UserAgent.owner_type == "admin" - ).order_by(UserAgent.sort_order, UserAgent.created_at).all() + return ( + self.db.query(UserAgent) + .filter(UserAgent.owner_type == "admin") + .order_by(UserAgent.sort_order, UserAgent.created_at) + .all() + ) def count_user_agents(self, user_id: str) -> int: - """Count agents owned by a specific user.""" - return self.db.query(func.count(UserAgent.agent_id)).filter( - UserAgent.owner_type == "user", - UserAgent.user_id == user_id, - ).scalar() or 0 - - def count_team_agents(self, team_id: str) -> int: - """Count agents belonging to a specific team.""" - return self.db.query(func.count(UserAgent.agent_id)).filter( - UserAgent.owner_type == "team", - UserAgent.team_id == team_id, - ).scalar() or 0 - - def list_for_team(self, team_id: str) -> List[UserAgent]: - """Return all agents of a team (enabled + disabled), for managers.""" - return self.db.query(UserAgent).filter( - UserAgent.owner_type == "team", - UserAgent.team_id == team_id, - ).order_by(UserAgent.sort_order, UserAgent.created_at).all() + return ( + self.db.query(func.count(UserAgent.agent_id)) + .filter(UserAgent.owner_type == "user", UserAgent.user_id == user_id) + .scalar() + or 0 + ) def create(self, data: Dict[str, Any]) -> UserAgent: agent = UserAgent(**data) @@ -128,3 +69,6 @@ def delete(self, agent_id: str) -> bool: self.db.delete(agent) self.db.commit() return True + + +__all__ = ["UserAgentRepository"] diff --git a/src/backend/core/db/repository/artifact.py b/src/backend/core/db/repository/artifact.py index 41440d19..30f2b16d 100644 --- a/src/backend/core/db/repository/artifact.py +++ b/src/backend/core/db/repository/artifact.py @@ -6,17 +6,12 @@ """ import logging -from typing import Optional, List, Dict, Any from datetime import datetime -import sqlalchemy as sa -from sqlalchemy.orm import Session -from sqlalchemy import and_, or_, desc, func, select -from core.db.models import ( - UserShadow, ChatSession, ChatMessage, CatalogOverride, - KBSpace, KBDocument, Artifact, AuditLog, UserAgent, - LocalUser, Team, TeamMember, TeamFolder, InviteCode, -) +from typing import Any, Dict, List, Optional +from core.db.models import Artifact, ChatSession +from sqlalchemy import desc, func, or_ +from sqlalchemy.orm import Session #: Frontend counterpart constant: src/frontend/src/utils/constants.ts:ROOT_FOLDER_SENTINEL. ROOT_FOLDER_SENTINEL = "__root__" @@ -30,31 +25,30 @@ def __init__(self, db: Session): def get_by_id(self, artifact_id: str) -> Optional[Artifact]: """Get artifact by ID.""" - return self.db.query(Artifact).filter( - Artifact.artifact_id == artifact_id, - Artifact.deleted_at.is_(None) - ).first() + return ( + self.db.query(Artifact) + .filter(Artifact.artifact_id == artifact_id, Artifact.deleted_at.is_(None)) + .first() + ) def list_by_user( - self, - user_id: str, - artifact_type: Optional[str] = None, - page: int = 1, - page_size: int = 20 + self, user_id: str, artifact_type: Optional[str] = None, page: int = 1, page_size: int = 20 ) -> tuple[List[Artifact], int]: """List artifacts for a user.""" query = self.db.query(Artifact).filter( - Artifact.user_id == user_id, - Artifact.deleted_at.is_(None) + Artifact.user_id == user_id, Artifact.deleted_at.is_(None) ) if artifact_type: query = query.filter(Artifact.type == artifact_type) total = query.count() - artifacts = query.order_by(desc(Artifact.created_at)).offset( - (page - 1) * page_size - ).limit(page_size).all() + artifacts = ( + query.order_by(desc(Artifact.created_at)) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) return artifacts, total @@ -76,21 +70,28 @@ def list_by_user_with_chat( keyword: fuzzy match on filename or title. source_kind: "user_upload" | "ai_generated"; filters on ``extra_data.source`` using a dialect-aware JSON accessor. - personal_only: when True (default) return only personal files with team_id NULL. + personal_only: when True (default), apply the edition's personal-file filter. folder_id: only takes effect when personal_only=True. "__root__" → root directory only (user_folder_id IS NULL); "" → direct child files of that folder; None → no folder filtering (legacy behavior, returns all personal files). """ - query = self.db.query(Artifact, ChatSession.title.label("chat_title")).outerjoin( - ChatSession, Artifact.chat_id == ChatSession.chat_id - ).filter( - Artifact.user_id == user_id, - Artifact.deleted_at.is_(None), + query = ( + self.db.query(Artifact, ChatSession.title.label("chat_title")) + .outerjoin(ChatSession, Artifact.chat_id == ChatSession.chat_id) + .filter( + Artifact.user_id == user_id, + Artifact.deleted_at.is_(None), + ) ) if personal_only: - query = query.filter(Artifact.team_id.is_(None)) + # Import lazily: ``core.services`` re-exports repositories, so importing + # this edition seam while the repository package is initializing would + # create a package-level cycle. + from core.services.artifact_edition import personal_artifact_predicates + + query = query.filter(*personal_artifact_predicates(Artifact)) if folder_id == ROOT_FOLDER_SENTINEL: query = query.filter(Artifact.user_folder_id.is_(None)) elif folder_id: @@ -115,9 +116,7 @@ def list_by_user_with_chat( else: # ai_generated = anything that is NOT explicitly user_upload, # including NULL / missing source metadata (e.g. backfill). - query = query.filter( - or_(json_source.is_(None), json_source != "user_upload") - ) + query = query.filter(or_(json_source.is_(None), json_source != "user_upload")) if keyword: like_pattern = f"%{keyword}%" @@ -129,25 +128,34 @@ def list_by_user_with_chat( ) total = query.count() - rows = query.order_by(desc(Artifact.created_at)).offset( - (page - 1) * page_size - ).limit(page_size).all() + rows = ( + query.order_by(desc(Artifact.created_at)) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) items = [] for artifact, chat_title in rows: - items.append({ - "artifact": artifact, - "chat_title": chat_title, - }) + items.append( + { + "artifact": artifact, + "chat_title": chat_title, + } + ) return items, total def soft_delete(self, artifact_id: str, user_id: str) -> bool: """Soft delete a personal MySpace artifact (set deleted_at).""" - artifact = self.db.query(Artifact).filter( - Artifact.artifact_id == artifact_id, - Artifact.user_id == user_id, - Artifact.deleted_at.is_(None), - ).first() + artifact = ( + self.db.query(Artifact) + .filter( + Artifact.artifact_id == artifact_id, + Artifact.user_id == user_id, + Artifact.deleted_at.is_(None), + ) + .first() + ) if not artifact: return False artifact.deleted_at = datetime.utcnow() @@ -161,100 +169,3 @@ def create(self, artifact_data: Dict[str, Any]) -> Artifact: self.db.commit() self.db.refresh(artifact) return artifact - - # ── Team-scoped queries ────────────────────────────────────────── - - def list_by_team_folder( - self, - team_id: str, - folder_id: Optional[str], - *, - mime_prefix: Optional[str] = None, - keyword: Optional[str] = None, - page: int = 1, - page_size: int = 20, - ) -> tuple[List[Dict[str, Any]], int]: - """List files under a team folder (folder_id=None means the team root).""" - query = self.db.query(Artifact, ChatSession.title.label("chat_title")).outerjoin( - ChatSession, Artifact.chat_id == ChatSession.chat_id - ).filter( - Artifact.team_id == team_id, - Artifact.deleted_at.is_(None), - ) - - if folder_id is None: - query = query.filter(Artifact.team_folder_id.is_(None)) - else: - query = query.filter(Artifact.team_folder_id == folder_id) - - if mime_prefix == "image/": - query = query.filter(Artifact.mime_type.like("image/%")) - elif mime_prefix == "document": - query = query.filter(~Artifact.mime_type.like("image/%")) - - if keyword: - pattern = f"%{keyword}%" - query = query.filter( - or_(Artifact.filename.ilike(pattern), Artifact.title.ilike(pattern)) - ) - - total = query.count() - rows = ( - query.order_by(desc(Artifact.created_at)) - .offset((page - 1) * page_size) - .limit(page_size) - .all() - ) - items: List[Dict[str, Any]] = [] - for artifact, chat_title in rows: - items.append({"artifact": artifact, "chat_title": chat_title}) - return items, total - - def move_artifact_to_team( - self, - artifact_id: str, - *, - team_id: str, - folder_id: Optional[str], - new_storage_key: Optional[str] = None, - new_storage_url: Optional[str] = None, - expected_uploader: Optional[str] = None, - ) -> Optional[Artifact]: - """Move a personal (or in-team) file into the specified team folder.""" - query = self.db.query(Artifact).filter( - Artifact.artifact_id == artifact_id, - Artifact.deleted_at.is_(None), - ) - if expected_uploader is not None: - query = query.filter(Artifact.user_id == expected_uploader) - artifact = query.first() - if artifact is None: - return None - artifact.team_id = team_id - artifact.team_folder_id = folder_id - if new_storage_key: - artifact.storage_key = new_storage_key - if new_storage_url is not None: - artifact.storage_url = new_storage_url - artifact.updated_at = datetime.utcnow() - self.db.commit() - self.db.refresh(artifact) - return artifact - - def soft_delete_team_file( - self, artifact_id: str, *, team_id: str, actor_user_id: Optional[str] = None - ) -> bool: - """Soft-delete a team file; if actor_user_id is given, require it to have been uploaded by that user.""" - query = self.db.query(Artifact).filter( - Artifact.artifact_id == artifact_id, - Artifact.team_id == team_id, - Artifact.deleted_at.is_(None), - ) - if actor_user_id is not None: - query = query.filter(Artifact.user_id == actor_user_id) - artifact = query.first() - if not artifact: - return False - artifact.deleted_at = datetime.utcnow() - self.db.commit() - return True diff --git a/src/backend/core/db/repository/audit.py b/src/backend/core/db/repository/audit.py index ee28825b..98018da9 100644 --- a/src/backend/core/db/repository/audit.py +++ b/src/backend/core/db/repository/audit.py @@ -1,315 +1,44 @@ -"""Data access layer — audit log repositories. +"""No-storage audit repository for the community edition.""" -Split out of the former monolithic ``core/db/repository.py``. The package -``__init__`` re-exports every repository class, so ``from core.db.repository -import XxxRepository`` keeps working unchanged. -""" - -import logging -from typing import Optional, List, Dict, Any -from datetime import datetime -import sqlalchemy as sa -from sqlalchemy.orm import Session -from sqlalchemy import and_, or_, desc, func, select -from core.db.models import ( - UserShadow, ChatSession, ChatMessage, CatalogOverride, - KBSpace, KBDocument, Artifact, AuditLog, UserAgent, - LocalUser, Team, TeamMember, TeamFolder, InviteCode, -) +from typing import Any, Dict, List, Optional class AuditLogRepository: - """Repository for audit log operations.""" + """Preserve call signatures while keeping governance storage out of CE.""" - def __init__(self, db: Session): + def __init__(self, db): self.db = db - def create(self, log_data: Dict[str, Any]) -> AuditLog: - """Create a new audit log entry.""" - log = AuditLog(**log_data) - self.db.add(log) - try: - self.db.commit() - self.db.refresh(log) - except Exception: - # Audit should not block the main business flow in local/dev setups. - self.db.rollback() - logging.getLogger(__name__).debug( - "audit log write failed for action=%s", log_data.get("action"), exc_info=True - ) - return log - - def log_denial( - self, - *, - user_id: Optional[str], - action: str, - reason: str, - required: Optional[str] = None, - actual: Optional[str] = None, - resource_type: Optional[str] = None, - resource_id: Optional[str] = None, - request: Any = None, - extra: Optional[Dict[str, Any]] = None, - ) -> AuditLog: - """Record a permission-denied event, storing required vs actual and the request context.""" - details: Dict[str, Any] = {"reason": reason} - if required is not None: - details["required"] = required - if actual is not None: - details["actual"] = actual - if extra: - details.update(extra) - - ip_address: Optional[str] = None - user_agent: Optional[str] = None - if request is not None: - try: - client = getattr(request, "client", None) - if client is not None: - ip_address = client.host - headers = getattr(request, "headers", None) - if headers is not None: - user_agent = headers.get("user-agent") - except Exception: - pass - # Middleware already pushed trace_id into the ContextVar; request.state has no such field. - from core.infra.logging import trace_id_var - trace_id: Optional[str] = trace_id_var.get() or None - - return self.create({ - "user_id": user_id, - "action": action, - "resource_type": resource_type, - "resource_id": resource_id, - "details": details, - "ip_address": ip_address, - "user_agent": user_agent, - "trace_id": trace_id, - "status": "failure", - "error_code": 403, - }) - - def list_by_user( - self, - user_id: str, - action: Optional[str] = None, - page: int = 1, - page_size: int = 50 - ) -> tuple[List[AuditLog], int]: - """List audit logs for a user.""" - query = self.db.query(AuditLog).filter( - AuditLog.user_id == user_id - ) - - if action: - query = query.filter(AuditLog.action == action) - - total = query.count() - logs = query.order_by(desc(AuditLog.created_at)).offset( - (page - 1) * page_size - ).limit(page_size).all() - - return logs, total + def create(self, log_data: Dict[str, Any]): + return None - def get_by_id(self, log_id: int) -> Optional[AuditLog]: - """Get audit log by ID.""" - return self.db.query(AuditLog).filter(AuditLog.log_id == log_id).first() + def log_denial(self, **kwargs): + return None - def list_with_filters( - self, - user_id: str, - action: Optional[str] = None, - resource_type: Optional[str] = None, - start_date: Optional[datetime] = None, - end_date: Optional[datetime] = None, - page: int = 1, - page_size: int = 50 - ) -> tuple[List[AuditLog], int]: - """List audit logs with multiple filters.""" - query = self.db.query(AuditLog).filter(AuditLog.user_id == user_id) + def list_by_user(self, *args, **kwargs) -> tuple[List[Any], int]: + return [], 0 - if action: - query = query.filter(AuditLog.action == action) - if resource_type: - query = query.filter(AuditLog.resource_type == resource_type) - if start_date: - query = query.filter(AuditLog.created_at >= start_date) - if end_date: - query = query.filter(AuditLog.created_at <= end_date) + def get_by_id(self, log_id: int): + return None - total = query.count() - logs = query.order_by(desc(AuditLog.created_at)).offset( - (page - 1) * page_size - ).limit(page_size).all() + def list_with_filters(self, *args, **kwargs) -> tuple[List[Any], int]: + return [], 0 - return logs, total - - def list_all( - self, - user_id: Optional[str] = None, - user_ids: Optional[List[str]] = None, - action: Optional[str] = None, - resource_type: Optional[str] = None, - status: Optional[str] = None, - sandbox_id: Optional[str] = None, - start_date: Optional[datetime] = None, - end_date: Optional[datetime] = None, - page: int = 1, - page_size: int = 50, - ) -> tuple[List[AuditLog], int]: - """Global audit-log query (for the security admin console, not restricted to a user_id). - - Difference from ``list_with_filters``: user_id is an optional filter rather than a mandatory - constraint, and it supports filtering by status / sandbox_id, used to investigate login - failures, permission denials, and tracing by sandbox instance. - - ``user_ids`` is used for "filter by username/name keyword" — the route layer first resolves - the keyword into a batch of user_ids and passes them in; an empty list means the keyword - matched nothing and should return an empty result (not the whole table). - """ - query = self.db.query(AuditLog) - if user_ids is not None: - if not user_ids: - return [], 0 - query = query.filter(AuditLog.user_id.in_(user_ids)) - if user_id: - query = query.filter(AuditLog.user_id == user_id) - if action: - query = query.filter(AuditLog.action == action) - if resource_type: - query = query.filter(AuditLog.resource_type == resource_type) - if status: - query = query.filter(AuditLog.status == status) - if sandbox_id: - query = query.filter(AuditLog.sandbox_id == sandbox_id) - if start_date: - query = query.filter(AuditLog.created_at >= start_date) - if end_date: - query = query.filter(AuditLog.created_at <= end_date) - - total = query.count() - logs = ( - query.order_by(desc(AuditLog.created_at)) - .offset((page - 1) * page_size) - .limit(page_size) - .all() - ) - return logs, total + def list_all(self, *args, **kwargs) -> tuple[List[Any], int]: + return [], 0 def distinct_filter_values(self) -> Dict[str, List[str]]: - """For frontend dropdowns: deduplicated action / resource_type / sandbox_id lists.""" - actions = [ - r[0] - for r in self.db.query(AuditLog.action).distinct().all() - if r[0] - ] - resource_types = [ - r[0] - for r in self.db.query(AuditLog.resource_type).distinct().all() - if r[0] - ] - sandbox_ids = [ - r[0] - for r in self.db.query(AuditLog.sandbox_id) - .filter(AuditLog.sandbox_id.isnot(None)) - .distinct() - .all() - if r[0] - ] - return { - "actions": sorted(actions), - "resource_types": sorted(resource_types), - "sandbox_ids": sorted(sandbox_ids), - } + return {"actions": [], "resource_types": [], "sandbox_ids": []} def get_global_stats(self, days: int = 7) -> Dict[str, Any]: - """Global audit overview (last `days` days): total / failures / permission denials / login failures / top actions.""" - from datetime import timedelta - - start_date = datetime.utcnow() - timedelta(days=days) - base = self.db.query(AuditLog).filter(AuditLog.created_at >= start_date) - - total = base.count() - # status is constrained to success/failure/error; failure = not success - failed = base.filter(AuditLog.status != "success").count() - # Permission denial: denial writes error_code=403 - denied = base.filter(AuditLog.error_code == 403).count() - # Login failure: action shaped like auth.*.failed / auth.login.failed - login_failed = base.filter( - AuditLog.action.like("auth.%"), - AuditLog.status != "success", - ).count() - - action_groups = ( - self.db.query(AuditLog.action, func.count(AuditLog.log_id)) - .filter(AuditLog.created_at >= start_date) - .group_by(AuditLog.action) - .order_by(desc(func.count(AuditLog.log_id))) - .limit(10) - .all() - ) - top_actions = [{"action": a, "count": c} for a, c in action_groups] - return { "period_days": days, - "total_actions": total, - "failed_actions": failed, - "denied_actions": denied, - "login_failed": login_failed, - "top_actions": top_actions, + "total_actions": 0, + "failed_actions": 0, + "denied_actions": 0, + "login_failed": 0, + "top_actions": [], } def get_user_stats(self, user_id: str, days: int = 7) -> Dict[str, Any]: - """Get audit statistics for a user.""" - from datetime import timedelta - - start_date = datetime.utcnow() - timedelta(days=days) - - query = self.db.query(AuditLog).filter( - AuditLog.user_id == user_id, - AuditLog.created_at >= start_date - ) - - # Total actions - total = query.count() - - # Failed actions - failed = query.filter(AuditLog.status == "failed").count() - - # Actions by type - actions_by_type = {} - action_groups = self.db.query( - AuditLog.action, func.count(AuditLog.log_id) - ).filter( - AuditLog.user_id == user_id, - AuditLog.created_at >= start_date - ).group_by(AuditLog.action).all() - - for action, count in action_groups: - actions_by_type[action] = count - - # Most active day - daily_counts = self.db.query( - func.date(AuditLog.created_at).label('date'), - func.count(AuditLog.log_id).label('count') - ).filter( - AuditLog.user_id == user_id, - AuditLog.created_at >= start_date - ).group_by(func.date(AuditLog.created_at)).order_by(desc('count')).first() - - most_active_day = None - if daily_counts: - most_active_day = { - 'date': daily_counts.date.isoformat() if hasattr(daily_counts.date, 'isoformat') else str(daily_counts.date), - 'count': daily_counts.count - } - - return { - 'period_days': days, - 'total_actions': total, - 'failed_actions': failed, - 'success_rate': round((total - failed) / total * 100, 2) if total > 0 else 0, - 'actions_by_type': actions_by_type, - 'most_active_day': most_active_day - } + return {"period_days": days, "total_actions": 0, "failed_actions": 0} diff --git a/src/backend/core/db/repository/catalog.py b/src/backend/core/db/repository/catalog.py index e6658835..3c68e2ad 100644 --- a/src/backend/core/db/repository/catalog.py +++ b/src/backend/core/db/repository/catalog.py @@ -6,16 +6,13 @@ """ import logging -from typing import Optional, List, Dict, Any from datetime import datetime +from typing import Any, Dict, List, Optional + import sqlalchemy as sa +from core.db.models import CatalogOverride +from sqlalchemy import and_, desc, func, or_, select from sqlalchemy.orm import Session -from sqlalchemy import and_, or_, desc, func, select -from core.db.models import ( - UserShadow, ChatSession, ChatMessage, CatalogOverride, - KBSpace, KBDocument, Artifact, AuditLog, UserAgent, - LocalUser, Team, TeamMember, TeamFolder, InviteCode, -) class CatalogRepository: @@ -26,24 +23,28 @@ def __init__(self, db: Session): def get_override(self, user_id: str, kind: str, item_id: str) -> Optional[CatalogOverride]: """Get catalog override for a specific item.""" - return self.db.query(CatalogOverride).filter( - CatalogOverride.user_id == user_id, - CatalogOverride.kind == kind, - CatalogOverride.item_id == item_id - ).first() + return ( + self.db.query(CatalogOverride) + .filter( + CatalogOverride.user_id == user_id, + CatalogOverride.kind == kind, + CatalogOverride.item_id == item_id, + ) + .first() + ) def list_overrides(self, user_id: str, kind: Optional[str] = None) -> List[CatalogOverride]: """List all catalog overrides for a user.""" - query = self.db.query(CatalogOverride).filter( - CatalogOverride.user_id == user_id - ) + query = self.db.query(CatalogOverride).filter(CatalogOverride.user_id == user_id) if kind: query = query.filter(CatalogOverride.kind == kind) return query.all() - def upsert_override(self, user_id: str, kind: str, item_id: str, enabled: bool, config: Dict = None) -> CatalogOverride: + def upsert_override( + self, user_id: str, kind: str, item_id: str, enabled: bool, config: Dict = None + ) -> CatalogOverride: """Create or update catalog override.""" override = self.get_override(user_id, kind, item_id) @@ -58,7 +59,7 @@ def upsert_override(self, user_id: str, kind: str, item_id: str, enabled: bool, kind=kind, item_id=item_id, enabled=enabled, - config_data=config or {} + config_data=config or {}, ) self.db.add(override) diff --git a/src/backend/core/db/repository/chat.py b/src/backend/core/db/repository/chat.py index c89a8739..621bb110 100644 --- a/src/backend/core/db/repository/chat.py +++ b/src/backend/core/db/repository/chat.py @@ -6,16 +6,13 @@ """ import logging -from typing import Optional, List, Dict, Any from datetime import datetime +from typing import Any, Dict, List, Optional + import sqlalchemy as sa +from core.db.models import ChatMessage, ChatSession +from sqlalchemy import and_, desc, func, or_, select from sqlalchemy.orm import Session -from sqlalchemy import and_, or_, desc, func, select -from core.db.models import ( - UserShadow, ChatSession, ChatMessage, CatalogOverride, - KBSpace, KBDocument, Artifact, AuditLog, UserAgent, - LocalUser, Team, TeamMember, TeamFolder, InviteCode, -) class ChatSessionRepository: @@ -26,10 +23,11 @@ def __init__(self, db: Session): def get_by_id(self, chat_id: str) -> Optional[ChatSession]: """Get chat session by ID.""" - return self.db.query(ChatSession).filter( - ChatSession.chat_id == chat_id, - ChatSession.deleted_at.is_(None) - ).first() + return ( + self.db.query(ChatSession) + .filter(ChatSession.chat_id == chat_id, ChatSession.deleted_at.is_(None)) + .first() + ) def list_by_user( self, @@ -42,8 +40,7 @@ def list_by_user( ) -> tuple[List[ChatSession], int]: """List chat sessions for a user with pagination.""" query = self.db.query(ChatSession).filter( - ChatSession.user_id == user_id, - ChatSession.deleted_at.is_(None) + ChatSession.user_id == user_id, ChatSession.deleted_at.is_(None) ) if pinned_only: @@ -65,9 +62,12 @@ def list_by_user( total = query.count() # Apply pagination and ordering - sessions = query.order_by(desc(ChatSession.updated_at)).offset( - (page - 1) * page_size - ).limit(page_size).all() + sessions = ( + query.order_by(desc(ChatSession.updated_at)) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) return sessions, total @@ -158,18 +158,26 @@ def search( # Fetch title-matched sessions first, then content-matched sessions title_sessions = ( - self.db.query(ChatSession) - .filter(ChatSession.chat_id.in_(title_id_set)) - .order_by(desc(ChatSession.updated_at)) - .all() - ) if title_id_set else [] + ( + self.db.query(ChatSession) + .filter(ChatSession.chat_id.in_(title_id_set)) + .order_by(desc(ChatSession.updated_at)) + .all() + ) + if title_id_set + else [] + ) content_sessions = ( - self.db.query(ChatSession) - .filter(ChatSession.chat_id.in_(content_id_set)) - .order_by(desc(ChatSession.updated_at)) - .all() - ) if content_id_set else [] + ( + self.db.query(ChatSession) + .filter(ChatSession.chat_id.in_(content_id_set)) + .order_by(desc(ChatSession.updated_at)) + .all() + ) + if content_id_set + else [] + ) # Merge: title matches first, then content matches ordered = title_sessions + content_sessions @@ -213,11 +221,13 @@ def search( snippet = snippet + "..." matched_snippet = snippet - results.append({ - "session": s, - "match_type": match_type, - "matched_snippet": matched_snippet, - }) + results.append( + { + "session": s, + "match_type": match_type, + "matched_snippet": matched_snippet, + } + ) return results, total @@ -230,15 +240,10 @@ def __init__(self, db: Session): def get_by_id(self, message_id: str) -> Optional[ChatMessage]: """Get message by ID.""" - return self.db.query(ChatMessage).filter( - ChatMessage.message_id == message_id - ).first() + return self.db.query(ChatMessage).filter(ChatMessage.message_id == message_id).first() def list_by_chat( - self, - chat_id: str, - page: int = 1, - page_size: int = 50 + self, chat_id: str, page: int = 1, page_size: int = 50 ) -> tuple[List[ChatMessage], int]: """List messages for a chat session with pagination. @@ -251,9 +256,12 @@ def list_by_chat( ) total = query.count() - messages = query.order_by(ChatMessage.created_at).offset( - (page - 1) * page_size - ).limit(page_size).all() + messages = ( + query.order_by(ChatMessage.created_at) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) return messages, total @@ -293,4 +301,3 @@ def update_extra_data(self, message_id: str, patch: Dict[str, Any]) -> Optional[ self.db.commit() self.db.refresh(message) return message - diff --git a/src/backend/core/db/repository/kb.py b/src/backend/core/db/repository/kb.py index 989bce9f..2537e194 100644 --- a/src/backend/core/db/repository/kb.py +++ b/src/backend/core/db/repository/kb.py @@ -6,17 +6,13 @@ """ import logging -from typing import Optional, List, Dict, Any from datetime import datetime +from typing import Any, Dict, List, Optional + import sqlalchemy as sa +from core.db.models import KBDocument, KBSpace +from sqlalchemy import and_, desc, func, or_, select from sqlalchemy.orm import Session -from sqlalchemy import and_, or_, desc, func, select -from core.db.models import ( - UserShadow, ChatSession, ChatMessage, CatalogOverride, - KBSpace, KBDocument, Artifact, AuditLog, UserAgent, - LocalUser, Team, TeamMember, TeamFolder, InviteCode, - KBGrant, -) class KBRepository: @@ -27,17 +23,19 @@ def __init__(self, db: Session): def get_space(self, kb_id: str) -> Optional[KBSpace]: """Get KB space by ID.""" - return self.db.query(KBSpace).filter( - KBSpace.kb_id == kb_id, - KBSpace.deleted_at.is_(None) - ).first() + return ( + self.db.query(KBSpace) + .filter(KBSpace.kb_id == kb_id, KBSpace.deleted_at.is_(None)) + .first() + ) def list_spaces(self, user_id: str) -> List[KBSpace]: """List all KB spaces for a user.""" - return self.db.query(KBSpace).filter( - KBSpace.user_id == user_id, - KBSpace.deleted_at.is_(None) - ).all() + return ( + self.db.query(KBSpace) + .filter(KBSpace.user_id == user_id, KBSpace.deleted_at.is_(None)) + .all() + ) def list_public_spaces(self) -> List[KBSpace]: """List all public KB spaces (visibility == 'public'), regardless of owner. @@ -45,36 +43,46 @@ def list_public_spaces(self) -> List[KBSpace]: Public spaces are admin-managed shared knowledge bases that every user can see in the catalog and retrieve from. Ordered by creation time (oldest first). """ - return self.db.query(KBSpace).filter( - KBSpace.visibility == "public", - KBSpace.deleted_at.is_(None) - ).order_by(KBSpace.created_at).all() + return ( + self.db.query(KBSpace) + .filter(KBSpace.visibility == "public", KBSpace.deleted_at.is_(None)) + .order_by(KBSpace.created_at) + .all() + ) def get_public_space(self, kb_id: str) -> Optional[KBSpace]: """Get a public KB space by ID (visibility == 'public', not deleted).""" - return self.db.query(KBSpace).filter( - KBSpace.kb_id == kb_id, - KBSpace.visibility == "public", - KBSpace.deleted_at.is_(None) - ).first() + return ( + self.db.query(KBSpace) + .filter( + KBSpace.kb_id == kb_id, KBSpace.visibility == "public", KBSpace.deleted_at.is_(None) + ) + .first() + ) def list_shared_spaces(self) -> List[KBSpace]: """List admin-managed shared KB spaces (visibility public or scoped). The admin console needs to manage both "public to everyone" and "designated-visibility" shared bases, so scoped is included. """ - return self.db.query(KBSpace).filter( - KBSpace.visibility.in_(("public", "scoped")), - KBSpace.deleted_at.is_(None) - ).order_by(KBSpace.created_at).all() + return ( + self.db.query(KBSpace) + .filter(KBSpace.visibility.in_(("public", "scoped")), KBSpace.deleted_at.is_(None)) + .order_by(KBSpace.created_at) + .all() + ) def get_shared_space(self, kb_id: str) -> Optional[KBSpace]: """Get a shared KB space (visibility public or scoped, not deleted).""" - return self.db.query(KBSpace).filter( - KBSpace.kb_id == kb_id, - KBSpace.visibility.in_(("public", "scoped")), - KBSpace.deleted_at.is_(None) - ).first() + return ( + self.db.query(KBSpace) + .filter( + KBSpace.kb_id == kb_id, + KBSpace.visibility.in_(("public", "scoped")), + KBSpace.deleted_at.is_(None), + ) + .first() + ) def create_space(self, space_data: Dict[str, Any]) -> KBSpace: """Create a new KB space.""" @@ -100,27 +108,27 @@ def update_space(self, kb_id: str, update_data: Dict[str, Any]) -> Optional[KBSp def get_document(self, document_id: str) -> Optional[KBDocument]: """Get KB document by ID.""" - return self.db.query(KBDocument).filter( - KBDocument.document_id == document_id, - KBDocument.deleted_at.is_(None) - ).first() + return ( + self.db.query(KBDocument) + .filter(KBDocument.document_id == document_id, KBDocument.deleted_at.is_(None)) + .first() + ) def list_documents( - self, - kb_id: str, - page: int = 1, - page_size: int = 20 + self, kb_id: str, page: int = 1, page_size: int = 20 ) -> tuple[List[KBDocument], int]: """List documents in a KB space.""" query = self.db.query(KBDocument).filter( - KBDocument.kb_id == kb_id, - KBDocument.deleted_at.is_(None) + KBDocument.kb_id == kb_id, KBDocument.deleted_at.is_(None) ) total = query.count() - documents = query.order_by(desc(KBDocument.uploaded_at)).offset( - (page - 1) * page_size - ).limit(page_size).all() + documents = ( + query.order_by(desc(KBDocument.uploaded_at)) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) return documents, total @@ -133,110 +141,4 @@ def create_document(self, document_data: Dict[str, Any]) -> KBDocument: return document -class KBGrantRepository: - """Data-access layer for knowledge-base grants + external-base visibility.""" - - def __init__(self, db: Session): - self.db = db - - # ── grants (common to local bases / Dify datasets) ────────────────────────────── - def list_for_principal(self, principal_type: str, principal_id: str) -> List[KBGrant]: - """Which resources a given user/team has been granted.""" - return self.db.query(KBGrant).filter( - KBGrant.principal_type == principal_type, - KBGrant.principal_id == principal_id, - ).all() - - def upsert( - self, - resource_id: str, - resource_type: str, - principal_type: str, - principal_id: str, - level: str, - granted_by: Optional[str] = None, - ) -> KBGrant: - grant = self.db.query(KBGrant).filter( - KBGrant.resource_id == resource_id, - KBGrant.resource_type == resource_type, - KBGrant.principal_type == principal_type, - KBGrant.principal_id == principal_id, - ).first() - if grant: - grant.level = level - if granted_by: - grant.granted_by = granted_by - else: - grant = KBGrant( - resource_id=resource_id, - resource_type=resource_type, - principal_type=principal_type, - principal_id=principal_id, - level=level, - granted_by=granted_by, - ) - self.db.add(grant) - self.db.commit() - self.db.refresh(grant) - return grant - - def replace_for_principal( - self, - principal_type: str, - principal_id: str, - grants: List[Dict[str, str]], - granted_by: Optional[str] = None, - ) -> int: - """Fully replace all grants of a given principal ("save" semantics of the user/team management page). - - Each ``grants`` item is like ``{resource_id, resource_type, level}``. Returns the number of rows written. - """ - self.db.query(KBGrant).filter( - KBGrant.principal_type == principal_type, - KBGrant.principal_id == principal_id, - ).delete() - count = 0 - for g in grants: - resource_id = str(g.get("resource_id") or "").strip() - resource_type = str(g.get("resource_type") or "local").strip() - level = str(g.get("level") or "view").strip() - if not resource_id or resource_type not in ("local", "dify") or level not in ("view", "edit", "admin"): - continue - self.db.add(KBGrant( - resource_id=resource_id, - resource_type=resource_type, - principal_type=principal_type, - principal_id=principal_id, - level=level, - granted_by=granted_by, - )) - count += 1 - self.db.commit() - return count - - def bulk_grant( - self, - resource_id: str, - resource_type: str, - principals: List[tuple[str, str]], - level: str, - granted_by: Optional[str] = None, - ) -> int: - """Add multiple grants to a resource at once (only for a just-created base, whose rows are guaranteed brand new). One commit. - - Each ``principals`` item is ``(principal_type, principal_id)``. Returns the number of rows written. - """ - for principal_type, principal_id in principals: - self.db.add(KBGrant( - resource_id=resource_id, - resource_type=resource_type, - principal_type=principal_type, - principal_id=principal_id, - level=level, - granted_by=granted_by, - )) - self.db.commit() - return len(principals) - - #: folder_id sentinel for list_by_user_with_chat: root directory only (user_folder_id IS NULL). diff --git a/src/backend/core/db/repository/role.py b/src/backend/core/db/repository/role.py deleted file mode 100644 index 5474cee2..00000000 --- a/src/backend/core/db/repository/role.py +++ /dev/null @@ -1,210 +0,0 @@ -"""Data access layer — roles / role assignments. - -Isomorphic to [[KBGrantRepository]]: ``set_principal_roles`` fully replaces a -principal's roles (the "Save" semantics of the user/team management page). -``principal_type`` is generalized (user/team, with department reserved) to leave a seam -for "later refactoring into an org tree". -""" - -from datetime import datetime -from typing import Any, Dict, List, Optional - -from sqlalchemy.orm import Session - -from core.db.models import Role, RoleAssignment, TeamMember - - -class RoleRepository: - """Roles + role-assignments Repository.""" - - def __init__(self, db: Session): - self.db = db - - # ── Role CRUD ──────────────────────────────────────────────── - def get(self, role_id: str) -> Optional[Role]: - return self.db.query(Role).filter(Role.role_id == role_id).first() - - def get_by_name(self, name: str) -> Optional[Role]: - return self.db.query(Role).filter(Role.name == name).first() - - def list_all(self) -> List[Role]: - return self.db.query(Role).order_by(Role.created_at.desc()).all() - - def list_team_default_role_ids(self) -> List[str]: - """IDs of roles marked as "new-team default" — auto-assigned when creating/syncing a team.""" - return [ - rid - for (rid,) in self.db.query(Role.role_id).filter(Role.is_team_default.is_(True)).all() - ] - - def create(self, data: Dict[str, Any]) -> Role: - role = Role(**data) - self.db.add(role) - self.db.commit() - self.db.refresh(role) - return role - - def update(self, role_id: str, data: Dict[str, Any]) -> Optional[Role]: - role = self.get(role_id) - if not role: - return None - for k, v in data.items(): - setattr(role, k, v) - role.updated_at = datetime.utcnow() - self.db.commit() - self.db.refresh(role) - return role - - def delete(self, role_id: str) -> bool: - role = self.get(role_id) - if not role: - return False - # Explicitly clear assignments (FK ondelete=CASCADE is not enforced by default in SQLite; deleting manually keeps both sides consistent) - self.db.query(RoleAssignment).filter(RoleAssignment.role_id == role_id).delete() - self.db.delete(role) - self.db.commit() - return True - - # ── Assignments ────────────────────────────────────────────── - def list_assignments(self, role_id: str) -> List[RoleAssignment]: - return self.db.query(RoleAssignment).filter(RoleAssignment.role_id == role_id).all() - - def assignment_counts_bulk(self, role_ids: List[str]) -> Dict[str, int]: - """Fetch assignment counts for a batch of roles at once (avoids list_roles N+1).""" - if not role_ids: - return {} - from sqlalchemy import func - - rows = ( - self.db.query(RoleAssignment.role_id, func.count()) - .filter(RoleAssignment.role_id.in_(role_ids)) - .group_by(RoleAssignment.role_id) - .all() - ) - return {rid: cnt for rid, cnt in rows} - - def list_principal_role_ids(self, principal_type: str, principal_id: str) -> List[str]: - """Role IDs directly assigned to a principal (excluding inheritance).""" - return [ - rid - for (rid,) in self.db.query(RoleAssignment.role_id) - .filter( - RoleAssignment.principal_type == principal_type, - RoleAssignment.principal_id == principal_id, - ) - .all() - ] - - def list_roles_for_principal(self, principal_type: str, principal_id: str) -> List[Role]: - """Role objects directly assigned to a principal (excluding inheritance).""" - return ( - self.db.query(Role) - .join(RoleAssignment, Role.role_id == RoleAssignment.role_id) - .filter( - RoleAssignment.principal_type == principal_type, - RoleAssignment.principal_id == principal_id, - ) - .order_by(Role.name) - .all() - ) - - def direct_role_ids_for_users_bulk(self, user_ids: List[str]) -> Dict[str, List[str]]: - """Role IDs **directly** assigned to each user in a batch (principal=user, excluding team inheritance).""" - if not user_ids: - return {} - rows = ( - self.db.query(RoleAssignment.principal_id, RoleAssignment.role_id) - .filter( - RoleAssignment.principal_type == "user", - RoleAssignment.principal_id.in_(user_ids), - ) - .all() - ) - grouped: Dict[str, List[str]] = {uid: [] for uid in user_ids} - for uid, rid in rows: - grouped.setdefault(uid, []).append(rid) - return grouped - - def team_role_ids_bulk(self, team_ids: List[str]) -> Dict[str, List[str]]: - """Role IDs assigned to each team in a batch (department default roles). Used for batch resolution in list_users, to avoid N+1.""" - if not team_ids: - return {} - rows = ( - self.db.query(RoleAssignment.principal_id, RoleAssignment.role_id) - .filter( - RoleAssignment.principal_type == "team", - RoleAssignment.principal_id.in_(team_ids), - ) - .all() - ) - grouped: Dict[str, List[str]] = {} - for tid, rid in rows: - grouped.setdefault(tid, []).append(rid) - return grouped - - def set_principal_roles( - self, principal_type: str, principal_id: str, role_ids: List[str] - ) -> int: - """Fully replace a principal's role assignments (the "Save" semantics of the - user/team management page). Returns the number of rows written. - - Only accepts role_ids that actually exist; writes after deduplication. - """ - self.db.query(RoleAssignment).filter( - RoleAssignment.principal_type == principal_type, - RoleAssignment.principal_id == principal_id, - ).delete() - valid = { - rid - for (rid,) in self.db.query(Role.role_id).filter(Role.role_id.in_(role_ids or [])).all() - } - count = 0 - for rid in dict.fromkeys(role_ids or []): # dedupe while preserving order - if rid not in valid: - continue - self.db.add( - RoleAssignment( - role_id=rid, principal_type=principal_type, principal_id=principal_id - ) - ) - count += 1 - self.db.commit() - return count - - def add_principal_roles( - self, principal_type: str, principal_id: str, role_ids: List[str] - ) -> int: - """Incrementally append role assignments to a principal (existing ones are skipped, none are removed). Returns the number of rows added.""" - if not role_ids: - return 0 - existing = set(self.list_principal_role_ids(principal_type, principal_id)) - valid = { - rid - for (rid,) in self.db.query(Role.role_id).filter(Role.role_id.in_(role_ids)).all() - } - count = 0 - for rid in dict.fromkeys(role_ids): - if rid in existing or rid not in valid: - continue - self.db.add( - RoleAssignment( - role_id=rid, principal_type=principal_type, principal_id=principal_id - ) - ) - count += 1 - if count: - self.db.commit() - return count - - def purge_principal(self, principal_type: str, principal_id: str) -> int: - """Delete all of a principal's role assignments (cleanup when deleting a user/team). Returns the number of rows deleted.""" - n = ( - self.db.query(RoleAssignment) - .filter( - RoleAssignment.principal_type == principal_type, - RoleAssignment.principal_id == principal_id, - ) - .delete() - ) - self.db.commit() - return n diff --git a/src/backend/core/db/repository/team.py b/src/backend/core/db/repository/team.py deleted file mode 100644 index 42377732..00000000 --- a/src/backend/core/db/repository/team.py +++ /dev/null @@ -1,201 +0,0 @@ -"""Data access layer — team repositories. - -Split out of the former monolithic ``core/db/repository.py``. The package -``__init__`` re-exports every repository class, so ``from core.db.repository -import XxxRepository`` keeps working unchanged. -""" - -import logging -from typing import Optional, List, Dict, Any -from datetime import datetime -import sqlalchemy as sa -from sqlalchemy.orm import Session -from sqlalchemy import and_, or_, desc, func, select -from core.db.models import ( - UserShadow, ChatSession, ChatMessage, CatalogOverride, - KBSpace, KBDocument, Artifact, AuditLog, UserAgent, - LocalUser, Team, TeamMember, TeamFolder, InviteCode, -) - - -class TeamRepository: - """Team repository.""" - - def __init__(self, db: Session): - self.db = db - - def get(self, team_id: str) -> Optional[Team]: - return self.db.query(Team).filter(Team.team_id == team_id).first() - - def get_by_name(self, name: str) -> Optional[Team]: - return self.db.query(Team).filter(Team.name == name).first() - - def list_all(self) -> List[Team]: - return self.db.query(Team).order_by(Team.created_at.desc()).all() - - def list_for_user(self, user_id: str) -> List[tuple[Team, str]]: - """Return [(team, role_in_team), ...].""" - rows = ( - self.db.query(Team, TeamMember.role) - .join(TeamMember, Team.team_id == TeamMember.team_id) - .filter(TeamMember.user_id == user_id) - .order_by(Team.name) - .all() - ) - # Return real tuples (Row is not a tuple subclass); the root fix is - # annotation_typing=False in docker/cython_build.py, this conversion is defensive. - # See user.py get_by_username for the rationale. - return [tuple(r) for r in rows] - - def create(self, data: Dict[str, Any]) -> Team: - team = Team(**data) - self.db.add(team) - self.db.commit() - self.db.refresh(team) - return team - - def update(self, team_id: str, data: Dict[str, Any]) -> Optional[Team]: - team = self.get(team_id) - if not team: - return None - for k, v in data.items(): - setattr(team, k, v) - team.updated_at = datetime.utcnow() - self.db.commit() - self.db.refresh(team) - return team - - def delete(self, team_id: str) -> bool: - team = self.get(team_id) - if not team: - return False - self.db.delete(team) - self.db.commit() - return True - - # ── Member management ─────────────────────────────────────── - def list_members(self, team_id: str) -> List[tuple[TeamMember, UserShadow]]: - rows = ( - self.db.query(TeamMember, UserShadow) - .join(UserShadow, TeamMember.user_id == UserShadow.user_id) - .filter(TeamMember.team_id == team_id) - .order_by(TeamMember.role, TeamMember.joined_at) - .all() - ) - # Same as list_for_user: return real tuples (defensive layer). - return [tuple(r) for r in rows] - - def get_member(self, team_id: str, user_id: str) -> Optional[TeamMember]: - return ( - self.db.query(TeamMember) - .filter(TeamMember.team_id == team_id, TeamMember.user_id == user_id) - .first() - ) - - def get_member_role(self, team_id: str, user_id: str) -> Optional[str]: - row = ( - self.db.query(TeamMember.role) - .filter(TeamMember.team_id == team_id, TeamMember.user_id == user_id) - .first() - ) - return row[0] if row else None - - def list_for_users_bulk(self, user_ids: List[str]) -> Dict[str, List[tuple[Team, str]]]: - """Fetch all team memberships for a batch of users in one query, avoiding list_users N+1.""" - if not user_ids: - return {} - rows = ( - self.db.query(TeamMember.user_id, Team, TeamMember.role) - .join(Team, Team.team_id == TeamMember.team_id) - .filter(TeamMember.user_id.in_(user_ids)) - .order_by(Team.name) - .all() - ) - grouped: Dict[str, List[tuple[Team, str]]] = {uid: [] for uid in user_ids} - for uid, team, role in rows: - grouped.setdefault(uid, []).append((team, role)) - return grouped - - def member_counts_bulk(self, team_ids: List[str]) -> Dict[str, int]: - """Get member counts for a batch of teams in one GROUP BY, avoiding list_teams N+1.""" - if not team_ids: - return {} - rows = ( - self.db.query(TeamMember.team_id, func.count(TeamMember.user_id)) - .filter(TeamMember.team_id.in_(team_ids)) - .group_by(TeamMember.team_id) - .all() - ) - return {tid: count for tid, count in rows} - - def add_member(self, team_id: str, user_id: str, role: str = "member") -> TeamMember: - existing = self.get_member(team_id, user_id) - if existing: - existing.role = role - self.db.commit() - return existing - tm = TeamMember(team_id=team_id, user_id=user_id, role=role) - self.db.add(tm) - self.db.commit() - return tm - - def remove_member(self, team_id: str, user_id: str) -> bool: - tm = self.get_member(team_id, user_id) - if not tm: - return False - self.db.delete(tm) - self.db.commit() - return True - - def set_member_role(self, team_id: str, user_id: str, role: str) -> bool: - tm = self.get_member(team_id, user_id) - if not tm: - return False - tm.role = role - self.db.commit() - return True - - -class InviteCodeRepository: - """Invite code repository.""" - - def __init__(self, db: Session): - self.db = db - - def get(self, code: str) -> Optional[InviteCode]: - return self.db.query(InviteCode).filter(InviteCode.code == code).first() - - def list_all( - self, - include_used: bool = True, - include_revoked: bool = True, - ) -> List[InviteCode]: - q = self.db.query(InviteCode) - if not include_used: - q = q.filter(InviteCode.used_by.is_(None)) - if not include_revoked: - q = q.filter(InviteCode.revoked.is_(False)) - return q.order_by(desc(InviteCode.created_at)).all() - - def create(self, data: Dict[str, Any]) -> InviteCode: - inv = InviteCode(**data) - self.db.add(inv) - self.db.commit() - self.db.refresh(inv) - return inv - - def revoke(self, code: str) -> bool: - inv = self.get(code) - if not inv: - return False - inv.revoked = True - self.db.commit() - return True - - def delete(self, code: str) -> bool: - inv = self.get(code) - if not inv: - return False - self.db.delete(inv) - self.db.commit() - return True diff --git a/src/backend/core/db/repository/user.py b/src/backend/core/db/repository/user.py index cda75b47..f4c7d50b 100644 --- a/src/backend/core/db/repository/user.py +++ b/src/backend/core/db/repository/user.py @@ -6,17 +6,19 @@ """ import logging -from typing import Optional, List, Dict, Any from datetime import datetime +from typing import Any, Dict, List, Optional + import sqlalchemy as sa -from sqlalchemy.orm import Session -from sqlalchemy import and_, or_, desc, func, select from core.db.models import ( - UserShadow, ChatSession, ChatMessage, CatalogOverride, - KBSpace, KBDocument, Artifact, AuditLog, UserAgent, - LocalUser, Team, TeamMember, TeamFolder, InviteCode, - DingTalkConnection, LarkConnection, EmailConnection, + DingTalkConnection, + EmailConnection, + LarkConnection, + LocalUser, + UserShadow, ) +from sqlalchemy import and_, desc, func, or_, select +from sqlalchemy.orm import Session class UserRepository: @@ -31,9 +33,7 @@ def get_by_id(self, user_id: str) -> Optional[UserShadow]: def get_by_user_center_id(self, user_center_id: str) -> Optional[UserShadow]: """Get user by user center ID.""" - return self.db.query(UserShadow).filter( - UserShadow.user_center_id == user_center_id - ).first() + return self.db.query(UserShadow).filter(UserShadow.user_center_id == user_center_id).first() def create(self, user_data: Dict[str, Any]) -> UserShadow: """Create a new user shadow.""" @@ -131,9 +131,7 @@ def __init__(self, db: Session): def get(self, user_id: str) -> Optional[DingTalkConnection]: return ( - self.db.query(DingTalkConnection) - .filter(DingTalkConnection.user_id == user_id) - .first() + self.db.query(DingTalkConnection).filter(DingTalkConnection.user_id == user_id).first() ) def ensure(self, user_id: str) -> DingTalkConnection: @@ -167,11 +165,7 @@ def __init__(self, db: Session): self.db = db def get(self, user_id: str) -> Optional[LarkConnection]: - return ( - self.db.query(LarkConnection) - .filter(LarkConnection.user_id == user_id) - .first() - ) + return self.db.query(LarkConnection).filter(LarkConnection.user_id == user_id).first() def ensure(self, user_id: str) -> LarkConnection: """Idempotent: return existing record, otherwise create a disconnected record.""" @@ -204,11 +198,7 @@ def __init__(self, db: Session): self.db = db def get(self, user_id: str) -> Optional[EmailConnection]: - return ( - self.db.query(EmailConnection) - .filter(EmailConnection.user_id == user_id) - .first() - ) + return self.db.query(EmailConnection).filter(EmailConnection.user_id == user_id).first() def ensure(self, user_id: str) -> EmailConnection: """Idempotent: return existing record, otherwise create a disconnected record.""" diff --git a/src/backend/core/kb/__init__.py b/src/backend/core/kb/__init__.py index 9de2c66b..90d8da71 100644 --- a/src/backend/core/kb/__init__.py +++ b/src/backend/core/kb/__init__.py @@ -1,4 +1,4 @@ -"""Knowledge-base subsystem: Dify client, document parser/chunker, Milvus vector store. +"""Knowledge-base subsystem: provider seam, document parser/chunker, and vector store. Relocated from the top-level ``utils/`` package (where it was mislabeled as generic utilities) into ``core`` alongside core.content.kb_processing and core.services.kb_service. diff --git a/src/backend/core/kb/dify_kb.py b/src/backend/core/kb/dify_kb.py deleted file mode 100644 index 13100389..00000000 --- a/src/backend/core/kb/dify_kb.py +++ /dev/null @@ -1,528 +0,0 @@ -"""Dify Knowledge Base client for dataset management.""" -from __future__ import annotations - -from datetime import datetime, timezone -import logging -import os -import re -from typing import Any, Dict, List, Optional - -import requests - -logger = logging.getLogger(__name__) - - -def _normalize_timestamp(value: Any) -> Optional[str]: - """Normalize Dify timestamp values to ISO8601 strings.""" - if value is None or value == "": - return None - - numeric_value: Optional[float] = None - if isinstance(value, (int, float)): - numeric_value = float(value) - elif isinstance(value, str): - text = value.strip() - if not text: - return None - try: - numeric_value = float(text) - except ValueError: - try: - parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) - if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=timezone.utc) - return parsed.astimezone(timezone.utc).isoformat() - except ValueError: - return None - else: - return None - - if numeric_value >= 1e15: - numeric_value /= 1_000_000 - elif numeric_value >= 1e12: - numeric_value /= 1_000 - - try: - return datetime.fromtimestamp(numeric_value, tz=timezone.utc).isoformat() - except (OverflowError, OSError, ValueError): - return None - - -def _resolve_documents_total( - *, - data: Dict[str, Any], - raw_docs: List[Dict[str, Any]], - dataset_id: str, - base_url: str, - auth_token: str, - page: int, - limit: int, - keyword: str, -) -> int: - """Resolve actual documents total, probing more pages when Dify omits it.""" - total_raw = data.get("total") - try: - total = int(total_raw) - if total >= 0: - return total - except (TypeError, ValueError): - pass - - has_more = bool(data.get("has_more")) - if not has_more: - return len(raw_docs) - - total = (max(page, 1) - 1) * limit + len(raw_docs) - next_page = max(page, 1) + 1 - - while True: - params: Dict[str, Any] = {"page": next_page, "limit": limit} - if keyword: - params["keyword"] = keyword - try: - resp = requests.get( - f"{base_url}/datasets/{dataset_id}/documents", - headers={"Authorization": f"Bearer {auth_token}"}, - params=params, - timeout=3, - ) - resp.raise_for_status() - next_payload = resp.json() - except Exception as exc: - logger.warning( - "Failed to probe Dify documents total dataset=%s page=%s: %s", - dataset_id, - next_page, - exc, - ) - break - - next_docs = next_payload.get("data", []) - if not isinstance(next_docs, list) or not next_docs: - break - - total += len(next_docs) - if not next_payload.get("has_more"): - break - next_page += 1 - - return total - - -def _resolve_dify_config() -> tuple[str, str]: - # DB-first: try SystemConfigService, fall back to env - try: - from core.services.system_config import SystemConfigService - svc = SystemConfigService.get_instance() - base_url = (svc.get("knowledge_base.url") or "").strip().rstrip("/") - auth_token = (svc.get("knowledge_base.api_key") or "").strip() - if base_url and auth_token: - return base_url, auth_token - except Exception: - pass - base_url = ( - os.getenv("DIFY_URL") or os.getenv("DIFY_BASE_URL") or "" - ).strip().rstrip("/") - auth_token = ( - os.getenv("DIFY_API_KEY") or os.getenv("DIFY_AUTH_TOKEN") or "" - ).strip() - return base_url, auth_token - - -def get_allowed_dataset_ids() -> set[str]: - """ - Optional allowlist of Dify dataset IDs. - - DB-first via SystemConfigService, falls back to env DIFY_ALLOWED_DATASET_IDS. - """ - raw = "" - try: - from core.services.system_config import SystemConfigService - raw = (SystemConfigService.get_instance().get("knowledge_base.allowed_dataset_ids") or "").strip() - except Exception: - pass - if not raw: - raw = (os.getenv("DIFY_ALLOWED_DATASET_IDS") or "").strip() - if not raw: - return set() - parts = [x.strip() for x in re.split(r"[,\n;]+", raw) if x.strip()] - return set(parts) - - -def is_dataset_allowed(dataset_id: str) -> bool: - """Check whether dataset_id is allowed by env allowlist.""" - target = (dataset_id or "").strip() - if not target: - return False - allowlist = get_allowed_dataset_ids() - if not allowlist: - return True - return target in allowlist - - -def is_dify_enabled() -> bool: - """ - Determine whether Dify-backed knowledge base should be used. - - Priority: - 1. DB config knowledge_base.provider - 2. Explicit KNOWLEDGE_BASE env setting (must be "dify") - 3. Fallback to presence of Dify URL + API key - """ - try: - from core.services.system_config import SystemConfigService - provider = (SystemConfigService.get_instance().get("knowledge_base.provider") or "").strip().lower() - if provider: - return provider == "dify" - except Exception: - pass - kb_backend = (os.getenv("KNOWLEDGE_BASE") or "").strip().lower() - if kb_backend: - return kb_backend == "dify" - - base_url, auth_token = _resolve_dify_config() - return bool(base_url and auth_token) - - -def _format_doc_desc(doc: Dict[str, Any]) -> str: - """Build a human-readable description for a Dify document.""" - parts: List[str] = [] - status = doc.get("indexing_status", "") - if status: - status_map = { - "completed": "已完成", - "indexing": "索引中", - "waiting": "等待中", - "error": "错误", - "paused": "已暂停", - } - parts.append(f"状态：{status_map.get(status, status)}") - word_count = doc.get("word_count") or doc.get("tokens", 0) - if word_count: - parts.append(f"{word_count:,} 词") - src = doc.get("data_source_type", "") - if src: - src_map = { - "upload_file": "上传文件", - "web_site": "网页", - "notion_import": "Notion", - } - parts.append(src_map.get(src, src)) - return " | ".join(parts) if parts else "文档" - - -def _dataset_to_kb_item(ds: Dict[str, Any]) -> Dict[str, Any]: - """Convert a Dify dataset object into a frontend KBItem-compatible dict.""" - tags: List[str] = [] - if ds.get("indexing_technique"): - tags.append(ds["indexing_technique"]) - if ds.get("embedding_model_provider"): - tags.append(ds["embedding_model_provider"]) - - doc_count: int = ds.get("document_count", 0) - word_count: int = ds.get("word_count", 0) - - detail = ( - f"### {ds.get('name', '知识库')}\n\n" - f"{ds.get('description') or '暂无简介'}\n" - ) - - return { - "id": ds["id"], - "kind": "knowledge_base", - "name": ds.get("name", ds["id"]), - "description": ds.get("description") or "无简介", - "desc": ds.get("description") or "无简介", - "enabled": True, - "version": "dify", - "tags": tags, - "detail": detail, - "provider": ds.get("embedding_model_provider", "Dify"), - "document_count": doc_count, - "word_count": word_count, - } - - -def list_datasets(page: int = 1, limit: int = 100, keyword: str = "", timeout=3) -> List[Dict[str, Any]]: - """Fetch all datasets from Dify and return as KBItem-compatible list. - - Args: - timeout: HTTP timeout in seconds. Can be a float or a - (connect_timeout, read_timeout) tuple. - """ - base_url, auth_token = _resolve_dify_config() - if not base_url or not auth_token: - logger.warning("Dify config missing: DIFY_URL and DIFY_API_KEY required") - return [] - - params: Dict[str, Any] = {"page": page, "limit": limit} - if keyword: - params["keyword"] = keyword - - try: - # Use a session with retries disabled when using short timeouts, - # to avoid blocking on unreachable Dify instances. - session = requests.Session() - if isinstance(timeout, tuple) or (isinstance(timeout, (int, float)) and timeout < 5): - from requests.adapters import HTTPAdapter - session.mount("http://", HTTPAdapter(max_retries=0)) - session.mount("https://", HTTPAdapter(max_retries=0)) - resp = session.get( - f"{base_url}/datasets", - headers={"Authorization": f"Bearer {auth_token}"}, - params=params, - timeout=timeout, - ) - resp.raise_for_status() - data = resp.json() - datasets = data.get("data", []) - items = [_dataset_to_kb_item(ds) for ds in datasets] - allowlist = get_allowed_dataset_ids() - if not allowlist: - return items - return [item for item in items if str(item.get("id", "")).strip() in allowlist] - except Exception as exc: - logger.error("Failed to fetch Dify datasets: %s", exc) - return [] - - -def get_dataset(dataset_id: str) -> Optional[Dict[str, Any]]: - """Fetch a single dataset detail from Dify.""" - if not is_dataset_allowed(dataset_id): - return None - - base_url, auth_token = _resolve_dify_config() - if not base_url or not auth_token: - return None - - try: - resp = requests.get( - f"{base_url}/datasets/{dataset_id}", - headers={"Authorization": f"Bearer {auth_token}"}, - timeout=3, - ) - resp.raise_for_status() - return resp.json() - except Exception as exc: - logger.error("Failed to fetch Dify dataset %s: %s", dataset_id, exc) - return None - - -def list_documents( - dataset_id: str, - page: int = 1, - limit: int = 20, - keyword: str = "", -) -> Dict[str, Any]: - """Fetch documents for a Dify dataset and convert to frontend-friendly format.""" - base_url, auth_token = _resolve_dify_config() - empty: Dict[str, Any] = { - "items": [], - "total": 0, - "page": page, - "page_size": limit, - "has_more": False, - } - if not is_dataset_allowed(dataset_id): - return empty - - if not base_url or not auth_token: - return empty - - params: Dict[str, Any] = {"page": page, "limit": limit} - if keyword: - params["keyword"] = keyword - - try: - resp = requests.get( - f"{base_url}/datasets/{dataset_id}/documents", - headers={"Authorization": f"Bearer {auth_token}"}, - params=params, - timeout=3, - ) - resp.raise_for_status() - data = resp.json() - except Exception as exc: - logger.error("Failed to fetch Dify documents for dataset %s: %s", dataset_id, exc) - return empty - - raw_docs = data.get("data", []) - items = [ - { - "id": doc.get("id", ""), - "title": doc.get("name", doc.get("id", "")), - "desc": _format_doc_desc(doc), - "word_count": doc.get("word_count", 0), - "indexing_status": doc.get("indexing_status", ""), - "enabled": doc.get("enabled", True), - "data_source_type": doc.get("data_source_type", ""), - "created_at": _normalize_timestamp(doc.get("created_at")), - } - for doc in raw_docs - ] - - total = _resolve_documents_total( - data=data, - raw_docs=raw_docs if isinstance(raw_docs, list) else [], - dataset_id=dataset_id, - base_url=base_url, - auth_token=auth_token, - page=page, - limit=limit, - keyword=keyword, - ) - - dataset_detail = get_dataset(dataset_id) - if isinstance(dataset_detail, dict): - detail_total = dataset_detail.get("document_count") - try: - detail_total_int = int(detail_total) - if detail_total_int > total: - total = detail_total_int - except (TypeError, ValueError): - pass - - return { - "items": items, - "total": total, - "page": data.get("page", page), - "page_size": data.get("limit", limit), - "has_more": data.get("has_more", False), - } - - -def _request_json(url: str, auth_token: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: - """Perform GET request and return JSON dict.""" - resp = requests.get( - url, - headers={"Authorization": f"Bearer {auth_token}"}, - params=params, - timeout=3, - ) - resp.raise_for_status() - data = resp.json() - return data if isinstance(data, dict) else {} - - -def _first_non_empty_str(*values: Any) -> str: - """Return the first non-empty string value.""" - for value in values: - if isinstance(value, str): - text = value.strip() - if text: - return text - return "" - - -def get_document_detail(dataset_id: str, document_id: str) -> Dict[str, Any]: - """ - Fetch Dify document detail and aggregate segment content for frontend modal display. - - Dify list API does not include full content. Content is assembled from the - document segments endpoint. - """ - base_url, auth_token = _resolve_dify_config() - empty: Dict[str, Any] = { - "id": document_id, - "title": document_id, - "desc": "文档详情暂不可用", - "content": "", - } - if not is_dataset_allowed(dataset_id): - return empty - - if not base_url or not auth_token: - return empty - - detail: Dict[str, Any] = {} - try: - detail = _request_json( - f"{base_url}/datasets/{dataset_id}/documents/{document_id}", - auth_token=auth_token, - ) - except Exception as exc: - logger.warning( - "Failed to fetch Dify document detail dataset=%s document=%s: %s", - dataset_id, - document_id, - exc, - ) - - segment_texts: List[str] = [] - page = 1 - limit = 100 - max_segments = 500 - - while len(segment_texts) < max_segments: - try: - seg_payload = _request_json( - f"{base_url}/datasets/{dataset_id}/documents/{document_id}/segments", - auth_token=auth_token, - params={"page": page, "limit": limit}, - ) - except Exception as exc: - logger.warning( - "Failed to fetch Dify document segments dataset=%s document=%s page=%s: %s", - dataset_id, - document_id, - page, - exc, - ) - break - - raw_segments = seg_payload.get("data", []) - if not isinstance(raw_segments, list): - raw_segments = [] - - for seg in raw_segments: - if not isinstance(seg, dict): - continue - text = seg.get("content") or seg.get("answer") or seg.get("segment") - if isinstance(text, str): - normalized = text.strip() - if normalized: - segment_texts.append(normalized) - if len(segment_texts) >= max_segments: - break - - has_more = bool(seg_payload.get("has_more", False)) - if not has_more: - break - page += 1 - - content = "\n\n".join(segment_texts).strip() - if content: - try: - from core.services.system_config import SystemConfigService - max_chars = int(SystemConfigService.get_instance().get("knowledge_base.detail_max_chars", "50000")) - except Exception: - max_chars = int(os.getenv("KB_DETAIL_CONTENT_MAX_CHARS", "50000")) - if len(content) > max_chars: - content = content[:max_chars].rstrip() + "\n\n...（内容过长，已截断）" - - # Dify detail endpoint may wrap document fields under "data". - detail_data = detail.get("data") if isinstance(detail.get("data"), dict) else detail - - title = ( - _first_non_empty_str( - detail_data.get("name") if isinstance(detail_data, dict) else None, - detail_data.get("title") if isinstance(detail_data, dict) else None, - detail.get("name"), - detail.get("title"), - ) - or (detail_data.get("id") if isinstance(detail_data, dict) else "") - or detail.get("id") - or document_id - ) - desc_source = detail_data if isinstance(detail_data, dict) else detail - desc = _format_doc_desc(desc_source) - if not desc or desc == "文档": - desc = "文档详情" - - return { - "id": document_id, - "title": title, - "desc": desc, - "content": content, - "segment_count": len(segment_texts), - } diff --git a/src/backend/core/kb/external_provider.py b/src/backend/core/kb/external_provider.py new file mode 100644 index 00000000..1c941046 --- /dev/null +++ b/src/backend/core/kb/external_provider.py @@ -0,0 +1,35 @@ +"""Community edition has no externally managed knowledge provider.""" + + +def is_enabled() -> bool: + return False + + +def list_collections(*args, **kwargs) -> list: + return [] + + +def list_documents(*args, **kwargs) -> list: + return [] + + +def get_document_detail(*args, **kwargs): + return None + + +def get_allowed_collection_ids(*args, **kwargs): + return None + + +def runtime_request_context(enabled_kb_ids: list[str]) -> tuple[dict[str, str], dict[str, str]]: + return {}, {} + + +__all__ = [ + "get_allowed_collection_ids", + "get_document_detail", + "is_enabled", + "list_collections", + "list_documents", + "runtime_request_context", +] diff --git a/src/backend/core/kb/external_retrieval.py b/src/backend/core/kb/external_retrieval.py new file mode 100644 index 00000000..fd054084 --- /dev/null +++ b/src/backend/core/kb/external_retrieval.py @@ -0,0 +1,39 @@ +"""Community edition disables external knowledge retrieval.""" + +MAX_RETRIEVE_TOKENS = 50_000 +RETRIEVE_REQUEST_TIMEOUT_SECONDS = 10 +RETRIEVE_TOTAL_TIMEOUT_SECONDS = 60 +RETRIEVE_MAX_CONCURRENCY = 3 + + +class DatasetRetrievalTimeoutError(TimeoutError): + pass + + +class DatasetRetrievalUnavailableError(RuntimeError): + pass + + +def retrieve_dataset_content(*args, **kwargs) -> list: + return [] + + +async def retrieve_dataset_content_async(*args, **kwargs) -> list: + return [] + + +def list_external_datasets(**kwargs) -> list: + return [] + + +__all__ = [ + "DatasetRetrievalTimeoutError", + "DatasetRetrievalUnavailableError", + "MAX_RETRIEVE_TOKENS", + "RETRIEVE_MAX_CONCURRENCY", + "RETRIEVE_REQUEST_TIMEOUT_SECONDS", + "RETRIEVE_TOTAL_TIMEOUT_SECONDS", + "list_external_datasets", + "retrieve_dataset_content", + "retrieve_dataset_content_async", +] diff --git a/src/backend/core/licensing/__init__.py b/src/backend/core/licensing/__init__.py deleted file mode 100644 index 225f382c..00000000 --- a/src/backend/core/licensing/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Edition / License facade (CE/EE shared interface layer). - -The CE derived tree only carries this package's stub semantics (``has()`` always False); -real signature verification / entitlement belongs to later commercial-edition work and does not appear in this package. -""" - -from .deps import requires_feature -from .features import Feature, FeatureNotLicensed, SeatLimitExceeded -from .manager import LicenseManager, license_manager - -__all__ = [ - "Feature", - "FeatureNotLicensed", - "LicenseManager", - "SeatLimitExceeded", - "license_manager", - "requires_feature", -] diff --git a/src/backend/core/licensing/_clock_guard.py b/src/backend/core/licensing/_clock_guard.py deleted file mode 100644 index 4e3b3b6c..00000000 --- a/src/backend/core/licensing/_clock_guard.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Clock-rollback protection for offline licenses (commercial-edition only — excluded from the CE derived tree). - -Offline license expiry checks depend on the system clock; a customer could -bypass expiry simply by setting the system time back. This module maintains a -high-water mark of the "largest date ever observed": every time today's date is -requested it returns ``max(system today, high-water mark)``, and persists the -new maximum whenever the date advances. Once the clock is rolled back, the -effective date stays at the high-water mark and never regresses, so expiry -checks cannot be bypassed by rewinding the clock. - -The high-water mark is persisted in two places (the effective value is the max -of both plus the system clock), so wiping/tampering with either one does not -defeat the guard: - -- DB: a ``content_blocks`` row (id=:data:`_HW_BLOCK_ID`) — volume-persisted, - survives rebuilds -- File: a sidecar next to the license file (``.seen``) — fallback when - the DB gets reset - -An additional in-process cache of the high-water mark avoids hitting the DB/disk -on every request — the two stores are only written back when the date advances -(roughly once per day). - -> Honest disclaimer: an attacker with full control of the host + DB can still -> bypass this by wiping both stores and rolling back the clock at the same -> time; that is the nature of offline licensing (the machine is in the -> customer's hands). This guard blocks casual bypasses like "just change the -> system time to extend the license", raising the bar to "must tamper with -> multiple pieces of persisted state simultaneously". -""" - -from __future__ import annotations - -import logging -from datetime import date -from pathlib import Path -from typing import Optional - -logger = logging.getLogger(__name__) - -# Id of the high-water-mark row in content_blocks. Underscore prefix + -# non-business semantics, to avoid confusion with operational content. -_HW_BLOCK_ID = "_license_clock_hw" - -# In-process high-water-mark cache (seeded once from storage on cold start). -_hw_cache: Optional[date] = None -_seeded = False - - -def _sidecar_path(license_path: str) -> Path: - return Path(license_path + ".seen") - - -def _parse(value: object) -> Optional[date]: - if not value: - return None - try: - return date.fromisoformat(str(value).strip()) - except (TypeError, ValueError): - return None - - -# ── sidecar file ────────────────────────────────────────────────────────── - -def _read_sidecar(license_path: str) -> Optional[date]: - try: - return _parse(_sidecar_path(license_path).read_text(encoding="utf-8")) - except OSError: - return None - - -def _write_sidecar(license_path: str, d: date) -> None: - try: - p = _sidecar_path(license_path) - tmp = p.with_suffix(p.suffix + ".tmp") - tmp.write_text(d.isoformat(), encoding="utf-8") - tmp.replace(p) - except OSError as e: # Write failure is non-fatal: DB still backs us up, retry next time - logger.warning("[license] 时钟高水位 sidecar 写入失败: %s", e) - - -# ── DB (content_blocks, best-effort, never raises) ───────────────────────── - -def _read_db() -> Optional[date]: - try: - from core.db.engine import SessionLocal - from core.db.models.artifact import ContentBlock - - with SessionLocal() as db: - row = db.get(ContentBlock, _HW_BLOCK_ID) - return _parse(row.payload) if row else None - except Exception: # noqa: BLE001 - silently degrade to sidecar when DB is unavailable / table missing - return None - - -def _write_db(d: date) -> None: - try: - from core.db.engine import SessionLocal - from core.db.models.artifact import ContentBlock - - with SessionLocal() as db: - row = db.get(ContentBlock, _HW_BLOCK_ID) - if row is None: - db.add(ContentBlock(id=_HW_BLOCK_ID, payload=d.isoformat(), - updated_by="license_clock_guard")) - else: - row.payload = d.isoformat() - db.commit() - except Exception as e: # noqa: BLE001 - logger.warning("[license] 时钟高水位 DB 写入失败: %s", e) - - -# ── public API ────────────────────────────────────────────────────────────── - -def monotonic_today(license_path: Optional[str]) -> date: - """Return a monotonically non-decreasing "today": ``max(system today, largest date ever observed)``. - - When no license path is configured there is no expiry semantics to guard, - so return the system date directly (no storage reads/writes). - """ - global _hw_cache, _seeded - - today = date.today() - if not license_path: - return today - - if not _seeded: - # Cold start: seed the high-water mark from both stores (each best-effort). - candidates = [d for d in (_read_sidecar(license_path), _read_db()) if d] - _hw_cache = max(candidates) if candidates else None - _seeded = True - - effective = today if _hw_cache is None else max(today, _hw_cache) - if _hw_cache is None or effective > _hw_cache: - _hw_cache = effective - _write_sidecar(license_path, effective) - _write_db(effective) - return effective - - -def reset_cache() -> None: - """Clear the in-process cache (used by tests and license hot-reload).""" - global _hw_cache, _seeded - _hw_cache = None - _seeded = False diff --git a/src/backend/core/licensing/deps.py b/src/backend/core/licensing/deps.py deleted file mode 100644 index 833600dc..00000000 --- a/src/backend/core/licensing/deps.py +++ /dev/null @@ -1,25 +0,0 @@ -"""FastAPI dependency: license feature-flag guard for EE routes (second line of defense). - -The first line of defense is the route registry (the CE tree physically lacks -EE routes); this guard protects against the scenario where "the commercial -edition deployed the full codebase, but the license did not purchase a given -capability pack." Under internal deployment (no license file) everything is -allowed through, consistent with historical behavior. -""" - -from __future__ import annotations - -from fastapi import Depends - -from .features import Feature, FeatureNotLicensed -from .manager import license_manager - - -def requires_feature(feature: Feature): - async def _dep() -> None: - if not license_manager.has(feature): - # The envelope (402 / code 40201) has a single source in - # FeatureNotLicensed, rendered uniformly by the global error_handler. - raise FeatureNotLicensed(feature, data={"mode": license_manager.mode()}) - - return Depends(_dep) diff --git a/src/backend/core/licensing/features.py b/src/backend/core/licensing/features.py deleted file mode 100644 index 16e3d516..00000000 --- a/src/backend/core/licensing/features.py +++ /dev/null @@ -1,64 +0,0 @@ -"""EE-exclusive feature-flag enum. - -The boundary matches the official pricing page: automation / batch execution / -data canvas (personal) / L2-L3 memory are community-edition capabilities and are -not in this enum — this only lists "organization-level" commercial capabilities. -""" - -from enum import Enum -from typing import Optional - -from core.infra.exceptions import AppException - - -class FeatureNotLicensed(AppException): - """The current license does not authorize this feature flag (both CE/EE trees share the same definition). - - An AppException subclass — mapped by the global error_handler uniformly into - the 402 envelope {code:40201, message, data:{feature, ...}}; this is the - **only** source of the license 402 envelope, so route/service layers should - not hand-roll HTTPException(402) again. 402 rather than 403: a 403 would be - treated by the frontend as an expired token and force a logout. - """ - - def __init__(self, feature: "Feature", message: Optional[str] = None, - data: Optional[dict] = None): - self.feature = feature - payload = {"feature": feature.value} - if data: - payload.update(data) - super().__init__( - code=40201, - message=message or f"该功能未在当前 license 中授权: {feature.value}", - status_code=402, - data=payload, - ) - - -class SeatLimitExceeded(AppException): - """Insufficient seats / invalid license prevents adding a user (402, code 40202). - - The service layer raises this exception rather than HTTPException — the HTTP - semantics are honored uniformly by the error_handler, and non-HTTP callers - (optional-auth, scripts) can catch it precisely by type. - """ - - def __init__(self, message: str, data: Optional[dict] = None): - super().__init__(code=40202, message=message, status_code=402, data=data or {}) - - -class Feature(str, Enum): - SSO = "sso" - MULTI_TENANCY = "multi_tenancy" - AUDIT = "audit" - MEMORY_AUDIT = "memory_audit" - BILLING = "billing" - QUOTA = "quota" - PERSISTENT_SANDBOX = "persistent_sandbox" - CLOUD_STORAGE = "cloud_storage" - INDUSTRY_TOOLS = "industry_tools" - CONTENT_ADMIN = "content_admin" - SYSTEM_CONFIG = "system_config" - CANVAS_COLLAB = "canvas_collab" - WHITELABEL = "whitelabel" - MODEL_GATEWAY = "model_gateway" diff --git a/src/backend/core/licensing/manager.py b/src/backend/core/licensing/manager.py deleted file mode 100644 index 0cf6d36b..00000000 --- a/src/backend/core/licensing/manager.py +++ /dev/null @@ -1,50 +0,0 @@ -"""License 门面 —— 社区版 stub。 - -社区版没有 license 概念：全部商业能力位恒 False、不限席位。 -本文件不含任何验签逻辑实现体（验签属商业版闭源代码）。 -""" - -from __future__ import annotations - -from typing import Optional - -from .features import Feature, FeatureNotLicensed # noqa: F401 (re-export 兼容) - - -class LicenseManager: - def reload(self) -> None: - return None - - def mode(self) -> str: - return "ce" - - def is_active(self) -> bool: - """CE is always runnable and is never gated by commercial licensing.""" - return True - - def has(self, feature: Feature) -> bool: - return False - - def features_map(self) -> dict[str, bool]: - return {f.value: False for f in Feature} - - def require(self, feature: Feature) -> None: - raise FeatureNotLicensed(feature) - - def seats_allow(self, active_users: int) -> bool: - return True - - def info(self) -> Optional[dict]: - return None - - def status(self) -> dict: - return { - "edition": "ce", - "mode": "ce", - "required": False, - "license": None, - "features": self.features_map(), - } - - -license_manager = LicenseManager() diff --git a/src/backend/core/licensing/seats.py b/src/backend/core/licensing/seats.py deleted file mode 100644 index f4069d24..00000000 --- a/src/backend/core/licensing/seats.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Single source of truth for seat counting (shared by CE/EE). - -The definition of "one seat" appears here exactly once: shared by the config -panel display (seats_used) and enforcement before adding a user -(seat_available — local registration, SSO auto account creation), so the two -never drift. Under CE / internal / unlimited-seat license, seat_available is -always True. -""" - -from __future__ import annotations - -from sqlalchemy.orm import Session - -from core.db.models import UserShadow - -from .manager import license_manager - - -def seats_used(db: Session) -> int: - """Current occupied seat count = total number of users_shadow rows (including SSO shadow accounts).""" - return db.query(UserShadow).count() - - -def seat_available(db: Session) -> bool: - """Seat check before adding a user.""" - return license_manager.seats_allow(seats_used(db)) - - -SEAT_LIMIT_MESSAGE = "已达 license 席位上限，请联系管理员扩容或更新 license" -LICENSE_BLOCK_MESSAGE = "当前 license 无效或未激活，无法新增用户——请联系管理员在 系统配置 → License 上传有效 license" - - -def seat_block_reason(db: Session) -> str | None: - """Reason text when adding a user is rejected; returns None when allowed. - - seats_allow is always False under expired/invalid/missing modes — in that case - the real reason is the license state rather than the seat count, so the text - must be distinguished; otherwise a new deployment (path configured but no file - provided) would be misled by "seat limit" into investigating a nonexistent - seat problem. - """ - if seat_available(db): - return None - mode = license_manager.mode() - return SEAT_LIMIT_MESSAGE if mode in ("licensed", "grace") else LICENSE_BLOCK_MESSAGE diff --git a/src/backend/core/llm/agent_factory.py b/src/backend/core/llm/agent_factory.py index a4b8a58b..68ae3e88 100644 --- a/src/backend/core/llm/agent_factory.py +++ b/src/backend/core/llm/agent_factory.py @@ -125,7 +125,7 @@ def _effective_mcp_server_keys( allow &= spec_set # Note: empty enabled_kb_ids [] means no KBs selected in frontend (e.g. catalog - # KB list was empty due to Dify being unreachable). We do NOT remove the tool + # KB list was empty because an external provider was unreachable). We do NOT remove the tool # in this case — the MCP impl will auto-resolve available KBs at call time. # "Database query" umbrella expansion: when the user/catalog selects the @@ -177,7 +177,7 @@ def _filter_skill_ids_for_user(skill_ids: list[str], user_id: Optional[str]) -> def _filter_kb_ids_for_user(kb_ids: list[str], user_id: Optional[str]) -> list[str]: - """Strip out KB ids the current user has no access to (local KBs + Dify datasets), preventing unauthorized ids passed in from the frontend. + """Strip out KB ids the current user has no access to (local + external collections), preventing unauthorized ids passed in from the frontend. Single source of truth ``core.auth.kb_permissions``: public KBs are visible to everyone, private KBs to their owner, and scoped-visibility KBs per @@ -259,9 +259,11 @@ def _inject_runtime_headers( if not enabled_servers: return enabled_servers + from core.kb.external_provider import runtime_request_context + normalized = [str(x).strip() for x in (enabled_kb_ids or []) if str(x).strip()] - dify_ids = [x for x in normalized if not x.startswith("kb_")] local_ids = [x for x in normalized if x.startswith("kb_")] + external_headers, external_env = runtime_request_context(normalized) origin = channel_origin or {} ctx_headers = { @@ -273,17 +275,17 @@ def _inject_runtime_headers( "X-Chat-Id": chat_id or "", "X-Channel-Id": origin.get("channel_id") or "", "X-Conversation-Id": origin.get("conversation_id") or "", - "X-Allowed-Dataset-Ids": ",".join(dify_ids), "X-Allowed-Kb-Ids": ",".join(local_ids), "X-Reranker-Enabled": "true" if reranker_enabled else "false", } + ctx_headers.update(external_headers) ctx_env = { "CURRENT_USER_ID": current_user_id or "", "CURRENT_CHAT_ID": chat_id or "", - "DIFY_ALLOWED_DATASET_IDS": ",".join(dify_ids), "LOCAL_KB_ALLOWED_IDS": ",".join(local_ids), "RERANKER_ENABLED": "true" if reranker_enabled else "false", } + ctx_env.update(external_env) out: dict = {} for key, cfg in enabled_servers.items(): diff --git a/src/backend/core/llm/tools/_common.py b/src/backend/core/llm/tools/_common.py index 939d9047..527c2287 100644 --- a/src/backend/core/llm/tools/_common.py +++ b/src/backend/core/llm/tools/_common.py @@ -10,9 +10,10 @@ from typing import Any, Optional from agentscope.message import TextBlock + # AgentScope 2.0: tool functions must return ToolChunk; aliased (its fields are a superset of ToolResponse). from agentscope.tool._response import ToolChunk as ToolResponse - +from core.llm.tools.edition_myspace_vfs import organization_mutation_blocked from core.services.project_scope import ProjectScope logger = logging.getLogger(__name__) @@ -24,14 +25,19 @@ def resp_json(payload: dict[str, Any]) -> ToolResponse: Mirrors ``core.llm.tool._resp_json``. Re-implemented here so this package has zero cross-imports back to the large monolithic ``tool.py``. """ - return ToolResponse(content=[TextBlock( - type="text", - text=json.dumps(payload, ensure_ascii=False), - )]) + return ToolResponse( + content=[ + TextBlock( + type="text", + text=json.dumps(payload, ensure_ascii=False), + ) + ] + ) def resolve_sandbox_session( - sandbox_session_id: Optional[str], chat_id: Optional[str], + sandbox_session_id: Optional[str], + chat_id: Optional[str], ) -> Optional[str]: """``sandbox_session_id`` wins; ``None`` means 'unspecified' → fall back to ``chat_id`` (legacy behavior). Explicit ``""`` stays ephemeral.""" @@ -39,8 +45,13 @@ def resolve_sandbox_session( async def myspace_write_guard( - *, chat_id: Optional[str], op: str, logical_path: str, - is_myspace: bool, interactive: bool, summary: str, + *, + chat_id: Optional[str], + op: str, + logical_path: str, + is_myspace: bool, + interactive: bool, + summary: str, ) -> Optional[ToolResponse]: """§13 gate (Claude Code shape): an unconfirmed /myspace write **suspends the current tool coroutine** to wait for the user's out-of-band decision; approve @@ -58,9 +69,13 @@ async def myspace_write_guard( if not is_myspace: return None from core.llm.tools import _myspace_confirm as _mc + blk = await _mc.gate( - chat_id=chat_id, op=op, logical_path=logical_path, - interactive=interactive, summary=summary, + chat_id=chat_id, + op=op, + logical_path=logical_path, + interactive=interactive, + summary=summary, ) return resp_json(blk) if blk is not None else None @@ -80,22 +95,19 @@ async def sandbox_exec_bash( resolved ``_sess`` (``sandbox_session_id`` or chat_id fallback), never a DB-scoping chat id. Kept named ``chat_id`` to avoid churning call sites. """ - from core.sandbox import ( - ExecuteRequest, - SandboxError, - SandboxConnectError, - get_sandbox_provider, - ) + from core.sandbox import ExecuteRequest, SandboxConnectError, SandboxError, get_sandbox_provider try: provider = get_sandbox_provider() - result = await provider.execute(ExecuteRequest( - script_content=script, - script_name="_tool_helper.sh", - language="bash", - timeout=max(1, min(int(timeout or 30), 60)), - session_id=chat_id, - )) + result = await provider.execute( + ExecuteRequest( + script_content=script, + script_name="_tool_helper.sh", + language="bash", + timeout=max(1, min(int(timeout or 30), 60)), + session_id=chat_id, + ) + ) return result.exit_code, result.stdout, result.stderr except (SandboxError, SandboxConnectError) as exc: return -1, "", str(exc) @@ -139,14 +151,9 @@ def upsert_myspace_artifact( logger.warning("[artifact-sync] missing user_id or filename; skip") return None - # PR 1 read-only: under a team-project scope, files produced by - # sandbox_get_artifact / bash are not auto-upserted for now (otherwise it - # would create orphan artifacts with user_id=self/team_id=NULL, polluting the - # personal MySpace root). PR 2 will enable "auto-place into the team-linked - # folder" after adding the permission gate. - if scope is not None and scope.is_team: + if organization_mutation_blocked(scope): logger.info( - "[artifact-sync] team scope: skip auto-upsert for filename=%s (PR 1 read-only)", + "[artifact-sync] edition scope blocked auto-upsert for filename=%s", filename, ) return None @@ -158,6 +165,7 @@ def upsert_myspace_artifact( # ── 1. Mirror to myspace_cache so next sandbox seed sees the update ── try: from core.sandbox._common import myspace_cache_dir + cache_dir = myspace_cache_dir(user_id) cache_dir.mkdir(parents=True, exist_ok=True) (cache_dir / name).write_bytes(content) @@ -168,6 +176,7 @@ def upsert_myspace_artifact( # ── 2. Try in-place update of an existing live artifact ──────────── try: from datetime import datetime, timezone + from core.db.engine import SessionLocal from core.db.models import Artifact from core.storage import get_storage @@ -194,7 +203,8 @@ def upsert_myspace_artifact( except Exception as exc: # noqa: BLE001 logger.warning( "[artifact-sync] upload_bytes failed for %s: %s", - row.storage_key, exc, + row.storage_key, + exc, ) return None row.size_bytes = max(len(content), 1) @@ -203,7 +213,10 @@ def upsert_myspace_artifact( db.commit() logger.info( "[artifact-sync] in-place updated artifact %s (user=%s name=%s size=%d)", - row.artifact_id, user_id, name, len(content), + row.artifact_id, + user_id, + name, + len(content), ) return { "file_id": row.artifact_id, @@ -239,12 +252,14 @@ def upsert_myspace_artifact( return None refs = _store_generated_files( - [{ - "name": name, - "size": len(content), - "content_b64": base64.b64encode(content).decode("ascii"), - "mime_type": mime, - }], + [ + { + "name": name, + "size": len(content), + "content_b64": base64.b64encode(content).decode("ascii"), + "mime_type": mime, + } + ], user_id=user_id, source="myspace_sync", extra_metadata={"chat_id": chat_id} if chat_id else None, @@ -260,6 +275,7 @@ def upsert_myspace_artifact( if new_file_id and chat_id: try: from datetime import datetime, timezone + from core.db.engine import SessionLocal from core.db.models import Artifact except Exception as exc: # noqa: BLE001 @@ -270,31 +286,35 @@ def upsert_myspace_artifact( # register_as_artifact path carries no logical path, so sync_upsert's # myspace_rel prefix cannot cover it — we must explicitly set folder here. _proj_folder_id: Optional[str] = ( - scope.root_folder_id - if scope is not None and scope.is_personal - else None + scope.root_folder_id if scope is not None and scope.is_personal else None ) db = SessionLocal() try: # Guard against race / duplicate - existing = db.query(Artifact).filter( - Artifact.artifact_id == new_file_id, - ).first() + existing = ( + db.query(Artifact) + .filter( + Artifact.artifact_id == new_file_id, + ) + .first() + ) if existing is None: - db.add(Artifact( - artifact_id=new_file_id, - chat_id=chat_id, - user_id=user_id, - user_folder_id=_proj_folder_id, - type="other", - title=name, - filename=name, - size_bytes=max(len(content), 1), - mime_type=mime, - storage_key=ref.get("storage_key") or f"artifacts/{new_file_id}", - storage_url=ref.get("url"), - extra_data={"source": "myspace_sync"}, - )) + db.add( + Artifact( + artifact_id=new_file_id, + chat_id=chat_id, + user_id=user_id, + user_folder_id=_proj_folder_id, + type="other", + title=name, + filename=name, + size_bytes=max(len(content), 1), + mime_type=mime, + storage_key=ref.get("storage_key") or f"artifacts/{new_file_id}", + storage_url=ref.get("url"), + extra_data={"source": "myspace_sync"}, + ) + ) db.commit() except Exception as exc: # noqa: BLE001 logger.warning("[artifact-sync] DB insert failed for %s: %s", new_file_id, exc) @@ -304,7 +324,9 @@ def upsert_myspace_artifact( logger.info( "[artifact-sync] new artifact %s registered (user=%s name=%s)", - new_file_id, user_id, name, + new_file_id, + user_id, + name, ) return ref @@ -326,6 +348,7 @@ def pin_artifact_to_workspace(ref: dict[str, Any]) -> bool: return False try: from core.llm import workspace as _workspace + pinned = _workspace.pin( file_id=str(file_id), name=ref.get("name"), diff --git a/src/backend/core/llm/tools/edition_artifact_recovery.py b/src/backend/core/llm/tools/edition_artifact_recovery.py new file mode 100644 index 00000000..a03a3c5a --- /dev/null +++ b/src/backend/core/llm/tools/edition_artifact_recovery.py @@ -0,0 +1,10 @@ +"""Community Edition has no organization artifact recovery.""" + +from typing import Any, Optional + + +def recover_organization_artifact(*, file_path: str, scope: Any) -> tuple[bool, Optional[bytes]]: + return False, None + + +__all__ = ["recover_organization_artifact"] diff --git a/src/backend/core/llm/tools/edition_myspace.py b/src/backend/core/llm/tools/edition_myspace.py new file mode 100644 index 00000000..cfe0d9d5 --- /dev/null +++ b/src/backend/core/llm/tools/edition_myspace.py @@ -0,0 +1,35 @@ +"""Community Edition has no organization-scoped MySpace tools.""" + +from typing import Any, Optional + + +def project_subtree_folder_ids(db: Any, scope: Any) -> None: + return None + + +def list_organization_project_files( + db: Any, + *, + scope: Any, + folder_id: str, + mime_prefix: Optional[str], + keyword: str, + limit: int, +) -> None: + return None + + +def find_organization_project_artifact(db: Any, *, scope: Any, reference: str) -> tuple[bool, Any]: + return False, None + + +def register_organization_tools(toolkit: Any, user_id: str) -> None: + return None + + +__all__ = [ + "find_organization_project_artifact", + "list_organization_project_files", + "project_subtree_folder_ids", + "register_organization_tools", +] diff --git a/src/backend/core/llm/tools/edition_myspace_vfs.py b/src/backend/core/llm/tools/edition_myspace_vfs.py new file mode 100644 index 00000000..ab53ac17 --- /dev/null +++ b/src/backend/core/llm/tools/edition_myspace_vfs.py @@ -0,0 +1,46 @@ +"""Community Edition has no organization-scoped MySpace storage.""" + +from typing import Any, Optional + + +def organization_scope_id(scope: Any) -> Optional[str]: + return None + + +def resolve_organization_folder( + db: Any, scope: Any, folder_names: list[str], *, create: bool = False +) -> None: + return None + + +def resolve_organization_artifact( + db: Any, scope: Any, folder_id: Optional[str], filename: str +) -> None: + return None + + +def iter_organization_tree(db: Any, scope: Any, root_folder_id: Optional[str]) -> None: + return None + + +def organization_subtree_folder_ids(db: Any, scope: Any, root_folder_id: str) -> None: + return None + + +def organization_cache_file(scope: Any, rel: str) -> None: + return None + + +def organization_mutation_blocked(scope: Any) -> bool: + return False + + +__all__ = [ + "iter_organization_tree", + "organization_cache_file", + "organization_mutation_blocked", + "organization_scope_id", + "organization_subtree_folder_ids", + "resolve_organization_artifact", + "resolve_organization_folder", +] diff --git a/src/backend/core/llm/tools/myspace_tool.py b/src/backend/core/llm/tools/myspace_tool.py index a7636446..11284aec 100644 --- a/src/backend/core/llm/tools/myspace_tool.py +++ b/src/backend/core/llm/tools/myspace_tool.py @@ -4,7 +4,6 @@ ``register_myspace_tools``. """ -import base64 import json import logging from typing import Any, Optional @@ -12,8 +11,17 @@ from agentscope.message import TextBlock from agentscope.tool import Toolkit from agentscope.tool._response import ToolChunk as ToolResponse - -from core.llm.tools._tool_helpers import _resolve_artifact_files +from core.llm.tools.edition_myspace import ( + find_organization_project_artifact, + list_organization_project_files, + project_subtree_folder_ids, + register_organization_tools, +) +from core.services.artifact_edition import ( + artifact_scope_fields, + artifact_scope_folder_id, + extend_artifact_item, +) logger = logging.getLogger(__name__) @@ -26,13 +34,8 @@ def register_myspace_tools( """Register MySpace access tools for Lab code execution sessions. ``scope`` (project mode): pass a :class:`ProjectScope`. - - personal scope: ``list_myspace_files`` queries UserFolder + user_id; - ``stage_myspace_file`` validates access by user_id + user_folder_id. - - team scope: ``list_myspace_files`` queries TeamFolder + team_id; - ``stage_myspace_file`` validates access by team_id + team_folder_id. The fallback backfill - row also writes the correct ``team_id`` / ``team_folder_id`` per scope — avoiding the - earlier bug where the team path only wrote user_id, producing NULL/NULL/NULL orphans that - polluted the personal MySpace root. + - personal scopes use the shared UserFolder implementation; + - edition-specific scopes delegate to code that isn't shipped in Community Edition; - no scope (non-project conversation): personal behavior unchanged. """ if not user_id: @@ -41,11 +44,9 @@ def register_myspace_tools( import base64 as _b64 import json - from core.sandbox import ( - SandboxError as _SandboxError, - StageFile as _StageFile, - get_sandbox_provider as _get_provider, - ) + from core.sandbox import SandboxError as _SandboxError + from core.sandbox import StageFile as _StageFile + from core.sandbox import get_sandbox_provider as _get_provider def _project_subtree_folder_ids(db: Any) -> Optional[set]: """In project mode, compute the full subtree folder_id set of the linked folder (root included). @@ -57,46 +58,31 @@ def _project_subtree_folder_ids(db: Any) -> Optional[set]: root = scope.root_folder_id if not root: return set() - from core.db.models import TeamFolder, UserFolder + organization_subtree = project_subtree_folder_ids(db, scope) + if organization_subtree is not None: + return organization_subtree + + from core.db.models import UserFolder out: set = set() stack = [root] guard = 0 - if scope.is_personal: - while stack and guard < 5000: - guard += 1 - fid = stack.pop() - if fid in out: - continue - out.add(fid) - rows = ( - db.query(UserFolder.folder_id) - .filter( - UserFolder.user_id == user_id, - UserFolder.parent_folder_id == fid, - UserFolder.deleted_at.is_(None), - ) - .all() + while stack and guard < 5000: + guard += 1 + fid = stack.pop() + if fid in out: + continue + out.add(fid) + rows = ( + db.query(UserFolder.folder_id) + .filter( + UserFolder.user_id == user_id, + UserFolder.parent_folder_id == fid, + UserFolder.deleted_at.is_(None), ) - stack.extend(r[0] for r in rows) - else: - team_id = scope.team_id - while stack and guard < 5000: - guard += 1 - fid = stack.pop() - if fid in out: - continue - out.add(fid) - rows = ( - db.query(TeamFolder.folder_id) - .filter( - TeamFolder.team_id == team_id, - TeamFolder.parent_folder_id == fid, - TeamFolder.deleted_at.is_(None), - ) - .all() - ) - stack.extend(r[0] for r in rows) + .all() + ) + stack.extend(r[0] for r in rows) return out async def list_myspace_files( @@ -105,7 +91,7 @@ async def list_myspace_files( keyword: str = "", limit: int = 20, ) -> ToolResponse: - """列出"我的空间"或团队项目挂钩文件夹中的内容（子文件夹 + 文件）。 + """列出当前"我的空间"项目范围中的内容（子文件夹 + 文件）。 每次调用返回当前所在位置的直接子文件夹（`sub_folders`）和文件（`items`） —— 想浏览嵌套结构时递归调用即可（拿到 `sub_folders[i].folder_id` 后再次调用本工具）。 @@ -115,7 +101,7 @@ async def list_myspace_files( 文件夹 ID。留空（默认）即当前作用域的根： - 非项目对话：个人 MySpace 根 - 个人项目：挂钩 UserFolder - - 团队项目：挂钩 TeamFolder + - 其他版本项目：由版本扩展实现 file_type (`str`): 文件过滤：'all'（默认） / 'image' / 'document'。 keyword (`str`): @@ -132,9 +118,8 @@ async def list_myspace_files( """ try: from core.db.engine import SessionLocal - from core.db.models import Artifact, ChatSession, TeamFolder, UserFolder + from core.db.models import UserFolder from core.db.repository import ArtifactRepository - from sqlalchemy import desc, or_ limit = min(int(limit), 100) mime_prefix: Optional[str] = None @@ -143,8 +128,6 @@ async def list_myspace_files( elif file_type == "document": mime_prefix = "document" - _is_team = scope is not None and scope.is_team - db = SessionLocal() try: # Project mode: default root → linked_folder_id; any out-of-scope folder_id is rejected outright @@ -153,24 +136,21 @@ async def list_myspace_files( if not folder_id: folder_id = (scope.root_folder_id if scope is not None else "") or "" elif folder_id not in _subtree: - raise ValueError( - "项目模式下不能访问挂钩文件夹之外的文件夹" - ) + raise ValueError("项目模式下不能访问挂钩文件夹之外的文件夹") - # Current folder (None at the root) — personal queries UserFolder + user_id; team queries TeamFolder + team_id. - folder_info: Optional[Dict[str, Any]] = None - if folder_id: - if _is_team: - current = ( - db.query(TeamFolder) - .filter( - TeamFolder.folder_id == folder_id, - TeamFolder.team_id == scope.team_id, - TeamFolder.deleted_at.is_(None), - ) - .first() - ) - else: + organization_listing = list_organization_project_files( + db, + scope=scope, + folder_id=folder_id, + mime_prefix=mime_prefix, + keyword=keyword, + limit=limit, + ) + if organization_listing is not None: + folder_info, sub_folders, items_rows, total = organization_listing + else: + folder_info: Optional[dict[str, Any]] = None + if folder_id: current = ( db.query(UserFolder) .filter( @@ -180,23 +160,12 @@ async def list_myspace_files( ) .first() ) - if current is None: - raise ValueError(f"文件夹 {folder_id} 不存在") - folder_info = {"folder_id": current.folder_id, "name": current.name} - - # Direct subfolders (team queries TeamFolder + team_id; personal queries UserFolder + user_id) - if _is_team: - sub_folder_rows = ( - db.query(TeamFolder) - .filter( - TeamFolder.team_id == scope.team_id, - TeamFolder.parent_folder_id == (folder_id or None), - TeamFolder.deleted_at.is_(None), - ) - .order_by(TeamFolder.name.asc()) - .all() - ) - else: + if current is None: + raise ValueError(f"文件夹 {folder_id} 不存在") + folder_info = { + "folder_id": current.folder_id, + "name": current.name, + } sub_folder_rows = ( db.query(UserFolder) .filter( @@ -207,37 +176,12 @@ async def list_myspace_files( .order_by(UserFolder.name.asc()) .all() ) - sub_folders = [ - {"folder_id": f.folder_id, "name": f.name} - for f in sub_folder_rows - ] - - # Files at the current level — team goes by team_id+team_folder_id; personal reuses the personal repo helper. - if _is_team: - iq = ( - db.query(Artifact, ChatSession.title.label("chat_title")) - .outerjoin(ChatSession, Artifact.chat_id == ChatSession.chat_id) - .filter( - Artifact.team_id == scope.team_id, - Artifact.team_folder_id == folder_id, - Artifact.deleted_at.is_(None), - ) - ) - if mime_prefix == "image/": - iq = iq.filter(Artifact.mime_type.like("image/%")) - elif mime_prefix == "document": - iq = iq.filter(~Artifact.mime_type.like("image/%")) - if keyword: - like_pattern = f"%{keyword}%" - iq = iq.filter(or_( - Artifact.filename.ilike(like_pattern), - Artifact.title.ilike(like_pattern), - )) - total = iq.count() - rows = iq.order_by(desc(Artifact.created_at)).limit(limit).all() - items_rows = [{"artifact": a, "chat_title": ct} for a, ct in rows] - else: + sub_folders = [ + {"folder_id": folder.folder_id, "name": folder.name} + for folder in sub_folder_rows + ] from core.db.repository import ROOT_FOLDER_SENTINEL + repo = ArtifactRepository(db) items_rows, total = repo.list_by_user_with_chat( user_id=user_id, @@ -262,7 +206,7 @@ async def list_myspace_files( source = "user_upload" else: source = "ai_generated" - items.append({ + item = { "artifact_id": art.artifact_id, "name": art.filename or art.title, "title": art.title, @@ -271,10 +215,10 @@ async def list_myspace_files( "size_bytes": art.size_bytes, "source": source, "user_folder_id": art.user_folder_id, - "team_folder_id": art.team_folder_id, "chat_title": row.get("chat_title"), "created_at": art.created_at.isoformat() if art.created_at else None, - }) + } + items.append(extend_artifact_item(art, item)) payload = { "folder": folder_info, @@ -287,56 +231,47 @@ async def list_myspace_files( ) except Exception as exc: return ToolResponse( - content=[TextBlock(type="text", text=json.dumps( - {"error": str(exc)}, ensure_ascii=False - ))], + content=[ + TextBlock(type="text", text=json.dumps({"error": str(exc)}, ensure_ascii=False)) + ], ) async def stage_myspace_file(artifact_id: str) -> ToolResponse: """将"我的空间"中的文件暂存到代码执行工作区。""" try: + from core.content.artifact_refs import resolve_artifact_storage_key from core.db.engine import SessionLocal from core.db.repository import ArtifactRepository from core.storage.factory import get_storage - from core.content.artifact_refs import resolve_artifact_storage_key db = SessionLocal() try: from core.db.models import Artifact as ArtifactModel from sqlalchemy import func - _is_team = scope is not None and scope.is_team - repo = ArtifactRepository(db) - art = repo.get_by_id(artifact_id) - if not art: - # Filename fallback: switch the ownership column by scope, so files from teammates are findable in team projects. + organization_handled, art = find_organization_project_artifact( + db, scope=scope, reference=artifact_id + ) + if not organization_handled: + repo = ArtifactRepository(db) + art = repo.get_by_id(artifact_id) + if not art and not organization_handled: _q = db.query(ArtifactModel).filter( ArtifactModel.deleted_at.is_(None), func.lower(ArtifactModel.filename) == func.lower(artifact_id), + ArtifactModel.user_id == user_id, ) - if _is_team: - _q = _q.filter(ArtifactModel.team_id == scope.team_id) - else: - _q = _q.filter(ArtifactModel.user_id == user_id) art = _q.order_by(ArtifactModel.created_at.desc()).first() if art: - # Ownership check: team mode checks team_id (membership is gated at the chat - # session layer); personal mode checks user_id. - if _is_team: - if art.team_id != scope.team_id: - raise PermissionError("无权访问该文件") - else: - if art.user_id != user_id: - raise PermissionError("无权访问该文件") + if not organization_handled and art.user_id != user_id: + raise PermissionError("无权访问该文件") # Project mode: the artifact must live within the linked folder subtree _subtree2 = _project_subtree_folder_ids(db) if _subtree2 is not None: - _folder_col = art.team_folder_id if _is_team else art.user_folder_id + _folder_col = artifact_scope_folder_id(scope, art) if not _folder_col or _folder_col not in _subtree2: - raise PermissionError( - "项目模式下不能 stage 项目沙盒外的文件" - ) + raise PermissionError("项目模式下不能 stage 项目沙盒外的文件") storage_key = resolve_artifact_storage_key(art.artifact_id, art.storage_key) if not storage_key: raise ValueError(f"文件 {artifact_id} 缺少有效的存储地址") @@ -358,15 +293,14 @@ async def stage_myspace_file(artifact_id: str) -> ToolResponse: if not item: raise ValueError(f"文件 {artifact_id} 不存在或已删除") item_user = (item.get("metadata") or {}).get("user_id") - if item_user and item_user != user_id and not _is_team: - # In team mode the file may have been registered by a teammate, so a plain - # user_id check can't block it; real ownership is enforced by the later backfill's team_id validation. + if item_user and item_user != user_id and not organization_handled: raise PermissionError("无权访问该文件") filename = item.get("name") or "file" mime_type = item.get("mime_type") or "application/octet-stream" size_bytes = int(item.get("size") or 0) storage_key = resolve_artifact_storage_key( - artifact_id, item.get("storage_key"), + artifact_id, + item.get("storage_key"), ) if not storage_key: raise ValueError(f"文件 {artifact_id} 缺少有效的存储地址") @@ -376,36 +310,34 @@ async def stage_myspace_file(artifact_id: str) -> ToolResponse: try: from core.content.artifact_refs import infer_artifact_type - # Project-scope aware: team scope writes team_id+team_folder_id; - # personal scope writes user_folder_id; otherwise all NULL, landing at the root. - _bf_user_folder: Optional[str] = None - _bf_team_id: Optional[str] = None - _bf_team_folder: Optional[str] = None - if _is_team: - _bf_team_id = scope.team_id - _bf_team_folder = scope.root_folder_id or None - elif scope is not None and scope.is_personal: - _bf_user_folder = scope.root_folder_id or None - - exists = db.query(ArtifactModel.artifact_id).filter( - ArtifactModel.artifact_id == artifact_id, - ).first() + scope_fields = artifact_scope_fields(scope) + + exists = ( + db.query(ArtifactModel.artifact_id) + .filter( + ArtifactModel.artifact_id == artifact_id, + ) + .first() + ) if not exists: - db.add(ArtifactModel( - artifact_id=artifact_id, - user_id=user_id, - user_folder_id=_bf_user_folder, - team_id=_bf_team_id, - team_folder_id=_bf_team_folder, - type=infer_artifact_type(mime_type), - title=filename, filename=filename, - size_bytes=max(size_bytes, 1), - mime_type=mime_type, - storage_key=storage_key, - storage_url=f"/files/{artifact_id}", - extra_data={"source": "ai_generated", - "tool_name": "stage_myspace_file"}, - )) + db.add( + ArtifactModel( + artifact_id=artifact_id, + user_id=user_id, + type=infer_artifact_type(mime_type), + title=filename, + filename=filename, + size_bytes=max(size_bytes, 1), + mime_type=mime_type, + storage_key=storage_key, + storage_url=f"/files/{artifact_id}", + extra_data={ + "source": "ai_generated", + "tool_name": "stage_myspace_file", + }, + **scope_fields, + ) + ) db.commit() except Exception as _bf_exc: # noqa: BLE001 db.rollback() @@ -423,7 +355,8 @@ async def stage_myspace_file(artifact_id: str) -> ToolResponse: try: provider = _get_provider() staged = await provider.stage_files( - user_id, [_StageFile(name=filename, content_b64=content_b64)], + user_id, + [_StageFile(name=filename, content_b64=content_b64)], ) except _SandboxError as e: raise RuntimeError(f"暂存失败：{e}") from e @@ -442,9 +375,9 @@ async def stage_myspace_file(artifact_id: str) -> ToolResponse: ) except Exception as exc: return ToolResponse( - content=[TextBlock(type="text", text=json.dumps( - {"error": str(exc)}, ensure_ascii=False - ))], + content=[ + TextBlock(type="text", text=json.dumps({"error": str(exc)}, ensure_ascii=False)) + ], ) async def list_favorite_chats( @@ -454,8 +387,8 @@ async def list_favorite_chats( """列出"我的空间"中收藏的会话。""" try: from core.db.engine import SessionLocal - from core.db.repository import ChatSessionRepository from core.db.models import ChatMessage + from core.db.repository import ChatSessionRepository limit = min(int(limit), 50) db = SessionLocal() @@ -472,34 +405,44 @@ async def list_favorite_chats( for s in sessions: if keyword and keyword.lower() not in (s.title or "").lower(): continue - last_msg = db.query(ChatMessage).filter( - ChatMessage.chat_id == s.chat_id, - ChatMessage.role == "assistant", - ).order_by(ChatMessage.created_at.desc()).first() + last_msg = ( + db.query(ChatMessage) + .filter( + ChatMessage.chat_id == s.chat_id, + ChatMessage.role == "assistant", + ) + .order_by(ChatMessage.created_at.desc()) + .first() + ) preview = "" if last_msg: preview = (last_msg.content or "")[:200] - results.append({ - "chat_id": s.chat_id, - "title": s.title or "未命名会话", - "created_at": s.created_at.isoformat() if s.created_at else None, - "updated_at": s.updated_at.isoformat() if s.updated_at else None, - "last_message_preview": preview, - }) + results.append( + { + "chat_id": s.chat_id, + "title": s.title or "未命名会话", + "created_at": s.created_at.isoformat() if s.created_at else None, + "updated_at": s.updated_at.isoformat() if s.updated_at else None, + "last_message_preview": preview, + } + ) finally: db.close() return ToolResponse( - content=[TextBlock(type="text", text=json.dumps( - {"total": total, "items": results}, ensure_ascii=False - ))], + content=[ + TextBlock( + type="text", + text=json.dumps({"total": total, "items": results}, ensure_ascii=False), + ) + ], ) except Exception as exc: return ToolResponse( - content=[TextBlock(type="text", text=json.dumps( - {"error": str(exc)}, ensure_ascii=False - ))], + content=[ + TextBlock(type="text", text=json.dumps({"error": str(exc)}, ensure_ascii=False)) + ], ) async def get_chat_messages( @@ -509,284 +452,69 @@ async def get_chat_messages( """获取指定收藏会话的完整消息记录。""" try: from core.db.engine import SessionLocal - from core.db.models import ChatSession, ChatMessage + from core.db.models import ChatMessage, ChatSession from sqlalchemy import asc limit = min(int(limit), 200) db = SessionLocal() try: - session = db.query(ChatSession).filter( - ChatSession.chat_id == chat_id, - ChatSession.user_id == user_id, - ChatSession.deleted_at.is_(None), - ).first() + session = ( + db.query(ChatSession) + .filter( + ChatSession.chat_id == chat_id, + ChatSession.user_id == user_id, + ChatSession.deleted_at.is_(None), + ) + .first() + ) if not session: raise ValueError(f"会话 {chat_id} 不存在或无权访问") if not session.favorite: raise PermissionError("该会话未被收藏，无法读取（仅限收藏会话）") - messages = db.query(ChatMessage).filter( - ChatMessage.chat_id == chat_id, - ChatMessage.role.in_(["user", "assistant"]), - ).order_by(asc(ChatMessage.created_at)).limit(limit).all() + messages = ( + db.query(ChatMessage) + .filter( + ChatMessage.chat_id == chat_id, + ChatMessage.role.in_(["user", "assistant"]), + ) + .order_by(asc(ChatMessage.created_at)) + .limit(limit) + .all() + ) results = [] for m in messages: - results.append({ - "role": m.role, - "content": (m.content or "")[:5000], - "created_at": m.created_at.isoformat() if m.created_at else None, - }) - finally: - db.close() - - return ToolResponse( - content=[TextBlock(type="text", text=json.dumps( - {"chat_id": chat_id, "messages": results}, ensure_ascii=False - ))], - ) - except Exception as exc: - return ToolResponse( - content=[TextBlock(type="text", text=json.dumps( - {"error": str(exc)}, ensure_ascii=False - ))], - ) - - async def list_team_files( - team_id: str = "", - folder_id: str = "", - file_type: str = "all", - keyword: str = "", - limit: int = 20, - ) -> ToolResponse: - """列出用户所在团队的团队文件夹内容。 - - Args: - team_id (`str`): - 团队 ID。留空时返回当前用户所在的所有团队列表（供 Agent 选择后再次调用）。 - folder_id (`str`): - 团队文件夹 ID。留空或未提供即团队根目录。 - file_type (`str`): - 过滤类型：'all' / 'image' / 'document'。 - keyword (`str`): - 按文件名/标题模糊搜索。 - limit (`int`): - 返回条数上限，默认 20，最大 100。 - """ - try: - from core.db.engine import SessionLocal - from core.db.models import Team, TeamMember, TeamFolder - from core.db.repository import ArtifactRepository - from core.auth.permissions_iface import ( - resolve_team_file_permission, - has_permission, - ) - - limit = min(int(limit), 100) - mime_prefix: Optional[str] = None - if file_type == "image": - mime_prefix = "image/" - elif file_type == "document": - mime_prefix = "document" - - db = SessionLocal() - try: - if not team_id: - rows = ( - db.query(Team, TeamMember) - .join(TeamMember, TeamMember.team_id == Team.team_id) - .filter(TeamMember.user_id == user_id) - .order_by(Team.name.asc()) - .all() - ) - teams = [ + results.append( { - "team_id": t.team_id, - "name": t.name, - "description": t.description, "role": m.role, - "file_permission": ( - "admin" - if m.role in ("owner", "admin") - else ("edit" if m.file_permission == "editor" else "view") - ), + "content": (m.content or "")[:5000], + "created_at": m.created_at.isoformat() if m.created_at else None, } - for t, m in rows - ] - payload = { - "hint": "请指定 team_id 再次调用以查看团队文件；也可传 folder_id 浏览子目录。", - "teams": teams, - } - return ToolResponse( - content=[TextBlock(type="text", text=json.dumps(payload, ensure_ascii=False))], - ) - - perm = resolve_team_file_permission(db, user_id, team_id) - if perm == "none": - raise PermissionError("团队不存在或你未加入") - if not has_permission(perm, "view"): - raise PermissionError("当前权限不足") - - folder_info = None - folders = [] - if folder_id: - folder = ( - db.query(TeamFolder) - .filter( - TeamFolder.folder_id == folder_id, - TeamFolder.team_id == team_id, - TeamFolder.deleted_at.is_(None), - ) - .first() ) - if not folder: - raise ValueError(f"文件夹 {folder_id} 不存在") - folder_info = {"folder_id": folder.folder_id, "name": folder.name} - - sub_folders = ( - db.query(TeamFolder) - .filter( - TeamFolder.team_id == team_id, - TeamFolder.parent_folder_id == (folder_id or None), - TeamFolder.deleted_at.is_(None), - ) - .order_by(TeamFolder.name.asc()) - .all() - ) - folders = [ - {"folder_id": f.folder_id, "name": f.name} for f in sub_folders - ] - - repo = ArtifactRepository(db) - items, total = repo.list_by_team_folder( - team_id=team_id, - folder_id=folder_id or None, - mime_prefix=mime_prefix, - keyword=keyword or None, - page=1, - page_size=limit, - ) - finally: - db.close() - - results = [] - for row in items: - art = row["artifact"] - results.append({ - "artifact_id": art.artifact_id, - "name": art.filename or art.title, - "title": art.title, - "type": art.type, - "mime_type": art.mime_type, - "size_bytes": art.size_bytes, - "team_id": art.team_id, - "team_folder_id": art.team_folder_id, - "created_at": art.created_at.isoformat() if art.created_at else None, - }) - - payload = { - "team_id": team_id, - "permission": perm, - "folder": folder_info, - "sub_folders": folders, - "total": total, - "items": results, - } - return ToolResponse( - content=[TextBlock(type="text", text=json.dumps(payload, ensure_ascii=False))], - ) - except Exception as exc: - return ToolResponse( - content=[TextBlock(type="text", text=json.dumps( - {"error": str(exc)}, ensure_ascii=False - ))], - ) - - async def stage_team_file(artifact_id: str) -> ToolResponse: - """将团队文件夹中的文件暂存到代码执行工作区。需对所在团队具备 view 权限。 - - Args: - artifact_id (`str`): - 团队文件的 artifact_id（来自 list_team_files 返回）。 - """ - try: - from core.db.engine import SessionLocal - from core.db.repository import ArtifactRepository - from core.storage.factory import get_storage - from core.auth.permissions_iface import ( - resolve_artifact_access, - has_permission, - ) - from core.content.artifact_refs import resolve_artifact_storage_key - - db = SessionLocal() - try: - repo = ArtifactRepository(db) - art = repo.get_by_id(artifact_id) - if not art: - raise ValueError(f"文件 {artifact_id} 不存在或已删除") - if not art.team_id: - raise PermissionError("该文件不属于团队文件夹，请使用 stage_myspace_file") - - # owner ∪ team composite permission (same rule as the files API): the file owner - # can always access — one's own team files aren't locked out in CE / left-the-team scenarios - perm = resolve_artifact_access(db, user_id, art.user_id, art.team_id) - if perm == "none": - raise PermissionError("你不是该团队成员，无法访问该文件") - if not has_permission(perm, "view"): - raise PermissionError("当前权限不足") - - storage_key = resolve_artifact_storage_key(art.artifact_id, art.storage_key) - if not storage_key: - raise ValueError(f"文件 {artifact_id} 缺少有效的存储地址") - if storage_key != art.storage_key: - art.storage_key = storage_key - db.commit() - filename = art.filename or art.title or "file" - mime_type = art.mime_type or "application/octet-stream" - size_bytes = art.size_bytes or 0 - team_id_val = art.team_id - team_folder_id_val = art.team_folder_id finally: db.close() - storage = get_storage() - file_bytes = storage.download_bytes(storage_key) - content_b64 = _b64.b64encode(file_bytes).decode() - - try: - provider = _get_provider() - staged = await provider.stage_files( - user_id, [_StageFile(name=filename, content_b64=content_b64)], - ) - except _SandboxError as e: - raise RuntimeError(f"暂存失败：{e}") from e - - if not staged: - raise RuntimeError("暂存失败：sandbox provider 未返回路径") - - result = { - "path": staged[0].path, - "name": filename, - "size_bytes": size_bytes, - "mime_type": mime_type, - "team_id": team_id_val, - "team_folder_id": team_folder_id_val, - } return ToolResponse( - content=[TextBlock(type="text", text=json.dumps(result, ensure_ascii=False))], + content=[ + TextBlock( + type="text", + text=json.dumps( + {"chat_id": chat_id, "messages": results}, ensure_ascii=False + ), + ) + ], ) except Exception as exc: return ToolResponse( - content=[TextBlock(type="text", text=json.dumps( - {"error": str(exc)}, ensure_ascii=False - ))], + content=[ + TextBlock(type="text", text=json.dumps({"error": str(exc)}, ensure_ascii=False)) + ], ) toolkit.register_tool_function(list_myspace_files, namesake_strategy="override") toolkit.register_tool_function(stage_myspace_file, namesake_strategy="override") toolkit.register_tool_function(list_favorite_chats, namesake_strategy="override") toolkit.register_tool_function(get_chat_messages, namesake_strategy="override") - toolkit.register_tool_function(list_team_files, namesake_strategy="override") - toolkit.register_tool_function(stage_team_file, namesake_strategy="override") - logger.info("[factory] Registered 6 MySpace/Team tools for Lab session (user=%s)", user_id) + register_organization_tools(toolkit, user_id) + logger.info("[factory] Registered MySpace tools for Lab session (user=%s)", user_id) diff --git a/src/backend/core/llm/tools/myspace_vfs.py b/src/backend/core/llm/tools/myspace_vfs.py index adbd28cc..5926e777 100644 --- a/src/backend/core/llm/tools/myspace_vfs.py +++ b/src/backend/core/llm/tools/myspace_vfs.py @@ -2,8 +2,7 @@ Design goal: make ``/myspace//`` inside the sandbox a **faithful, lazily-loaded, bidirectionally synced** view of the user's real -MySpace (the ``artifacts`` table + the ``user_folders`` tree, personal scope -``team_id IS NULL``). +MySpace (the ``artifacts`` table + the ``user_folders`` tree). - **Path model**: ``/myspace/a/b/c.txt`` maps to the artifact named ``c.txt`` under the ``a/b`` folder in the UserFolder tree; physically it lands at @@ -20,7 +19,7 @@ on any specific tool directly, and is shared by the read/edit/write/delete/move tools to keep DB logic from being duplicated everywhere. -**Project scope**: every function that needs project awareness (personal/team) +**Project scope**: every function that needs project awareness takes an explicit ``scope: Optional[ProjectScope]`` parameter. **ContextVar is no longer used** — ContextVar gets reset across async generator finally boundaries, which once caused chats.py's finalizing ``_persist_artifacts`` to @@ -41,6 +40,15 @@ from typing import Any, Optional from core.config.settings import settings +from core.llm.tools.edition_myspace_vfs import ( + iter_organization_tree, + organization_cache_file, + organization_mutation_blocked, + organization_scope_id, + resolve_organization_artifact, + resolve_organization_folder, +) +from core.services.artifact_edition import personal_artifact_predicates from core.services.project_scope import ProjectScope logger = logging.getLogger(__name__) @@ -60,8 +68,8 @@ def _apply_scope_to_rel(rel: Optional[str], scope: Optional[ProjectScope]) -> Op - path is the root ``""``: return the project folder name - otherwise: ``"/"`` - Both personal and team kinds get the prefix redirect: when the frontend - starts a project conversation it confines the entry path to the anchor + Every project kind gets the prefix redirect: when the frontend starts a + project conversation it confines the entry path to the anchor folder, but the model may still think in relative names (``foo.txt``); here we uniformly re-attach such "bare paths" under the project folder. """ @@ -94,7 +102,7 @@ def myspace_rel( - ``/workspace/myspace/{uid}/a/b.txt`` → ``a/b.txt`` - non-myspace path → ``None`` - Project-scope aware: when ``scope`` is non-empty and is a personal project, + Project-scope aware: when ``scope`` is non-empty, the result is prefixed with the project anchor folder name (not repeated if already present). Thus ``/myspace/foo.txt`` in a project conversation automatically becomes ``/foo.txt``, and every path that @@ -108,13 +116,13 @@ def myspace_rel( if p == MYSPACE_LOGICAL: rel = "" elif p.startswith(MYSPACE_LOGICAL + "/"): - rel = p[len(MYSPACE_LOGICAL) + 1:] + rel = p[len(MYSPACE_LOGICAL) + 1 :] elif user_id: phys_root = f"{WORKSPACE_ROOT}/myspace/{user_id}" if p == phys_root: rel = "" elif p.startswith(phys_root + "/"): - rel = p[len(phys_root) + 1:] + rel = p[len(phys_root) + 1 :] if rel is None: return None return _apply_scope_to_rel(rel, scope) @@ -144,8 +152,8 @@ def split_rel(rel: str) -> tuple[list[str], Optional[str]]: # ────────────────────────────────────────────────────────────────────────── @dataclass class FolderResolve: - found: bool # whether every folder on the path exists (meaningful when create=False) - folder_id: Optional[str] # None = root directory + found: bool # whether every folder on the path exists (meaningful when create=False) + folder_id: Optional[str] # None = root directory def resolve_folder_id( @@ -180,6 +188,7 @@ def resolve_folder_id( if not create: return FolderResolve(found=False, folder_id=parent_id) from core.services.user_folder_service import UserFolderService + res = UserFolderService(db).create_folder( user_id=user_id, parent_folder_id=parent_id, @@ -201,8 +210,8 @@ def resolve_file_id( """Resolve a ``/myspace`` file path to an artifact_id (file_id), ``None`` if absent. Used by Read to fall back to ``fetch_parsed_text`` in the binary office - document scenario. Under team-project scope, goes through TeamFolder + team - artifact resolution (visible across members). + document scenario. Edition-specific project scopes are resolved through a + separate implementation that is absent from Community Edition. """ if not user_id: return None @@ -218,16 +227,16 @@ def resolve_file_id( return None db = SessionLocal() try: - if scope is not None and scope.is_team and scope.team_id: - fr = resolve_team_folder_id(db, scope.team_id, folder_names, create=False) - if not fr.found: - return None - art = resolve_team_artifact(db, scope.team_id, fr.folder_id, filename) - else: + organization_folder = resolve_organization_folder(db, scope, folder_names, create=False) + if organization_folder is None: fr = resolve_folder_id(db, user_id, folder_names, create=False) if not fr.found: return None art = resolve_artifact(db, user_id, fr.folder_id, filename) + else: + if not organization_folder.found: + return None + art = resolve_organization_artifact(db, scope, organization_folder.folder_id, filename) return art.artifact_id if art is not None else None finally: db.close() @@ -249,7 +258,7 @@ def resolve_artifact( q = db.query(Artifact).filter( Artifact.user_id == user_id, Artifact.filename == filename, - Artifact.team_id.is_(None), + *personal_artifact_predicates(Artifact), Artifact.deleted_at.is_(None), ) if folder_id is None: @@ -259,150 +268,6 @@ def resolve_artifact( return q.order_by(Artifact.created_at.desc()).first() -# ────────────────────────────────────────────────────────────────────────── -# Team-mode resolvers (PR 1 read-only) — symmetric to the personal functions; -# the differences are the tables + filters -# ────────────────────────────────────────────────────────────────────────── -def resolve_team_folder_id( - db: Any, - team_id: str, - folder_names: list[str], - *, - create: bool = False, -) -> FolderResolve: - """Walk the TeamFolder tree level by level by name segments and return the final folder_id (None=team root). - - PR 1 is read-only; ``create=True`` is not supported yet (the write path - opens in PR 2 once the permission gate is added). - """ - from core.db.models import TeamFolder - - if create: - # Creating team folders requires admin permission + going through - # TeamFolderService. In PR 1 the write tools already skip registration - # on the agent_factory side; refuse defensively here. - logger.warning("[myspace.team] resolve_team_folder_id(create=True) 在 PR 1 未启用") - return FolderResolve(found=False, folder_id=None) - - parent_id: Optional[str] = None - for name in folder_names: - q = db.query(TeamFolder).filter( - TeamFolder.team_id == team_id, - TeamFolder.name == name, - TeamFolder.deleted_at.is_(None), - ) - if parent_id is None: - q = q.filter(TeamFolder.parent_folder_id.is_(None)) - else: - q = q.filter(TeamFolder.parent_folder_id == parent_id) - row = q.first() - if row is None: - return FolderResolve(found=False, folder_id=parent_id) - parent_id = row.folder_id - return FolderResolve(found=True, folder_id=parent_id) - - -def resolve_team_artifact( - db: Any, - team_id: str, - folder_id: Optional[str], - filename: str, -) -> Any: - """Locate the latest live artifact by filename under the given team folder (None=root). - - Key difference (vs personal): no ``user_id`` filter — a file uploaded by - team member A should also be Read-able by member B in another chat - (cross-user visibility is the whole point of teams). - """ - from core.db.models import Artifact - - q = db.query(Artifact).filter( - Artifact.team_id == team_id, - Artifact.filename == filename, - Artifact.deleted_at.is_(None), - ) - if folder_id is None: - q = q.filter(Artifact.team_folder_id.is_(None)) - else: - q = q.filter(Artifact.team_folder_id == folder_id) - return q.order_by(Artifact.created_at.desc()).first() - - -def iter_team_tree( - db: Any, - team_id: str, - root_folder_id: Optional[str], -) -> list[tuple[str, Any]]: - """Recursively traverse all artifacts under a team folder (None=team root). - - Returns ``[(rel_path, art)]``; ``rel_path`` is relative to ``root`` and - includes the subfolder prefix. - """ - from core.db.models import Artifact, TeamFolder - - out: list[tuple[str, Any]] = [] - stack: list[tuple[Optional[str], str]] = [(root_folder_id, "")] - guard = 0 - while stack and guard < 5000: - guard += 1 - fid, prefix = stack.pop() - fq = db.query(Artifact).filter( - Artifact.team_id == team_id, - Artifact.deleted_at.is_(None), - ) - fq = fq.filter( - Artifact.team_folder_id.is_(None) - if fid is None - else Artifact.team_folder_id == fid - ) - for art in fq.all(): - if art.filename: - out.append((prefix + art.filename, art)) - sq = db.query(TeamFolder).filter( - TeamFolder.team_id == team_id, - TeamFolder.deleted_at.is_(None), - ) - sq = sq.filter( - TeamFolder.parent_folder_id.is_(None) - if fid is None - else TeamFolder.parent_folder_id == fid - ) - for sub in sq.all(): - stack.append((sub.folder_id, f"{prefix}{sub.name}/")) - return out - - -def team_subtree_folder_ids(db: Any, team_id: str, root_folder_id: str) -> set: - """Set of folder_ids of the whole subtree under the team anchor folder (root included). - - Used by read_tool's fallback — the path side is already locked inside the - project folder by ``validate_project_scope_path``; the DB layer adds another - ``team_folder_id IN subtree`` guard against privilege escalation. - """ - from core.db.models import TeamFolder - - out: set = set() - stack = [root_folder_id] - guard = 0 - while stack and guard < 5000: - guard += 1 - fid = stack.pop() - if fid in out: - continue - out.add(fid) - rows = ( - db.query(TeamFolder.folder_id) - .filter( - TeamFolder.team_id == team_id, - TeamFolder.parent_folder_id == fid, - TeamFolder.deleted_at.is_(None), - ) - .all() - ) - stack.extend(r[0] for r in rows) - return out - - # ────────────────────────────────────────────────────────────────────────── # Cache mirroring (subdirectory-aware) # ────────────────────────────────────────────────────────────────────────── @@ -413,27 +278,16 @@ def myspace_cache_file(user_id: str, rel: str) -> Path: return myspace_cache_dir(user_id) / rel -def team_cache_file(team_id: str, rel: str) -> Path: - """Team-level mirror cache file path (``team_cache/{team_id}/``, shared by members).""" - from core.sandbox._common import team_cache_dir - - return team_cache_dir(team_id) / rel - - def mirror_to_cache( user_id: str, rel: str, content: bytes, *, - team_id: Optional[str] = None, + scope: Optional[ProjectScope] = None, ) -> None: - """Mirror bytes into the backend cache (preserving subdirectories); failures only warn, never block. - - When ``team_id`` is given, lands in ``team_cache/{team_id}/`` (shared by - members); otherwise in the personal ``myspace_cache/{user_id}/``. - """ + """Mirror bytes into the edition-appropriate backend cache.""" try: - fp = team_cache_file(team_id, rel) if team_id else myspace_cache_file(user_id, rel) + fp = organization_cache_file(scope, rel) or myspace_cache_file(user_id, rel) fp.parent.mkdir(parents=True, exist_ok=True) fp.write_bytes(content) except Exception as exc: # noqa: BLE001 @@ -466,11 +320,8 @@ async def materialize_into_sandbox( pass the already-resolved ``_sess``); it is only used by ``provider.put_file`` to select the sandbox, not a DB dimension. - Under team scope (``scope.is_team`` with a team_id): goes through TeamFolder - + team artifact resolution (visible across members), materializes to the - current chat's sandbox physical path ``/workspace/myspace/{user_id}/{rel}`` - (the sandbox is per-chat=per-user anyway), and the cache lands in the team - shared directory ``team_cache/{team_id}/{rel}``. + Edition-specific scopes resolve through the edition seam and use their own + cache location. Community Edition only executes the personal branch. """ if not user_id: return None @@ -488,19 +339,18 @@ async def materialize_into_sandbox( logger.warning("[myspace] materialize deps 不可用: %s", exc) return None - team_id = scope.team_id if (scope is not None and scope.is_team) else None db = SessionLocal() try: - if team_id: - fr = resolve_team_folder_id(db, team_id, folder_names, create=False) - if not fr.found: - return None - art = resolve_team_artifact(db, team_id, fr.folder_id, filename) - else: + organization_folder = resolve_organization_folder(db, scope, folder_names, create=False) + if organization_folder is None: fr = resolve_folder_id(db, user_id, folder_names, create=False) if not fr.found: return None art = resolve_artifact(db, user_id, fr.folder_id, filename) + else: + if not organization_folder.found: + return None + art = resolve_organization_artifact(db, scope, organization_folder.folder_id, filename) if art is None: return None storage_key = str(art.storage_key) @@ -536,10 +386,12 @@ async def materialize_into_sandbox( logger.warning("[myspace] put_file 自愈失败 %s: %s", physical, exc) # Even if refilling the sandbox fails, hand the bytes back to the caller # (at least this round can read them) - mirror_to_cache(user_id, rel, data, team_id=team_id) + mirror_to_cache(user_id, rel, data, scope=scope) logger.info( "[myspace] materialized %s (artifact, %d bytes, scope=%s)", - logical_path, len(data), "team" if team_id else "personal", + logical_path, + len(data), + "organization" if organization_scope_id(scope) else "personal", ) return data @@ -567,13 +419,10 @@ def sync_upsert( in_place_update}``) or ``None`` (on sync failure the caller should soft-warn, not block the write itself). """ - if scope is not None and scope.is_team: - # PR 1 is read-only: Write/Edit tools should not be registered in team - # projects (agent_factory already gates this); add another safeguard here - # to prevent writing into personal MySpace. + if organization_mutation_blocked(scope): logger.warning( - "[myspace] sync_upsert 在 team scope 下被调用（应被 agent_factory 闸住）" - " path=%s — 拒绝以避免污染 personal MySpace", logical_path, + "[myspace] sync_upsert was blocked by the edition scope policy: %s", + logical_path, ) return None rel = myspace_rel(logical_path, user_id, scope) @@ -617,7 +466,10 @@ def sync_upsert( db.commit() logger.info( "[myspace] in-place 更新 %s (artifact=%s folder=%s %dB)", - rel, art.artifact_id, folder_id, len(content), + rel, + art.artifact_id, + folder_id, + len(content), ) return { "file_id": art.artifact_id, @@ -637,12 +489,14 @@ def sync_upsert( return None refs = _store_generated_files( - [{ - "name": name, - "size": len(content), - "content_b64": base64.b64encode(content).decode("ascii"), - "mime_type": mime, - }], + [ + { + "name": name, + "size": len(content), + "content_b64": base64.b64encode(content).decode("ascii"), + "mime_type": mime, + } + ], user_id=user_id, source="myspace_sync", extra_metadata={"chat_id": chat_id} if chat_id else None, @@ -660,24 +514,30 @@ def sync_upsert( # DB row → the file was completely invisible in MySpace. That guard is removed # here. if new_file_id: - existing = db.query(Artifact).filter( - Artifact.artifact_id == new_file_id, - ).first() + existing = ( + db.query(Artifact) + .filter( + Artifact.artifact_id == new_file_id, + ) + .first() + ) if existing is None: - db.add(Artifact( - artifact_id=new_file_id, - chat_id=chat_id, - user_id=user_id, - user_folder_id=folder_id, - type="other", - title=name, - filename=name, - size_bytes=max(len(content), 1), - mime_type=mime, - storage_key=ref.get("storage_key") or f"artifacts/{new_file_id}", - storage_url=ref.get("url"), - extra_data={"source": "myspace_sync"}, - )) + db.add( + Artifact( + artifact_id=new_file_id, + chat_id=chat_id, + user_id=user_id, + user_folder_id=folder_id, + type="other", + title=name, + filename=name, + size_bytes=max(len(content), 1), + mime_type=mime, + storage_key=ref.get("storage_key") or f"artifacts/{new_file_id}", + storage_url=ref.get("url"), + extra_data={"source": "myspace_sync"}, + ) + ) db.commit() else: # Row already exists (same-run race) → only patch the folder ownership @@ -686,7 +546,9 @@ def sync_upsert( db.commit() logger.info( "[myspace] 新建 artifact %s (rel=%s folder=%s)", - new_file_id, rel, folder_id, + new_file_id, + rel, + folder_id, ) return ref except Exception as exc: # noqa: BLE001 @@ -713,8 +575,8 @@ def sync_delete( ``{ok, kind: 'file'|'folder', removed, artifacts_affected?}`` or ``{error}``. Also cleans up the myspace_cache mirror. """ - if scope is not None and scope.is_team: - return {"error": "团队项目暂不支持通过 agent 删除文件（PR 1 只读）"} + if organization_mutation_blocked(scope): + return {"error": "当前项目范围不支持通过 agent 删除文件"} rel = myspace_rel(logical_path, user_id, scope) if rel is None or rel == "": return {"error": f"不是合法的我的空间路径或不允许删根: {logical_path}"} @@ -745,15 +607,20 @@ def sync_delete( fr2 = resolve_folder_id(db, user_id, all_names, create=False) if fr2.found and fr2.folder_id: from core.services.user_folder_service import UserFolderService + res, affected = UserFolderService(db).delete_folder(fr2.folder_id, user_id) if res.ok: _remove_cache(user_id, rel, is_dir=True) logger.info( "[myspace] 软删文件夹 %s (folder=%s, %d 文件)", - rel, fr2.folder_id, affected, + rel, + fr2.folder_id, + affected, ) return { - "ok": True, "kind": "folder", "removed": rel, + "ok": True, + "kind": "folder", + "removed": rel, "artifacts_affected": affected, } return {"error": res.message} @@ -785,8 +652,8 @@ def sync_move( Returns ``{ok, kind, src, dst}`` or ``{error}``. """ - if scope is not None and scope.is_team: - return {"error": "团队项目暂不支持通过 agent 移动文件（PR 1 只读）"} + if organization_mutation_blocked(scope): + return {"error": "当前项目范围不支持通过 agent 移动文件"} src_rel = myspace_rel(src_path, user_id, scope) dst_rel = myspace_rel(dst_path, user_id, scope) if not src_rel: @@ -868,8 +735,8 @@ def sync_mkdir( ``created=False`` means the folder already existed (idempotent success, not an error). """ - if scope is not None and scope.is_team: - return {"error": "团队项目暂不支持通过 agent 创建文件夹（PR 1 只读）"} + if organization_mutation_blocked(scope): + return {"error": "当前项目范围不支持通过 agent 创建文件夹"} rel = myspace_rel(logical_path, user_id, scope) if rel is None: return {"error": f"不是合法我的空间路径: {logical_path}"} @@ -896,7 +763,10 @@ def sync_mkdir( return {"error": f"创建文件夹失败: {rel}"} logger.info("[myspace] 创建文件夹 %s (created=%s)", rel, not already) return { - "ok": True, "kind": "folder", "path": rel, "created": not already, + "ok": True, + "kind": "folder", + "path": rel, + "created": not already, } except Exception as exc: # noqa: BLE001 db.rollback() @@ -927,13 +797,11 @@ def iter_tree( fid, prefix = stack.pop() fq = db.query(Artifact).filter( Artifact.user_id == user_id, - Artifact.team_id.is_(None), + *personal_artifact_predicates(Artifact), Artifact.deleted_at.is_(None), ) fq = fq.filter( - Artifact.user_folder_id.is_(None) - if fid is None - else Artifact.user_folder_id == fid + Artifact.user_folder_id.is_(None) if fid is None else Artifact.user_folder_id == fid ) for art in fq.all(): if art.filename: @@ -958,17 +826,15 @@ def _resolve_root( root_logical: str, scope: Optional[ProjectScope], ) -> Optional[FolderResolve]: - """Resolve a directory-like logical path to the root folder (the whole string is interpreted as folders). - - Under team scope, walks the TeamFolder tree (when ``scope.is_team`` with a team_id). - """ + """Resolve a directory-like logical path to the current scope root.""" rel = myspace_rel(root_logical, user_id, scope) if rel is None: return None rel = rel.strip("/") names = [s for s in rel.split("/") if s] if rel else [] - if scope is not None and scope.is_team and scope.team_id: - return resolve_team_folder_id(db, scope.team_id, names, create=False) + organization_folder = resolve_organization_folder(db, scope, names, create=False) + if organization_folder is not None: + return organization_folder return resolve_folder_id(db, user_id, names, create=False) @@ -978,7 +844,7 @@ def glob_tree( pattern: str, scope: Optional[ProjectScope] = None, ) -> Optional[list[str]]: - """Glob-match files in the MySpace / team anchor folder tree, returning a + """Glob-match files in the current MySpace anchor folder tree, returning a list of ``/myspace/...`` logical paths. - contains ``**`` → match the full relative path across subdirectories. @@ -1001,10 +867,11 @@ def glob_tree( fr = _resolve_root(db, user_id, root_logical, scope) if fr is None or not fr.found: return [] - if scope is not None and scope.is_team and scope.team_id: - entries = iter_team_tree(db, scope.team_id, fr.folder_id) - else: + organization_entries = iter_organization_tree(db, scope, fr.folder_id) + if organization_entries is None: entries = iter_tree(db, user_id, fr.folder_id) + else: + entries = organization_entries finally: db.close() @@ -1013,9 +880,7 @@ def glob_tree( hits: list[str] = [] for rel_path, _art in entries: if recursive: - if fnmatch.fnmatch(rel_path, pat) or fnmatch.fnmatch( - rel_path, pat.lstrip("*/") - ): + if fnmatch.fnmatch(rel_path, pat) or fnmatch.fnmatch(rel_path, pat.lstrip("*/")): hits.append(f"{base}/{rel_path}") else: if "/" in rel_path: @@ -1029,10 +894,42 @@ def glob_tree( # even when materialized — they only slow things down and flood output. materialize_tree # pulls only these text-like extensions. _TEXT_EXT = { - ".txt", ".md", ".markdown", ".csv", ".tsv", ".json", ".jsonl", ".log", - ".py", ".js", ".ts", ".tsx", ".jsx", ".html", ".htm", ".xml", ".yaml", - ".yml", ".toml", ".ini", ".cfg", ".conf", ".sh", ".bash", ".sql", ".css", - ".scss", ".java", ".go", ".rs", ".c", ".h", ".cpp", ".rb", ".php", ".env", + ".txt", + ".md", + ".markdown", + ".csv", + ".tsv", + ".json", + ".jsonl", + ".log", + ".py", + ".js", + ".ts", + ".tsx", + ".jsx", + ".html", + ".htm", + ".xml", + ".yaml", + ".yml", + ".toml", + ".ini", + ".cfg", + ".conf", + ".sh", + ".bash", + ".sql", + ".css", + ".scss", + ".java", + ".go", + ".rs", + ".c", + ".h", + ".cpp", + ".rb", + ".php", + ".env", } @@ -1075,16 +972,16 @@ async def materialize_tree( except Exception: # noqa: BLE001 return 0 - team_id = scope.team_id if (scope is not None and scope.is_team) else None db = SessionLocal() try: fr = _resolve_root(db, user_id, root_logical, scope) if fr is None or not fr.found: return 0 - if team_id: - raw_entries = iter_team_tree(db, team_id, fr.folder_id) - else: + organization_entries = iter_organization_tree(db, scope, fr.folder_id) + if organization_entries is None: raw_entries = iter_tree(db, user_id, fr.folder_id) + else: + raw_entries = organization_entries entries = [(rp, art) for rp, art in raw_entries if _is_text_name(rp)] finally: db.close() @@ -1102,22 +999,20 @@ async def _pull(rel_path: str, art: Any) -> None: physical = f"{WORKSPACE_ROOT}/myspace/{user_id}/{full_rel}" async with sem: try: - data = await asyncio.to_thread( - storage.download_bytes, str(art.storage_key) - ) + data = await asyncio.to_thread(storage.download_bytes, str(art.storage_key)) await provider.put_file(chat_id, physical, data, user_id=user_id) - mirror_to_cache(user_id, full_rel, data, team_id=team_id) + mirror_to_cache(user_id, full_rel, data, scope=scope) done += 1 except Exception as exc: # noqa: BLE001 - logger.warning( - "[myspace] materialize_tree 跳过 %s: %s", full_rel, exc - ) + logger.warning("[myspace] materialize_tree 跳过 %s: %s", full_rel, exc) if entries: await asyncio.gather(*(_pull(rp, a) for rp, a in entries)) logger.info( "[myspace] materialize_tree %s → %d 文本文件物化（候选文本 %d）", - root_logical, done, total_text, + root_logical, + done, + total_text, ) return done diff --git a/src/backend/core/llm/tools/read_tool.py b/src/backend/core/llm/tools/read_tool.py index 0b401cd2..a849c5af 100644 --- a/src/backend/core/llm/tools/read_tool.py +++ b/src/backend/core/llm/tools/read_tool.py @@ -19,7 +19,6 @@ from typing import Optional from agentscope.tool import Toolkit - from core.services.project_scope import ProjectScope from . import myspace_vfs as _ms @@ -33,6 +32,7 @@ validate_workspace_path, ) from ._state import ReadEntry, ReadStateTracker +from .edition_artifact_recovery import recover_organization_artifact logger = logging.getLogger(__name__) @@ -50,7 +50,9 @@ def _is_binary(blob: bytes) -> bool: def _format_with_line_numbers( - text: str, start_line: int, end_line: int, + text: str, + start_line: int, + end_line: int, ) -> str: """``cat -n`` style line numbers: `` 1\\tcontent``.""" lines = text.splitlines() @@ -70,13 +72,9 @@ def _fallback_recover_from_artifact( scope: Optional[ProjectScope], ) -> Optional[bytes]: """If the file is missing from the sandbox, try recovering it from the - artifact storage. Two scopes with different strategies: - - - **personal** (default / personal project): same-named artifact with the same - chat and same user. - - **team** (team-project scope): drop the user_id filter and look up a - same-named artifact by ``team_id`` + the folder subtree bound to that - project. Visible across members. + artifact storage. Edition-specific project scopes are handled by their own + implementation; the shared fallback is restricted to the current user and + chat. Returns the recovered bytes on hit, ``None`` otherwise. """ @@ -84,6 +82,12 @@ def _fallback_recover_from_artifact( if not fname: return None + handled, data = recover_organization_artifact(file_path=file_path, scope=scope) + if handled: + return data + + if not chat_id or not user_id: + return None try: from core.db.engine import SessionLocal from core.db.models import Artifact @@ -91,64 +95,6 @@ def _fallback_recover_from_artifact( except Exception as exc: logger.warning("[read.fallback] deps unavailable: %s", exc) return None - - # Team scope first: when scope exists and kind=="team", use the cross-member team query - if scope is not None and scope.is_team: - team_id = scope.team_id - root_folder_id = scope.root_folder_id - if not team_id or not root_folder_id: - logger.info( - "[read.fallback] team scope missing team_id/folder_id name=%s", - fname, - ) - return None - db = SessionLocal() - try: - allowed = _ms.team_subtree_folder_ids(db, team_id, root_folder_id) - if not allowed: - logger.info( - "[read.fallback] team subtree empty team=%s root=%s", - team_id, root_folder_id, - ) - return None - row = ( - db.query(Artifact) - .filter( - Artifact.team_id == team_id, - Artifact.team_folder_id.in_(allowed), - Artifact.filename == fname, - Artifact.deleted_at.is_(None), - ) - .order_by(Artifact.created_at.desc()) - .first() - ) - if row is None: - logger.info( - "[read.fallback] no team artifact match team=%s root=%s name=%s", - team_id, root_folder_id, fname, - ) - return None - storage_key = str(row.storage_key) - artifact_id = row.artifact_id - finally: - db.close() - try: - data = get_storage().download_bytes(storage_key) - logger.info( - "[read.fallback] recovered %s from team artifact %s (%d bytes)", - file_path, artifact_id, len(data), - ) - return data - except Exception as exc: # noqa: BLE001 - logger.warning( - "[read.fallback] download_bytes failed (team) key=%s: %s", - storage_key, exc, - ) - return None - - # Personal scope / no scope: keep the original behavior - if not chat_id or not user_id: - return None db = SessionLocal() try: row = ( @@ -165,20 +111,25 @@ def _fallback_recover_from_artifact( if row is None: logger.info( "[read.fallback] no artifact match chat=%s user=%s name=%s", - chat_id, user_id, fname, + chat_id, + user_id, + fname, ) return None try: data = get_storage().download_bytes(str(row.storage_key)) logger.info( "[read.fallback] recovered %s from artifact %s (%d bytes)", - file_path, row.artifact_id, len(data), + file_path, + row.artifact_id, + len(data), ) return data except Exception as exc: # noqa: BLE001 logger.warning( "[read.fallback] download_bytes failed for storage_key=%s: %s", - row.storage_key, exc, + row.storage_key, + exc, ) return None finally: @@ -209,11 +160,9 @@ async def Read( offset: int = 0, limit: int = 0, ) -> "ToolResponse": # type: ignore[name-defined] - from core.sandbox import ( - SandboxConnectError as _SCE, - SandboxError as _SE, - get_sandbox_provider as _get_provider, - ) + from core.sandbox import SandboxConnectError as _SCE + from core.sandbox import SandboxError as _SE + from core.sandbox import get_sandbox_provider as _get_provider path_err = validate_workspace_path(file_path) if path_err: @@ -238,7 +187,11 @@ async def Read( # mirror cache) try: data = await _ms.materialize_into_sandbox( - provider, _sess, user_id, file_path, scope=scope, + provider, + _sess, + user_id, + file_path, + scope=scope, ) except Exception as mexc: # noqa: BLE001 logger.warning("[read] myspace 懒加载失败 %s: %s", file_path, mexc) @@ -256,7 +209,8 @@ async def Read( except (_SE, _SCE) as put_exc: logger.warning( "[read.fallback] put_file %s failed (continuing): %s", - physical, put_exc, + physical, + put_exc, ) if data is None: return resp_json({"error": f"读取失败: {exc}"}) @@ -275,38 +229,43 @@ async def Read( fid = _ms.resolve_file_id(user_id, file_path, scope=scope) if fid: from core.content.artifact_reader import fetch_parsed_text + pt = fetch_parsed_text(fid, user_id) if pt: parsed_text = pt except Exception as pexc: # noqa: BLE001 logger.warning("[read] 解析文本回退失败 %s: %s", file_path, pexc) if parsed_text is None: - return resp_json({ - "type": "binary", - "file_path": file_path, - "physical_path": physical, - "size": len(content_bytes), - "hint": ( - "文件是二进制（如 docx/xlsx/pdf/图片），且不在「我的空间」" - "或无法解析。如需让用户下载，使用 sandbox_get_artifact" - "(src_path)；如需在沙盒内处理，用 bash 调用命令行工具。" - ), - }) + return resp_json( + { + "type": "binary", + "file_path": file_path, + "physical_path": physical, + "size": len(content_bytes), + "hint": ( + "文件是二进制（如 docx/xlsx/pdf/图片），且不在「我的空间」" + "或无法解析。如需让用户下载，使用 sandbox_get_artifact" + "(src_path)；如需在沙盒内处理，用 bash 调用命令行工具。" + ), + } + ) # Continue the paginated rendering with the parsed text instead of the raw bytes content_bytes = parsed_text.encode("utf-8") parsed_fallback = True if len(content_bytes) > MAX_TEXT_BYTES: - return resp_json({ - "type": "too_large", - "file_path": file_path, - "physical_path": physical, - "size": len(content_bytes), - "hint": ( - f"文件超过 {MAX_TEXT_BYTES} 字节，请用 bash 的 head/tail/sed" - "切片，或用 offset/limit 参数分段读取。" - ), - }) + return resp_json( + { + "type": "too_large", + "file_path": file_path, + "physical_path": physical, + "size": len(content_bytes), + "hint": ( + f"文件超过 {MAX_TEXT_BYTES} 字节，请用 bash 的 head/tail/sed" + "切片，或用 offset/limit 参数分段读取。" + ), + } + ) try: text = content_bytes.decode("utf-8") @@ -332,7 +291,7 @@ async def Read( # Slice and render into a line-numbered string # Note _format_with_line_numbers expects the substring starting at line `start` - selected = "\n".join(all_lines[start - 1:end]) + selected = "\n".join(all_lines[start - 1 : end]) numbered = _format_with_line_numbers(selected, start, end) truncated = end < total_lines @@ -394,7 +353,7 @@ async def Read( if truncated: payload["hint"] = ( f"已截断（显示 {start}-{end}/{total_lines} 行）。" - f"继续读后续：Read(file_path=\"{file_path}\", offset={end + 1})" + f'继续读后续：Read(file_path="{file_path}", offset={end + 1})' ) if recovered_from_artifact: payload["recovered_from_artifact"] = True @@ -405,8 +364,7 @@ async def Read( return resp_json(payload) Read.__doc__ = ( - "读取文本文件，返回带行号的内容（``cat -n`` 风格）。\n\n" - + PATH_POLICY_DOC + "\n\n" + "读取文本文件，返回带行号的内容（``cat -n`` 风格）。\n\n" + PATH_POLICY_DOC + "\n\n" "本工具说明：\n" "- 默认读沙盒里的文件（``/workspace/...``）。\n" "- 当用户提到他「我的空间」里的文件时，可直接读 ``/myspace/...``：\n" diff --git a/src/backend/core/memory/context.py b/src/backend/core/memory/context.py index 408cc69c..b782529f 100644 --- a/src/backend/core/memory/context.py +++ b/src/backend/core/memory/context.py @@ -10,7 +10,6 @@ from dataclasses import dataclass, field from typing import Literal, Optional - Confidentiality = Literal["public", "internal", "sensitive"] @@ -44,10 +43,8 @@ class MemoryContext: # caller at the workflow layer (whether to invoke the retrieve path) and is # not part of ctx write_enabled: bool = False - # mem0 scope identifier. Under a team project = "team:", all members - # share reads and writes; default / personal projects = None, falling back to - # user_id. audit / metadata.author_user_id still record the real user_id so - # audits stay traceable. + # Optional edition scope identifier. Default and personal projects use None, + # falling back to user_id. Audit metadata still records the real user_id. scope_user_id: Optional[str] = None @property @@ -56,7 +53,7 @@ def effective_actor(self) -> str: @property def effective_scope_user_id(self) -> str: - """The user_id passed into mem0. Team scope under team projects, otherwise the real user.""" + """Return the memory scope identifier, or the real user by default.""" return self.scope_user_id or self.user_id def with_confidentiality(self, level: Confidentiality) -> "MemoryContext": diff --git a/src/backend/core/sandbox/_common.py b/src/backend/core/sandbox/_common.py index e6a0bd95..cbc262e8 100644 --- a/src/backend/core/sandbox/_common.py +++ b/src/backend/core/sandbox/_common.py @@ -11,29 +11,61 @@ import os import re from pathlib import Path -from shlex import quote as shell_escape # re-export so providers can `from ._common import shell_escape` +from shlex import ( + quote as shell_escape, # re-export so providers can `from ._common import shell_escape` +) from core.config.settings import settings __all__ = [ - "ALLOWED_EXTENSIONS", "INTERPRETER_CMD", "MAX_FILE_COUNT", "MAX_FILE_SIZE", - "MAX_OUTPUT_BYTES", "MAX_STDERR_BYTES", "MAX_TOTAL_FILE_SIZE", - "STDIN_FILE", "USER_ID_RE", "WORKSPACE", - "myspace_cache_dir", "team_cache_dir", "dws_cache_dir", "dws_home_dir", + "ALLOWED_EXTENSIONS", + "INTERPRETER_CMD", + "MAX_FILE_COUNT", + "MAX_FILE_SIZE", + "MAX_OUTPUT_BYTES", + "MAX_STDERR_BYTES", + "MAX_TOTAL_FILE_SIZE", + "STDIN_FILE", + "USER_ID_RE", + "WORKSPACE", + "myspace_cache_dir", + "dws_cache_dir", + "dws_home_dir", "dws_extra_envs", - "lark_cache_dir", "lark_home_dir", "lark_app_home_dir", - "email_cache_dir", "email_home_dir", "email_himalaya_config", - "yida_cache_dir", "yida_workspace_dir", "yida_shared_workspace_dir", - "safe_user_id", "shell_escape", + "lark_cache_dir", + "lark_home_dir", + "lark_app_home_dir", + "email_cache_dir", + "email_home_dir", + "email_himalaya_config", + "yida_cache_dir", + "yida_workspace_dir", + "yida_shared_workspace_dir", + "safe_user_id", + "shell_escape", ] logger = logging.getLogger(__name__) # Kept aligned with services/script_runner_service/server.py ALLOWED_EXTENSIONS = { - ".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", - ".csv", ".xlsx", ".xls", ".json", ".txt", ".pdf", - ".html", ".htm", ".docx", ".pptx", ".md", + ".png", + ".jpg", + ".jpeg", + ".gif", + ".svg", + ".webp", + ".csv", + ".xlsx", + ".xls", + ".json", + ".txt", + ".pdf", + ".html", + ".htm", + ".docx", + ".pptx", + ".md", } MAX_FILE_SIZE = 10 * 1024 * 1024 MAX_TOTAL_FILE_SIZE = 20 * 1024 * 1024 @@ -58,6 +90,7 @@ def safe_user_id(user_id: str | None) -> str: Uniform replacement for the scattered ``uid if uid and USER_ID_RE.match(uid) else ""`` idiom.""" return user_id if user_id and USER_ID_RE.match(user_id) else "" + INTERPRETER_CMD = { "python": "python3 -u", "javascript": "node", @@ -260,13 +293,3 @@ def yida_shared_workspace_dir() -> Path: backend uid 1000; cross-uid writes need wide-open permissions). """ return settings.storage.root / "yida_cache" / "__shared__" / "workspace" - - -def team_cache_dir(team_id: str) -> Path: - """Backend-local team file cache directory (shared among members of the same team). - - Bytes the agent reads via ``/myspace//...`` in team projects try to - hit this cache first, then fall back to object storage; different members and - different chats of the same team share one cache, reducing S3/OSS egress traffic. - """ - return settings.storage.root / "team_cache" / team_id diff --git a/src/backend/core/services/__init__.py b/src/backend/core/services/__init__.py index 373e4868..bd408101 100644 --- a/src/backend/core/services/__init__.py +++ b/src/backend/core/services/__init__.py @@ -1,32 +1,19 @@ -"""Business logic layer - Service classes. +"""Community-edition service exports.""" -Re-exports all service classes for backwards compatibility with -``from core.services import UserService`` etc. -""" - -from core.services.user_service import UserService -from core.services.chat_service import ChatService +from core.services.api_key_service import ApiKeyService +from core.services.artifact_service import ArtifactService from core.services.catalog_service import CatalogService +from core.services.chat_service import ChatService from core.services.kb_service import KBService -from core.services.kb_permission_service import KBPermissionService -from core.services.artifact_service import ArtifactService from core.services.user_agent_service import UserAgentService -from core.services.api_key_service import ApiKeyService -from core.services.role_service import RoleService - -# Note: ProjectService / ProjectFileService are not exported here, because project_service.py -# depends on core.auth.project_permissions (which depends on core.auth.backend, and backend in turn -# imports this module → circular import). So import them directly via ``from core.services.project_service import -# ProjectService``. +from core.services.user_service import UserService __all__ = [ - "UserService", - "ChatService", + "ApiKeyService", + "ArtifactService", "CatalogService", + "ChatService", "KBService", - "KBPermissionService", - "ArtifactService", "UserAgentService", - "ApiKeyService", - "RoleService", + "UserService", ] diff --git a/src/backend/core/services/artifact_edition.py b/src/backend/core/services/artifact_edition.py new file mode 100644 index 00000000..4e8bc5c8 --- /dev/null +++ b/src/backend/core/services/artifact_edition.py @@ -0,0 +1,62 @@ +"""Community artifact lists are always personal.""" + +from typing import Any, Optional + + +def artifact_list_scope() -> str: + return "personal" + + +def extend_artifact_item(artifact, item: dict) -> dict: + return item + + +def personal_artifact_predicates(artifact_model: Any) -> list[Any]: + return [] + + +def is_personal_artifact(artifact: Any) -> bool: + return True + + +def personal_artifact_create_fields() -> dict[str, None]: + return {} + + +def artifact_scope_fields(scope: Any) -> dict[str, Optional[str]]: + return { + "user_folder_id": ( + scope.root_folder_id or None if scope is not None and scope.is_personal else None + ) + } + + +def artifact_scope_folder_id(scope: Any, artifact: Any) -> Optional[str]: + return artifact.user_folder_id + + +def can_access_artifact(db: Any, user_id: str, artifact: Any) -> bool: + return str(artifact.user_id) == str(user_id) + + +def artifact_access_metadata(artifact: Any) -> dict[str, Optional[str]]: + return {"owner_id": artifact.user_id} + + +def can_access_artifact_metadata(db: Any, user_id: str, metadata: dict[str, Any]) -> bool: + owner_id = metadata.get("owner_id") or metadata.get("user_id") + return owner_id is None or str(owner_id) == str(user_id) + + +__all__ = [ + "artifact_access_metadata", + "artifact_list_scope", + "artifact_scope_fields", + "artifact_scope_folder_id", + "can_access_artifact", + "can_access_artifact_metadata", + "extend_artifact_item", + "is_personal_artifact", + "personal_artifact_create_fields", + "personal_artifact_predicates", +] diff --git a/src/backend/core/services/artifact_service.py b/src/backend/core/services/artifact_service.py index bf969a33..5598fbb2 100644 --- a/src/backend/core/services/artifact_service.py +++ b/src/backend/core/services/artifact_service.py @@ -3,13 +3,13 @@ import logging import os import uuid -from typing import Any, Dict, List, Optional, TYPE_CHECKING - -from sqlalchemy.orm import Session +from typing import TYPE_CHECKING, Any, Dict, List, Optional from core.content.artifact_refs import infer_artifact_type, resolve_artifact_storage_key from core.db.models import Artifact as ArtifactModel from core.db.repository import ArtifactRepository +from core.services.artifact_edition import artifact_scope_fields +from sqlalchemy.orm import Session if TYPE_CHECKING: from core.services.project_scope import ProjectScope @@ -83,7 +83,7 @@ def create_artifact( size_bytes: int, mime_type: str, storage_key: str, - chat_id: Optional[str] = None + chat_id: Optional[str] = None, ) -> Dict[str, Any]: """Create a new artifact.""" artifact_data = { @@ -95,7 +95,7 @@ def create_artifact( "filename": filename, "size_bytes": size_bytes, "mime_type": mime_type, - "storage_key": storage_key + "storage_key": storage_key, } artifact = self.repo.create(artifact_data) @@ -105,7 +105,7 @@ def create_artifact( "type": artifact.type, "title": artifact.title, "filename": artifact.filename, - "created_at": artifact.created_at.isoformat() + "created_at": artifact.created_at.isoformat(), } def get_artifact(self, artifact_id: str, user_id: str) -> Optional[Dict[str, Any]]: @@ -123,7 +123,7 @@ def get_artifact(self, artifact_id: str, user_id: str) -> Optional[Dict[str, Any "storage_key": artifact.storage_key, "size_bytes": artifact.size_bytes, "mime_type": artifact.mime_type, - "created_at": artifact.created_at.isoformat() + "created_at": artifact.created_at.isoformat(), } @@ -152,58 +152,58 @@ def persist_artifacts( User-uploaded files take a separate path (`api/routes/v1/file_upload.py`, extra_data.source = "user_upload") and never go through this helper. - Project-mode auto-placement: ``scope`` is passed in explicitly. - - ``scope.is_personal``: the artifact automatically lands in - ``scope.root_folder_id`` (UserFolder), without relying on the LLM to - pass a path explicitly. - - ``scope.is_team``: the artifact is written with ``team_id`` + - ``team_folder_id`` (TeamFolder subtree), so all team-project members can - see the output — no more leaking into the personal MySpace root. - - ``scope is None``: non-project chat; lands in the MySpace root - (``user_folder_id=NULL``). + Project-mode auto-placement is delegated to the edition scope seam. The + shared implementation only knows the personal folder field; commercial + ownership columns are supplied by the enterprise module. **Important**: ``scope`` must be constructed explicitly by the caller from the workflow context and passed in. This function no longer reads the ContextVar — on the old path, by the time chats.py made the wrap-up call, - workflow.py's finally had already reset the ContextVar, causing team-project - AI outputs to leak into the personal MySpace root (trace 9d218075…). + workflow.py's finally had already reset the ContextVar, causing project + outputs to land in the wrong scope. """ if not collected: return - user_folder_id_val: Optional[str] = None - team_id_val: Optional[str] = None - team_folder_id_val: Optional[str] = None - if scope is not None and scope.is_team: - team_id_val = scope.team_id - team_folder_id_val = scope.root_folder_id or None - elif scope is not None and scope.is_personal: - user_folder_id_val = scope.root_folder_id or None + scope_fields = artifact_scope_fields(scope) all_fids = [a["file_id"] for a in collected if a.get("file_id")] - existing_ids = set( - r[0] for r in db.query(ArtifactModel.artifact_id) - .filter(ArtifactModel.artifact_id.in_(all_fids)).all() - ) if all_fids else set() + existing_ids = ( + set( + r[0] + for r in db.query(ArtifactModel.artifact_id) + .filter(ArtifactModel.artifact_id.in_(all_fids)) + .all() + ) + if all_fids + else set() + ) for art in collected: art_id = art.get("file_id", "") if not art_id or art_id in existing_ids: continue mime = art.get("mime_type", "application/octet-stream") try: - storage_key = resolve_artifact_storage_key(art_id, art.get("storage_key")) or f"artifacts/{art_id}" - db.add(ArtifactModel( - artifact_id=art_id, chat_id=chat_id, user_id=user_id, - user_folder_id=user_folder_id_val, - team_id=team_id_val, - team_folder_id=team_folder_id_val, - type=infer_artifact_type(mime), - title=art.get("name", ""), filename=art.get("name", ""), - size_bytes=max(art.get("size", 0) or 0, 1), - mime_type=mime, storage_key=storage_key, - storage_url=art.get("url", ""), - extra_data={"source": "ai_generated", "tool_name": art.get("tool_name", "")}, - )) + storage_key = ( + resolve_artifact_storage_key(art_id, art.get("storage_key")) + or f"artifacts/{art_id}" + ) + db.add( + ArtifactModel( + artifact_id=art_id, + chat_id=chat_id, + user_id=user_id, + type=infer_artifact_type(mime), + title=art.get("name", ""), + filename=art.get("name", ""), + size_bytes=max(art.get("size", 0) or 0, 1), + mime_type=mime, + storage_key=storage_key, + storage_url=art.get("url", ""), + extra_data={"source": "ai_generated", "tool_name": art.get("tool_name", "")}, + **scope_fields, + ) + ) except Exception as e: logger.warning("artifact_db_insert_failed: %s", e) try: diff --git a/src/backend/core/services/chat_edition.py b/src/backend/core/services/chat_edition.py new file mode 100644 index 00000000..e0e73340 --- /dev/null +++ b/src/backend/core/services/chat_edition.py @@ -0,0 +1,12 @@ +"""Community chat presentation: owner-only fields and state.""" + + +def extend_session_view(db, session, user_id: str, level: str, base: dict) -> dict: + return base + + +def update_member_state(db, session, user_id: str, *, pinned=None, favorite=None) -> bool: + return False + + +__all__ = ["extend_session_view", "update_member_state"] diff --git a/src/backend/core/services/chat_service.py b/src/backend/core/services/chat_service.py index f0c6794a..e037bd8b 100644 --- a/src/backend/core/services/chat_service.py +++ b/src/backend/core/services/chat_service.py @@ -1,18 +1,14 @@ """Chat session and message business logic.""" -from typing import Optional, List, Dict, Any, Tuple -from datetime import datetime import uuid -from sqlalchemy.orm import Session +from datetime import datetime +from typing import Any, Dict, List, Optional, Tuple -from core.auth.permissions_iface import ( - ChatAccessLevel, - can_delete_session, - resolve_chat_access, -) -from core.db.repository import ChatSessionRepository, ChatMessageRepository, AuditLogRepository -from core.db.models import ChatSession, ChatMessage +from core.auth.permissions_iface import ChatAccessLevel, can_delete_session, resolve_chat_access +from core.db.models import ChatMessage, ChatSession +from core.db.repository import AuditLogRepository, ChatMessageRepository, ChatSessionRepository from core.ontology.revision import is_substantive_revision, normalize_revision_candidate +from sqlalchemy.orm import Session class ChatService: @@ -29,7 +25,7 @@ def create_session( user_id: str, title: str = "新对话", extra_data: Dict = None, - chat_id: Optional[str] = None + chat_id: Optional[str] = None, ) -> ChatSession: """Create a new chat session. @@ -47,18 +43,20 @@ def create_session( "chat_id": chat_id or f"chat_{uuid.uuid4().hex[:16]}", "user_id": user_id, "title": title, - "extra_data": extra_data or {} + "extra_data": extra_data or {}, } session = self.session_repo.create(session_data) # Audit log - self.audit_repo.create({ - "user_id": user_id, - "action": "chat.session.created", - "resource_type": "chat_session", - "resource_id": session.chat_id, - "status": "success" - }) + self.audit_repo.create( + { + "user_id": user_id, + "action": "chat.session.created", + "resource_type": "chat_session", + "resource_id": session.chat_id, + "status": "success", + } + ) return session @@ -115,7 +113,11 @@ def list_sessions( ) -> Tuple[List[ChatSession], int, int]: """List chat sessions with pagination.""" sessions, total = self.session_repo.list_by_user( - user_id, page, page_size, pinned_only, favorite_only, + user_id, + page, + page_size, + pinned_only, + favorite_only, exclude_automation=exclude_automation, ) @@ -126,9 +128,8 @@ def list_sessions( def get_session(self, chat_id: str, user_id: str) -> Optional[ChatSession]: """Get chat session with ownership check. - Historical semantics unchanged: only the owner can get the session. For **team-project - sharing scenarios**, use :py:meth:`get_session_with_access` instead, which decides based - on share_scope + project permissions. + Historical semantics unchanged: only the owner can get the session. For + edition-specific sharing, use :py:meth:`get_session_with_access`. """ session = self.session_repo.get_by_id(chat_id) @@ -177,43 +178,42 @@ def update_session_fields( merged.update(extra_patch) normalized["extra_data"] = merged updated = self.session_repo.update(chat_id, normalized) - self.audit_repo.create({ - "user_id": actor_user_id or session.user_id, - "action": "chat.session.updated", - "resource_type": "chat_session", - "resource_id": chat_id, - "details": normalized, - "status": "success", - }) + self.audit_repo.create( + { + "user_id": actor_user_id or session.user_id, + "action": "chat.session.updated", + "resource_type": "chat_session", + "resource_id": chat_id, + "details": normalized, + "status": "success", + } + ) return updated - def delete_session_force( - self, chat_id: str, *, actor_user_id: str - ) -> bool: + def delete_session_force(self, chat_id: str, *, actor_user_id: str) -> bool: """Forced delete in a sharing context (no ownership check). Caller handles permissions.""" session = self.session_repo.get_by_id(chat_id) if session is None: return False result = self.session_repo.soft_delete(chat_id) if result: - self.audit_repo.create({ - "user_id": actor_user_id, - "action": "chat.session.deleted", - "resource_type": "chat_session", - "resource_id": chat_id, - "details": { - "owner_user_id": session.user_id, - "deleted_by_owner": session.user_id == actor_user_id, - }, - "status": "success", - }) + self.audit_repo.create( + { + "user_id": actor_user_id, + "action": "chat.session.deleted", + "resource_type": "chat_session", + "resource_id": chat_id, + "details": { + "owner_user_id": session.user_id, + "deleted_by_owner": session.user_id == actor_user_id, + }, + "status": "success", + } + ) return result def update_session( - self, - chat_id: str, - user_id: str, - update_data: Dict[str, Any] + self, chat_id: str, user_id: str, update_data: Dict[str, Any] ) -> Optional[ChatSession]: """Update chat session.""" session = self.get_session(chat_id, user_id) @@ -230,14 +230,16 @@ def update_session( updated_session = self.session_repo.update(chat_id, normalized_update_data) # Audit log - self.audit_repo.create({ - "user_id": user_id, - "action": "chat.session.updated", - "resource_type": "chat_session", - "resource_id": chat_id, - "details": normalized_update_data, - "status": "success" - }) + self.audit_repo.create( + { + "user_id": user_id, + "action": "chat.session.updated", + "resource_type": "chat_session", + "resource_id": chat_id, + "details": normalized_update_data, + "status": "success", + } + ) return updated_session @@ -251,13 +253,15 @@ def delete_session(self, chat_id: str, user_id: str) -> bool: if result: # Audit log - self.audit_repo.create({ - "user_id": user_id, - "action": "chat.session.deleted", - "resource_type": "chat_session", - "resource_id": chat_id, - "status": "success" - }) + self.audit_repo.create( + { + "user_id": user_id, + "action": "chat.session.deleted", + "resource_type": "chat_session", + "resource_id": chat_id, + "status": "success", + } + ) return result @@ -283,7 +287,7 @@ def add_message( "tool_calls": tool_calls, "usage": usage, "error": error, - "extra_data": extra_data or {} + "extra_data": extra_data or {}, } message = self.message_repo.create(message_data) @@ -336,14 +340,19 @@ def upsert_message( self.db.commit() return msg or existing return self.add_message( - chat_id=chat_id, role=role, content=content, tool_calls=tool_calls, - usage=usage, extra_data=extra_data, message_id=message_id, + chat_id=chat_id, + role=role, + content=content, + tool_calls=tool_calls, + usage=usage, + extra_data=extra_data, + message_id=message_id, ) def list_all_messages(self, chat_id: str, user_id: str) -> Optional[List[ChatMessage]]: """List all messages in chronological order with access check. - Sharing context: team_read / team_edit members can also read. + Edition-specific sharing policies may also grant read access. **Excludes** compaction checkpoint rows (the only writer of role='system' in chat_messages is add_compaction_checkpoint) — internal artifacts, invisible to all @@ -356,17 +365,18 @@ def list_all_messages(self, chat_id: str, user_id: str) -> Optional[List[ChatMes if pair is None: return None - return self.db.query(ChatMessage).filter( - ChatMessage.chat_id == chat_id, - ChatMessage.role != "system", - ).order_by(ChatMessage.created_at).all() + return ( + self.db.query(ChatMessage) + .filter( + ChatMessage.chat_id == chat_id, + ChatMessage.role != "system", + ) + .order_by(ChatMessage.created_at) + .all() + ) def list_messages( - self, - chat_id: str, - user_id: str, - page: int = 1, - page_size: int = 50 + self, chat_id: str, user_id: str, page: int = 1, page_size: int = 50 ) -> Optional[Tuple[List[ChatMessage], int, int]]: """List messages in a chat session.""" # Check ownership @@ -384,24 +394,36 @@ def delete_messages_from(self, chat_id: str, message_id: str) -> int: Returns the number of messages deleted. """ - target = self.db.query(ChatMessage).filter( - ChatMessage.chat_id == chat_id, - ChatMessage.message_id == message_id, - ).first() + target = ( + self.db.query(ChatMessage) + .filter( + ChatMessage.chat_id == chat_id, + ChatMessage.message_id == message_id, + ) + .first() + ) if not target: return 0 - deleted = self.db.query(ChatMessage).filter( - ChatMessage.chat_id == chat_id, - ChatMessage.created_at >= target.created_at, - ).delete(synchronize_session="fetch") + deleted = ( + self.db.query(ChatMessage) + .filter( + ChatMessage.chat_id == chat_id, + ChatMessage.created_at >= target.created_at, + ) + .delete(synchronize_session="fetch") + ) # Update session message count session = self.session_repo.get_by_id(chat_id) if session: - remaining = self.db.query(ChatMessage).filter( - ChatMessage.chat_id == chat_id, - ).count() + remaining = ( + self.db.query(ChatMessage) + .filter( + ChatMessage.chat_id == chat_id, + ) + .count() + ) session.message_count = remaining session.updated_at = datetime.utcnow() @@ -433,22 +455,26 @@ def add_compaction_checkpoint( .first() ) - message = self.message_repo.create({ - "message_id": f"cmpct_{uuid.uuid4().hex[:16]}", - "chat_id": chat_id, - "role": "system", - "content": summary_text, - "extra_data": { - "kind": COMPACTION_CHECKPOINT_KIND, - "replacement_history": replacement_history, - "covers_up_to_message_id": last.message_id if last else None, - "covers_up_to_created_at": last.created_at.isoformat() if last and last.created_at else None, - # Pending-notice flag: consumed by the executor on the next turn's first frame - # (pop_compaction_notice) → emits a compaction_notice SSE event to tell the user - # compaction happened. - "notice_pending": True, - }, - }) + message = self.message_repo.create( + { + "message_id": f"cmpct_{uuid.uuid4().hex[:16]}", + "chat_id": chat_id, + "role": "system", + "content": summary_text, + "extra_data": { + "kind": COMPACTION_CHECKPOINT_KIND, + "replacement_history": replacement_history, + "covers_up_to_message_id": last.message_id if last else None, + "covers_up_to_created_at": ( + last.created_at.isoformat() if last and last.created_at else None + ), + # Pending-notice flag: consumed by the executor on the next turn's first frame + # (pop_compaction_notice) → emits a compaction_notice SSE event to tell the user + # compaction happened. + "notice_pending": True, + }, + } + ) self.db.commit() return message @@ -479,26 +505,42 @@ def get_latest_compaction_checkpoint(self, chat_id: str) -> Optional[ChatMessage def get_message_by_id(self, message_id: str) -> Optional[ChatMessage]: """Get a single message by its ID.""" - return self.db.query(ChatMessage).filter( - ChatMessage.message_id == message_id, - ).first() + return ( + self.db.query(ChatMessage) + .filter( + ChatMessage.message_id == message_id, + ) + .first() + ) def get_message_by_index(self, chat_id: str, index: int) -> Optional[ChatMessage]: """Get a message by its position (0-based) in the chat, ordered by created_at.""" - return self.db.query(ChatMessage).filter( - ChatMessage.chat_id == chat_id, - ).order_by(ChatMessage.created_at).offset(index).limit(1).first() + return ( + self.db.query(ChatMessage) + .filter( + ChatMessage.chat_id == chat_id, + ) + .order_by(ChatMessage.created_at) + .offset(index) + .limit(1) + .first() + ) def get_user_message_before(self, chat_id: str, message_id: str) -> Optional[ChatMessage]: """Get the user message immediately before the given message.""" target = self.get_message_by_id(message_id) if not target: return None - return self.db.query(ChatMessage).filter( - ChatMessage.chat_id == chat_id, - ChatMessage.role == "user", - ChatMessage.created_at < target.created_at, - ).order_by(ChatMessage.created_at.desc()).first() + return ( + self.db.query(ChatMessage) + .filter( + ChatMessage.chat_id == chat_id, + ChatMessage.role == "user", + ChatMessage.created_at < target.created_at, + ) + .order_by(ChatMessage.created_at.desc()) + .first() + ) def update_message_extra_data( self, diff --git a/src/backend/core/services/edition_service_probe.py b/src/backend/core/services/edition_service_probe.py new file mode 100644 index 00000000..edd27c07 --- /dev/null +++ b/src/backend/core/services/edition_service_probe.py @@ -0,0 +1,8 @@ +"""Community edition has no external knowledge-provider probe.""" + + +async def test_external_knowledge(base_url: str, api_key: str) -> dict: + return {"success": False, "latency_ms": 0, "error": "unsupported"} + + +__all__ = ["test_external_knowledge"] diff --git a/src/backend/core/services/edition_startup.py b/src/backend/core/services/edition_startup.py new file mode 100644 index 00000000..62a26a0b --- /dev/null +++ b/src/backend/core/services/edition_startup.py @@ -0,0 +1,8 @@ +"""Community-edition startup hooks.""" + + +def seed_default_roles(db) -> None: + return None + + +__all__ = ["seed_default_roles"] diff --git a/src/backend/core/services/edition_system_config.py b/src/backend/core/services/edition_system_config.py new file mode 100644 index 00000000..cefd7c72 --- /dev/null +++ b/src/backend/core/services/edition_system_config.py @@ -0,0 +1,6 @@ +"""Community edition contributes no enterprise service configuration.""" + +SEED_CONFIGS = [] +CONFIG_KEY_TO_ENV = {} + +__all__ = ["CONFIG_KEY_TO_ENV", "SEED_CONFIGS"] diff --git a/src/backend/core/services/kb_edition.py b/src/backend/core/services/kb_edition.py new file mode 100644 index 00000000..5217dd4b --- /dev/null +++ b/src/backend/core/services/kb_edition.py @@ -0,0 +1,12 @@ +"""Community knowledge-base grant policy: resources are owner-only.""" + +from __future__ import annotations + +from sqlalchemy.orm import Session + + +def initial_visibility_grants(db: Session, user_id: str, visibility: str) -> list[tuple[str, str]]: + return [] + + +__all__ = ["initial_visibility_grants"] diff --git a/src/backend/core/services/kb_permission_service.py b/src/backend/core/services/kb_permission_service.py deleted file mode 100644 index a754b829..00000000 --- a/src/backend/core/services/kb_permission_service.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Knowledge base permission assignment business logic (implicit authorization model). - -The KB management console does not configure visibility: shared bases are visible to -everyone by default, but **once any grant is assigned in "User Management / Team -Management" they switch to being visible only to grantees** (personal grants take -precedence over team grants). Visible-set / retrieval resolution lives in -``core.auth.kb_permissions``; this service only handles the grant CRUD for the two -management pages. -""" - -from __future__ import annotations - -from typing import Any, Dict, List, Optional - -from sqlalchemy.orm import Session - -from core.db.repository import KBGrantRepository, KBRepository - - -class KBPermissionService: - def __init__(self, db: Session): - self.db = db - self.repo = KBGrantRepository(db) - self.kb_repo = KBRepository(db) - - # ── Grantable resource list (selection source for the user/team management pages) ── - def list_grantable_resources(self) -> List[Dict[str, Any]]: - """All shared knowledge bases: local shared bases + Dify datasets (when enabled). Visibility is not distinguished or displayed.""" - out: List[Dict[str, Any]] = [] - for s in self.kb_repo.list_shared_spaces(): - out.append({ - "resource_id": s.kb_id, - "resource_type": "local", - "name": s.name, - "description": s.description or "", - }) - - try: - from core.kb.dify_kb import is_dify_enabled, list_datasets - if is_dify_enabled(): - for ds in list_datasets(page=1, limit=100, timeout=5): - ds_id = str(ds.get("id", "")).strip() - if not ds_id: - continue - out.append({ - "resource_id": ds_id, - "resource_type": "dify", - "name": ds.get("name", ds_id), - "description": ds.get("description") or ds.get("desc") or "", - }) - except Exception: - pass - return out - - # ── principal perspective (user/team management pages) ───────────────────── - def get_principal_grants(self, principal_type: str, principal_id: str) -> List[Dict[str, str]]: - return [ - {"resource_id": g.resource_id, "resource_type": g.resource_type, "level": g.level} - for g in self.repo.list_for_principal(principal_type, principal_id) - ] - - def replace_principal_grants( - self, - principal_type: str, - principal_id: str, - grants: List[Dict[str, str]], - granted_by: Optional[str] = None, - ) -> int: - """Fully replace a user's/team's KB grants ("Save" semantics of the management page). Returns the number of rows written. - - Row-level validation (valid resource_type / level, non-empty id) is handled uniformly by ``replace_for_principal``. - """ - return self.repo.replace_for_principal(principal_type, principal_id, grants or [], granted_by) diff --git a/src/backend/core/services/kb_service.py b/src/backend/core/services/kb_service.py index 859c2b2f..2e88790a 100644 --- a/src/backend/core/services/kb_service.py +++ b/src/backend/core/services/kb_service.py @@ -1,16 +1,16 @@ """Knowledge base business logic.""" -from typing import Optional, Dict, Any, Tuple -from datetime import datetime import os import uuid -from sqlalchemy.orm import Session +from datetime import datetime +from typing import Any, Dict, Optional, Tuple -from core.db.repository import KBRepository, AuditLogRepository, ArtifactRepository -from core.db.models import KBDocument, KBChunk, UserShadow -from core.storage import get_storage from core.content.kb_processing import vectorise_document_background - +from core.db.models import KBChunk, KBDocument, UserShadow +from core.db.repository import ArtifactRepository, AuditLogRepository, KBRepository +from core.services.artifact_edition import can_access_artifact +from core.storage import get_storage +from sqlalchemy.orm import Session # Fixed owner for admin-managed public knowledge bases. Public KB spaces are owned # by this synthetic system account so that admin endpoints can reuse the ownership- @@ -61,6 +61,7 @@ def _has_kb_level(self, space: Any, user_id: str, required: str) -> bool: if not space: return False from core.auth.kb_permissions import has_kb_permission, resolve_local_kb_level + return has_kb_permission(resolve_local_kb_level(self.db, user_id, space.kb_id), required) def _normalize_user_settings(self, user: Optional[UserShadow]) -> Dict[str, Any]: @@ -76,9 +77,9 @@ def ensure_system_owner(self) -> str: reuse the per-owner KBService methods. The row is created lazily on first use (no Alembic migration needed) to satisfy the KBSpace.user_id foreign key. """ - existing = self.db.query(UserShadow).filter( - UserShadow.user_id == SYSTEM_KB_OWNER_ID - ).first() + existing = ( + self.db.query(UserShadow).filter(UserShadow.user_id == SYSTEM_KB_OWNER_ID).first() + ) if existing: return SYSTEM_KB_OWNER_ID owner = UserShadow( @@ -129,7 +130,9 @@ def _list_all_space_documents(self, kb_id: str) -> list[KBDocument]: .all() ) - def _find_managed_document_by_artifact(self, kb_id: str, artifact_id: str) -> Optional[KBDocument]: + def _find_managed_document_by_artifact( + self, kb_id: str, artifact_id: str + ) -> Optional[KBDocument]: for document in self._list_all_space_documents(kb_id): meta = document.extra_data if isinstance(document.extra_data, dict) else {} if meta.get("source_artifact_id") == artifact_id: @@ -140,23 +143,27 @@ def ensure_my_space_sync_space(self, user_id: str) -> Tuple[Any, bool]: space = self._get_system_managed_space(user_id) created = False if not space: - space = self.repo.create_space({ - "kb_id": f"kb_{uuid.uuid4().hex[:16]}", - "user_id": user_id, - "name": MANAGED_SYNC_KB_NAME, - "description": MANAGED_SYNC_KB_DESCRIPTION, - "visibility": "private", - "chunk_method": "semantic", - "extra_data": dict(MANAGED_SYNC_KB_META), - }) + space = self.repo.create_space( + { + "kb_id": f"kb_{uuid.uuid4().hex[:16]}", + "user_id": user_id, + "name": MANAGED_SYNC_KB_NAME, + "description": MANAGED_SYNC_KB_DESCRIPTION, + "visibility": "private", + "chunk_method": "semantic", + "extra_data": dict(MANAGED_SYNC_KB_META), + } + ) created = True - self.audit_repo.create({ - "user_id": user_id, - "action": "kb.space.system_managed.created", - "resource_type": "kb_space", - "resource_id": space.kb_id, - "status": "success", - }) + self.audit_repo.create( + { + "user_id": user_id, + "action": "kb.space.system_managed.created", + "resource_type": "kb_space", + "resource_id": space.kb_id, + "status": "success", + } + ) else: space = self._ensure_managed_metadata(space) @@ -165,7 +172,11 @@ def ensure_my_space_sync_space(self, user_id: str) -> Tuple[Any, bool]: def get_my_space_sync_settings(self, user_id: str) -> Dict[str, Any]: user = self._get_user(user_id) settings = self._normalize_user_settings(user) - sync_settings = settings.get("my_space_sync_kb", {}) if isinstance(settings.get("my_space_sync_kb"), dict) else {} + sync_settings = ( + settings.get("my_space_sync_kb", {}) + if isinstance(settings.get("my_space_sync_kb"), dict) + else {} + ) space, _ = self.ensure_my_space_sync_space(user_id) return { "enabled": bool(sync_settings.get("enabled", False)), @@ -178,7 +189,11 @@ def _set_my_space_sync_enabled(self, user_id: str, enabled: bool) -> None: if not user: return metadata = self._normalize_user_settings(user) - sync_settings = metadata.get("my_space_sync_kb", {}) if isinstance(metadata.get("my_space_sync_kb"), dict) else {} + sync_settings = ( + metadata.get("my_space_sync_kb", {}) + if isinstance(metadata.get("my_space_sync_kb"), dict) + else {} + ) sync_settings["enabled"] = enabled sync_settings["updated_at"] = datetime.utcnow().isoformat() metadata["my_space_sync_kb"] = sync_settings @@ -189,7 +204,9 @@ def _set_my_space_sync_enabled(self, user_id: str, enabled: bool) -> None: def is_my_space_sync_enabled(self, user_id: str) -> bool: return bool(self.get_my_space_sync_settings(user_id).get("enabled")) - def sync_artifact_to_my_space_kb(self, artifact: Any, file_bytes: Optional[bytes] = None) -> Optional[Dict[str, Any]]: + def sync_artifact_to_my_space_kb( + self, artifact: Any, file_bytes: Optional[bytes] = None + ) -> Optional[Dict[str, Any]]: if not artifact or not artifact.user_id: return None @@ -201,7 +218,8 @@ def sync_artifact_to_my_space_kb(self, artifact: Any, file_bytes: Optional[bytes if not ( mime_type.startswith("image/") or mime_type.startswith("text/") - or mime_type in { + or mime_type + in { "application/pdf", "application/msword", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", @@ -225,23 +243,25 @@ def sync_artifact_to_my_space_kb(self, artifact: Any, file_bytes: Optional[bytes "uploaded_at": existing.uploaded_at.isoformat() if existing.uploaded_at else "", } - document = self.repo.create_document({ - "document_id": f"doc_{uuid.uuid4().hex[:16]}", - "kb_id": space.kb_id, - "title": artifact.title or artifact.filename, - "filename": artifact.filename, - "size_bytes": artifact.size_bytes, - "mime_type": mime_type, - "storage_key": artifact.storage_key, - "storage_url": artifact.storage_url, - "checksum": None, - "indexing_status": "processing", - "extra_data": { - "source": "my_space_sync", - "source_artifact_id": artifact.artifact_id, - "system_managed": True, - }, - }) + document = self.repo.create_document( + { + "document_id": f"doc_{uuid.uuid4().hex[:16]}", + "kb_id": space.kb_id, + "title": artifact.title or artifact.filename, + "filename": artifact.filename, + "size_bytes": artifact.size_bytes, + "mime_type": mime_type, + "storage_key": artifact.storage_key, + "storage_url": artifact.storage_url, + "checksum": None, + "indexing_status": "processing", + "extra_data": { + "source": "my_space_sync", + "source_artifact_id": artifact.artifact_id, + "system_managed": True, + }, + } + ) space.document_count = (space.document_count or 0) + 1 space.total_size_bytes = (space.total_size_bytes or 0) + max(artifact.size_bytes or 0, 0) @@ -263,14 +283,16 @@ def sync_artifact_to_my_space_kb(self, artifact: Any, file_bytes: Optional[bytes indexing_config=None, ) - self.audit_repo.create({ - "user_id": user_id, - "action": "kb.document.synced_from_my_space", - "resource_type": "kb_document", - "resource_id": document.document_id, - "details": {"artifact_id": artifact.artifact_id, "kb_id": space.kb_id}, - "status": "success", - }) + self.audit_repo.create( + { + "user_id": user_id, + "action": "kb.document.synced_from_my_space", + "resource_type": "kb_document", + "resource_id": document.document_id, + "details": {"artifact_id": artifact.artifact_id, "kb_id": space.kb_id}, + "status": "success", + } + ) return { "document_id": document.document_id, @@ -326,25 +348,29 @@ def delete_space(self, kb_id: str, user_id: str) -> bool: """Delete a KB space (soft delete).""" space = self.repo.get_space(kb_id) if space and self._is_system_managed_space(space): - self.audit_repo.create({ - "user_id": user_id, - "action": "kb.space.delete.failed", - "resource_type": "kb_space", - "resource_id": kb_id, - "status": "failed", - "details": {"reason": "system_managed"}, - }) + self.audit_repo.create( + { + "user_id": user_id, + "action": "kb.space.delete.failed", + "resource_type": "kb_space", + "resource_id": kb_id, + "status": "failed", + "details": {"reason": "system_managed"}, + } + ) return False if not self._has_kb_level(space, user_id, "admin"): # Audit failed attempt - self.audit_repo.create({ - "user_id": user_id, - "action": "kb.space.delete.failed", - "resource_type": "kb_space", - "resource_id": kb_id, - "status": "failed", - "details": {"reason": "not_found_or_unauthorized"} - }) + self.audit_repo.create( + { + "user_id": user_id, + "action": "kb.space.delete.failed", + "resource_type": "kb_space", + "resource_id": kb_id, + "status": "failed", + "details": {"reason": "not_found_or_unauthorized"}, + } + ) return False # Perform soft delete @@ -352,13 +378,15 @@ def delete_space(self, kb_id: str, user_id: str) -> bool: self.db.commit() # Audit log - self.audit_repo.create({ - "user_id": user_id, - "action": "kb.space.deleted", - "resource_type": "kb_space", - "resource_id": kb_id, - "status": "success" - }) + self.audit_repo.create( + { + "user_id": user_id, + "action": "kb.space.deleted", + "resource_type": "kb_space", + "resource_id": kb_id, + "status": "success", + } + ) return True @@ -372,14 +400,16 @@ def delete_document(self, document_id: str, user_id: str) -> bool: space = self.repo.get_space(document.kb_id) if not self._has_kb_level(space, user_id, "edit"): # Audit failed attempt - self.audit_repo.create({ - "user_id": user_id, - "action": "kb.document.delete.failed", - "resource_type": "kb_document", - "resource_id": document_id, - "status": "failed", - "details": {"reason": "not_found_or_unauthorized"} - }) + self.audit_repo.create( + { + "user_id": user_id, + "action": "kb.document.delete.failed", + "resource_type": "kb_document", + "resource_id": document_id, + "status": "failed", + "details": {"reason": "not_found_or_unauthorized"}, + } + ) return False # Perform soft delete @@ -392,14 +422,16 @@ def delete_document(self, document_id: str, user_id: str) -> bool: self.db.commit() # Audit log - self.audit_repo.create({ - "user_id": user_id, - "action": "kb.document.deleted", - "resource_type": "kb_document", - "resource_id": document_id, - "details": {"kb_id": document.kb_id, "filename": document.filename}, - "status": "success" - }) + self.audit_repo.create( + { + "user_id": user_id, + "action": "kb.document.deleted", + "resource_type": "kb_document", + "resource_id": document_id, + "details": {"kb_id": document.kb_id, "filename": document.filename}, + "status": "success", + } + ) return True @@ -411,7 +443,7 @@ def create_space( chunk_method: str = "semantic", metadata: Optional[Dict[str, Any]] = None, visibility: str = "private", - grant_team_ids: Optional[list] = None, + initial_grants: Optional[list[tuple[str, str]]] = None, granted_by: Optional[str] = None, ) -> Dict[str, Any]: """Create a new KB space. @@ -419,10 +451,9 @@ def create_space( ``visibility`` defaults to ``private`` so existing user-facing callers keep their behaviour; admin-managed public KBs pass ``visibility="public"``. - ``grant_team_ids``: teams granted visibility right after creating the KB (under the - default not-visible model, a public KB needs a grant before anyone can see it). - Creation and granting are unified in this single domain operation, avoiding each route - stitching a ``bulk_grant`` after create on its own and drifting apart. + ``initial_grants`` contains edition-owned principal grants applied atomically after + creation. CE supplies an empty list; EE decides which organization principals inherit + visibility without leaking those contracts into this shared service. """ space_data = { "kb_id": f"kb_{uuid.uuid4().hex[:16]}", @@ -436,30 +467,39 @@ def create_space( space = self.repo.create_space(space_data) - # After creating the KB, grant team visibility as needed (team members inherit view; the creator, as owner, can always manage) - team_ids = [str(t).strip() for t in (grant_team_ids or []) if str(t).strip()] - if team_ids: + grants = [ + (str(kind).strip(), str(principal_id).strip()) + for kind, principal_id in (initial_grants or []) + if str(kind).strip() and str(principal_id).strip() + ] + if grants: from core.db.repository import KBGrantRepository + KBGrantRepository(self.db).bulk_grant( - space.kb_id, "local", [("team", tid) for tid in team_ids], "view", + space.kb_id, + "local", + grants, + "view", granted_by=granted_by or user_id, ) # Audit log - self.audit_repo.create({ - "user_id": user_id, - "action": "kb.space.created", - "resource_type": "kb_space", - "resource_id": space.kb_id, - "status": "success" - }) + self.audit_repo.create( + { + "user_id": user_id, + "action": "kb.space.created", + "resource_type": "kb_space", + "resource_id": space.kb_id, + "status": "success", + } + ) return { "kb_id": space.kb_id, "name": space.name, "description": space.description, "document_count": space.document_count, - "created_at": space.created_at.isoformat() + "created_at": space.created_at.isoformat(), } def update_space( @@ -472,24 +512,28 @@ def update_space( """Update an existing KB space.""" space = self.repo.get_space(kb_id) if space and self._is_system_managed_space(space): - self.audit_repo.create({ - "user_id": user_id, - "action": "kb.space.update.failed", - "resource_type": "kb_space", - "resource_id": kb_id, - "status": "failed", - "details": {"reason": "system_managed"}, - }) + self.audit_repo.create( + { + "user_id": user_id, + "action": "kb.space.update.failed", + "resource_type": "kb_space", + "resource_id": kb_id, + "status": "failed", + "details": {"reason": "system_managed"}, + } + ) return None if not self._has_kb_level(space, user_id, "admin"): - self.audit_repo.create({ - "user_id": user_id, - "action": "kb.space.update.failed", - "resource_type": "kb_space", - "resource_id": kb_id, - "status": "failed", - "details": {"reason": "not_found_or_unauthorized"}, - }) + self.audit_repo.create( + { + "user_id": user_id, + "action": "kb.space.update.failed", + "resource_type": "kb_space", + "resource_id": kb_id, + "status": "failed", + "details": {"reason": "not_found_or_unauthorized"}, + } + ) return None update_data: Dict[str, Any] = {} @@ -502,13 +546,15 @@ def update_space( if not updated: return None - self.audit_repo.create({ - "user_id": user_id, - "action": "kb.space.updated", - "resource_type": "kb_space", - "resource_id": kb_id, - "status": "success", - }) + self.audit_repo.create( + { + "user_id": user_id, + "action": "kb.space.updated", + "resource_type": "kb_space", + "resource_id": kb_id, + "status": "success", + } + ) return { "kb_id": updated.kb_id, @@ -528,7 +574,7 @@ def upload_document( size_bytes: int, mime_type: str, storage_key: str, - checksum: Optional[str] = None + checksum: Optional[str] = None, ) -> Dict[str, Any]: """Upload a document to KB space.""" # Check ownership @@ -558,14 +604,16 @@ def upload_document( self.db.commit() # Audit log - self.audit_repo.create({ - "user_id": user_id, - "action": "kb.document.uploaded", - "resource_type": "kb_document", - "resource_id": document.document_id, - "details": {"kb_id": kb_id, "filename": filename}, - "status": "success" - }) + self.audit_repo.create( + { + "user_id": user_id, + "action": "kb.document.uploaded", + "resource_type": "kb_document", + "resource_id": document.document_id, + "details": {"kb_id": kb_id, "filename": filename}, + "status": "success", + } + ) return { "document_id": document.document_id, @@ -573,20 +621,16 @@ def upload_document( "title": document.title, "filename": document.filename, "size_bytes": document.size_bytes, - "uploaded_at": document.uploaded_at.isoformat() + "uploaded_at": document.uploaded_at.isoformat(), } def add_artifact_to_space(self, artifact_id: str, user_id: str, kb_id: str) -> Dict[str, Any]: """Create a KB document from an existing user artifact.""" - from core.auth.permissions_iface import has_permission, resolve_artifact_access - artifact_repo = ArtifactRepository(self.db) artifact = artifact_repo.get_by_id(artifact_id) if not artifact: raise ValueError("资源不存在或无权限") - # owner ∪ team composed-permission single point (owner always allowed; team view+ members can join a personal KB) - perm = resolve_artifact_access(self.db, user_id, artifact.user_id, artifact.team_id) - if not has_permission(perm, "view"): + if not can_access_artifact(self.db, user_id, artifact): raise ValueError("资源不存在或无权限") space = self.repo.get_space(kb_id) @@ -634,14 +678,16 @@ def add_artifact_to_space(self, artifact_id: str, user_id: str, kb_id: str) -> D space_extra = space.extra_data if isinstance(space.extra_data, dict) else {} - self.audit_repo.create({ - "user_id": user_id, - "action": "kb.document.added_from_my_space", - "resource_type": "kb_document", - "resource_id": document.document_id, - "details": {"kb_id": kb_id, "artifact_id": artifact_id}, - "status": "success", - }) + self.audit_repo.create( + { + "user_id": user_id, + "action": "kb.document.added_from_my_space", + "resource_type": "kb_document", + "resource_id": document.document_id, + "details": {"kb_id": kb_id, "artifact_id": artifact_id}, + "status": "success", + } + ) return { "document_id": document.document_id, diff --git a/src/backend/core/services/local_user_service.py b/src/backend/core/services/local_user_service.py index 92061e38..2391d0ae 100644 --- a/src/backend/core/services/local_user_service.py +++ b/src/backend/core/services/local_user_service.py @@ -49,33 +49,19 @@ def ce_onboarding_required(metadata: Optional[Dict[str, Any]]) -> bool: return bool(meta.get("onboarding_required")) and completed_version < CE_ONBOARDING_VERSION +from core.auth.account_policy import ( + add_account_to_scope, + add_invited_account_to_scope, + claim_registration_credential, + list_account_scopes, + registration_credential_id, + validate_account_scope, + validate_registration_credential, +) from core.auth.password import hash_password, verify_password - -# Seam S1: invite codes / team brief list belong to EE — the CE derived tree physically -# lacks these two modules; when missing, this service degrades automatically: open -# registration (no registration code), teams always empty. EE behavior unchanged. -try: - from core.auth.invite import claim_code, validate_code -except ModuleNotFoundError: - claim_code = None - validate_code = None -try: - from core.services.team_service import list_user_teams_brief -except ModuleNotFoundError: - - def list_user_teams_brief(db, user_id): - return [] - - from core.config.settings import settings -from core.db.models import LocalUser, TeamMember, UserShadow -from core.db.repository import ( - AuditLogRepository, - InviteCodeRepository, - LocalUserRepository, - TeamRepository, - UserRepository, -) +from core.db.models import LocalUser, UserShadow +from core.db.repository import AuditLogRepository, LocalUserRepository, UserRepository @dataclass @@ -101,8 +87,6 @@ def __init__(self, db: Session): self.db = db self.user_repo = UserRepository(db) self.local_repo = LocalUserRepository(db) - self.team_repo = TeamRepository(db) - self.invite_repo = InviteCodeRepository(db) self.audit_repo = AuditLogRepository(db) # ── Registration ─────────────────────────────────────────── @@ -144,20 +128,16 @@ def register( if self.db.query(UserShadow).filter(UserShadow.email == email).first() is not None: return RegisterResult(False, "邮箱已被使用") - # License seat cap (M4): internal deployments / unlimited seats always pass (counting and copy in licensing/seats.py) - from core.licensing.seats import seat_block_reason + from core.auth.account_policy import account_capacity_block_reason - seat_block = seat_block_reason(self.db) + seat_block = account_capacity_block_reason(self.db) if seat_block: return RegisterResult(False, seat_block) # First only validate the registration code (no state change); consume it after the user is successfully created - if validate_code is not None: - ok, reason, invite = validate_code(self.db, code) - if not ok: - return RegisterResult(False, reason or "注册码无效") - else: - invite = None + ok, reason, invite = validate_registration_credential(self.db, code) + if not ok: + return RegisterResult(False, reason or "注册码无效") user_id = f"user_{uuid.uuid4().hex[:16]}" try: @@ -183,28 +163,18 @@ def register( "real_name": (real_name or "").strip() or None, "phone": (phone or "").strip() or None, "status": "active", - "invited_by_code": invite.code if invite else None, + "invited_by_code": registration_credential_id(invite), "password_updated_at": datetime.utcnow(), } ) - # Pre-bind team - if invite and invite.preset_team_id: - team = self.team_repo.get(invite.preset_team_id) - if team is not None: - tm = TeamMember( - team_id=invite.preset_team_id, - user_id=user_id, - role=invite.preset_role or "member", - ) - self.db.add(tm) + add_invited_account_to_scope(self.db, invite, user_id) # The shadow is now persisted; atomically consume the registration code (conditional UPDATE ensures concurrency safety) - if claim_code is not None: - claimed_ok, claim_reason, _ = claim_code(self.db, code, user_id) - if not claimed_ok: - self.db.rollback() - return RegisterResult(False, claim_reason or "注册码已被使用") + claimed_ok, claim_reason = claim_registration_credential(self.db, code, user_id) + if not claimed_ok: + self.db.rollback() + return RegisterResult(False, claim_reason or "注册码已被使用") # Audit self.audit_repo.create( @@ -213,7 +183,7 @@ def register( "action": "user.register_local", "resource_type": "user", "resource_id": user_id, - "details": {"invite_code": invite.code if invite else None}, + "details": {"invite_code": registration_credential_id(invite)}, "status": "success", } ) @@ -241,21 +211,21 @@ def create_by_admin( real_name: Optional[str] = None, phone: Optional[str] = None, status: str = "active", - team_id: Optional[str] = None, - team_role: str = "member", + scope_id: Optional[str] = None, + scope_role: str = "member", actor: Optional[str] = None, ) -> RegisterResult: - """Create a local account directly from the Config console — no registration code, optional direct team binding. + """Create a local account directly from the Config console. Differences from :meth:`register`: no registration code required, email optional, - initial status can be specified, team can be bound explicitly. + initial status can be specified, and edition-specific scope binding is optional. """ username = (username or "").strip() nickname = (nickname or "").strip() or username email = (email or "").strip().lower() or None password = password or "" status = (status or "active").strip() - team_role = (team_role or "member").strip() or "member" + scope_role = (scope_role or "member").strip() or "member" if not username: return RegisterResult(False, "账号不能为空") @@ -280,15 +250,12 @@ def create_by_admin( ): return RegisterResult(False, "邮箱已被使用") - # Target team existence check (only when provided) - if team_id: - if self.team_repo.get(team_id) is None: - return RegisterResult(False, "目标团队不存在") + if not validate_account_scope(self.db, scope_id): + return RegisterResult(False, "目标范围不存在") - # License seat cap: internal deployments / unlimited seats always pass - from core.licensing.seats import seat_block_reason + from core.auth.account_policy import account_capacity_block_reason - seat_block = seat_block_reason(self.db) + seat_block = account_capacity_block_reason(self.db) if seat_block: return RegisterResult(False, seat_block) @@ -319,8 +286,7 @@ def create_by_admin( } ) - if team_id: - self.db.add(TeamMember(team_id=team_id, user_id=user_id, role=team_role)) + add_account_to_scope(self.db, scope_id, user_id, scope_role) self.audit_repo.create( { @@ -330,8 +296,8 @@ def create_by_admin( "resource_id": user_id, "details": { "created_by": actor or "config_admin", - "team_id": team_id, - "team_role": team_role if team_id else None, + "scope_id": scope_id, + "scope_role": scope_role if scope_id else None, }, "status": "success", } @@ -392,13 +358,11 @@ def get_or_create_external_account( if existing is not None: return existing, False - # License seat cap (M4): blocks new creation only; internal deployments / unlimited seats always pass - from core.licensing import SeatLimitExceeded - from core.licensing.seats import seat_block_reason + from core.auth.account_policy import AccountCapacityExceeded, account_capacity_block_reason - block = seat_block_reason(self.db) + block = account_capacity_block_reason(self.db) if block: - raise SeatLimitExceeded(block) + raise AccountCapacityExceeded(block) final_username = self._unique_username((username or external_id).strip() or external_id) user_id = f"user_{uuid.uuid4().hex[:16]}" @@ -514,7 +478,7 @@ def _build_user_info( "auth_source": "local", "must_change_password": bool(meta.get("must_change_password")), "onboarding_required": ce_onboarding_required(meta), - "teams": list_user_teams_brief(self.db, user_id), + "teams": list_account_scopes(self.db, user_id), } # ── Profile update ───────────────────────────────────────── diff --git a/src/backend/core/services/marketplace_listing.py b/src/backend/core/services/marketplace_listing.py index cbd99c8a..d5f4a621 100644 --- a/src/backend/core/services/marketplace_listing.py +++ b/src/backend/core/services/marketplace_listing.py @@ -1,41 +1,23 @@ -"""Marketplace listing state: shared by the plugin marketplace / skill marketplace. +"""Community marketplace listing state. -Controls whether a given plugin/skill is shown in the marketplace. **A missing row means -enabled** (all items listed by default, not cleared on upgrade); only when an admin -explicitly disables one is a row with ``enabled=false`` written. The user-facing -marketplace shows only enabled items; the admin backend shows all, annotated with -``market_enabled``. Physical deletion (uploaded items only) goes through their respective -delete endpoints; this module only handles visibility. - -Visibility scope: ``public`` (default, same as a missing row) visible to everyone; -``scoped`` visible only to authorized users/teams/roles (the allowlist is stored in -``marketplace_visibility_grants``, resolution see core/auth/marketplace_visibility.py). -The user-facing list/detail/install filter by this; the admin backend does not filter, -annotating with ``visibility`` plus a grant-config endpoint. Only handles marketplace -browsing and installation, does not trace back already-installed instances. +CE keeps the global list/delist switch. Organization-scoped visibility and +principal grants are intentionally absent: every enabled item is public. """ + from __future__ import annotations from datetime import datetime from typing import Any, Dict, List, Optional +from core.db.models import MarketplaceListingState from sqlalchemy.orm import Session -from core.db.models import MarketplaceListingState, MarketplaceVisibilityGrant -from core.infra.exceptions import BadRequestError, ResourceNotFoundError - KIND_PLUGIN = "plugin" KIND_SKILL = "skill" KIND_AGENT = "agent" -VISIBILITY_PUBLIC = "public" -VISIBILITY_SCOPED = "scoped" -_VISIBILITIES = (VISIBILITY_PUBLIC, VISIBILITY_SCOPED) -_PRINCIPAL_TYPES = ("user", "team", "role") - -def get_disabled_ids(db: Session, kind: str) -> set: - """Set of item_ids disabled by an admin in this marketplace (missing row=enabled, so only query enabled=false).""" +def get_disabled_ids(db: Session, kind: str) -> set[str]: return { row[0] for row in db.query(MarketplaceListingState.item_id) @@ -48,9 +30,13 @@ def get_disabled_ids(db: Session, kind: str) -> set: def set_listing_enabled( - db: Session, kind: str, item_id: str, enabled: bool, *, updated_by: Optional[str] = None + db: Session, + kind: str, + item_id: str, + enabled: bool, + *, + updated_by: Optional[str] = None, ) -> Dict[str, Any]: - """List/delist a marketplace item (idempotent upsert).""" row = ( db.query(MarketplaceListingState) .filter( @@ -59,12 +45,11 @@ def set_listing_enabled( ) .first() ) - now = datetime.utcnow() if row is None: row = MarketplaceListingState(kind=kind, item_id=item_id) db.add(row) row.enabled = bool(enabled) - row.updated_at = now + row.updated_at = datetime.utcnow() row.updated_by = updated_by db.commit() return {"kind": kind, "item_id": item_id, "enabled": bool(enabled)} @@ -79,148 +64,34 @@ def annotate_and_filter( include_disabled: bool, viewer_user_id: Optional[str] = None, ) -> List[Dict[str, Any]]: - """Annotate each item with ``market_enabled`` / ``visibility``; the user side - (include_disabled=False) filters out disabled items, and by visibility scope filters - out scoped items the current user (``viewer_user_id``) has no right to see. - The admin backend (include_disabled=True) sees all, relying on ``visibility`` to show scope state. - """ disabled = get_disabled_ids(db, kind) - scoped = get_scoped_ids(db, kind) - hidden: set = set() - if not include_disabled: - from core.auth.marketplace_visibility import get_hidden_item_ids - hidden = get_hidden_item_ids(db, kind, viewer_user_id) out: List[Dict[str, Any]] = [] - for it in items: - item_id = str(it.get(id_key)) - is_on = item_id not in disabled - if not include_disabled and (not is_on or item_id in hidden): + for item in items: + enabled = str(item.get(id_key)) not in disabled + if not include_disabled and not enabled: continue - it["market_enabled"] = is_on - it["visibility"] = VISIBILITY_SCOPED if item_id in scoped else VISIBILITY_PUBLIC - out.append(it) + item["market_enabled"] = enabled + out.append(item) return out -# ── Visibility scope (visibility + grants) ────────────────────────────────────────── - -def get_scoped_ids(db: Session, kind: str) -> set: - """Set of item_ids in this marketplace set to scoped (visible to a specified scope) (missing row=public, so only query scoped).""" - return { - row[0] - for row in db.query(MarketplaceListingState.item_id) - .filter( - MarketplaceListingState.kind == kind, - MarketplaceListingState.visibility == VISIBILITY_SCOPED, - ) - .all() - } - - -def get_listing_visibility(db: Session, kind: str, item_id: str) -> Dict[str, Any]: - """A single item's visibility scope config: {visibility, grants:[{principal_type, principal_id}]}.""" - row = ( - db.query(MarketplaceListingState.visibility) - .filter( - MarketplaceListingState.kind == kind, - MarketplaceListingState.item_id == item_id, - ) - .first() - ) - visibility = row[0] if row and row[0] in _VISIBILITIES else VISIBILITY_PUBLIC - grants = [ - {"principal_type": g.principal_type, "principal_id": g.principal_id} - for g in db.query(MarketplaceVisibilityGrant) - .filter( - MarketplaceVisibilityGrant.kind == kind, - MarketplaceVisibilityGrant.item_id == item_id, - ) - .order_by( - MarketplaceVisibilityGrant.principal_type, - MarketplaceVisibilityGrant.principal_id, - ) - .all() - ] - return {"kind": kind, "item_id": item_id, "visibility": visibility, "grants": grants} - - -def set_listing_visibility( +def ensure_item_visible( db: Session, kind: str, item_id: str, + user_id: Optional[str], *, - visibility: str, - grants: Optional[List[Dict[str, str]]] = None, - updated_by: Optional[str] = None, -) -> Dict[str, Any]: - """Set an item's visibility scope (idempotent upsert + full replacement of grants). - - ``public`` → clear grants; ``scoped`` → replace the entire allowlist with ``grants`` - (at least one, otherwise the item is invisible to all regular users, which is a - config error, straight 400). - """ - if visibility not in _VISIBILITIES: - raise BadRequestError(message=f"visibility 仅支持 {', '.join(_VISIBILITIES)}") - grants = grants or [] - seen: set = set() - normalized: List[Dict[str, str]] = [] - for g in grants: - ptype = str(g.get("principal_type") or "").strip() - pid = str(g.get("principal_id") or "").strip() - if ptype not in _PRINCIPAL_TYPES: - raise BadRequestError(message=f"principal_type 仅支持 {', '.join(_PRINCIPAL_TYPES)}") - if not pid: - raise BadRequestError(message="principal_id 不能为空") - if (ptype, pid) in seen: - continue - seen.add((ptype, pid)) - normalized.append({"principal_type": ptype, "principal_id": pid}) - if visibility == VISIBILITY_SCOPED and not normalized: - raise BadRequestError(message="指定范围可见时至少需要一条授权（用户/团队/角色）") - - row = ( - db.query(MarketplaceListingState) - .filter( - MarketplaceListingState.kind == kind, - MarketplaceListingState.item_id == item_id, - ) - .first() - ) - if row is None: - row = MarketplaceListingState(kind=kind, item_id=item_id, enabled=True) - db.add(row) - row.visibility = visibility - row.updated_at = datetime.utcnow() - row.updated_by = updated_by - - db.query(MarketplaceVisibilityGrant).filter( - MarketplaceVisibilityGrant.kind == kind, - MarketplaceVisibilityGrant.item_id == item_id, - ).delete(synchronize_session=False) - if visibility == VISIBILITY_SCOPED: - for g in normalized: - db.add( - MarketplaceVisibilityGrant( - kind=kind, - item_id=item_id, - principal_type=g["principal_type"], - principal_id=g["principal_id"], - created_by=updated_by, - ) - ) - db.commit() - return { - "kind": kind, - "item_id": item_id, - "visibility": visibility, - "grants": normalized if visibility == VISIBILITY_SCOPED else [], - } - - -def ensure_item_visible( - db: Session, kind: str, item_id: str, user_id: Optional[str], *, resource: str + resource: str, ) -> None: - """Detail/install path guard: when an item is invisible to the current user, treat it as "not found" (404, does not leak existence).""" - from core.auth.marketplace_visibility import is_item_visible - if not is_item_visible(db, kind, item_id, user_id): - raise ResourceNotFoundError(resource, item_id) + """All enabled CE marketplace items are visible.""" + + +__all__ = [ + "KIND_AGENT", + "KIND_PLUGIN", + "KIND_SKILL", + "annotate_and_filter", + "ensure_item_visible", + "get_disabled_ids", + "set_listing_enabled", +] diff --git a/src/backend/core/services/mcp_service.py b/src/backend/core/services/mcp_service.py index a488a4b5..1553af2c 100644 --- a/src/backend/core/services/mcp_service.py +++ b/src/backend/core/services/mcp_service.py @@ -389,6 +389,44 @@ def is_removed_builtin_mcp_server( return server_id in builtin_ids and server_id not in PORTS +def prune_removed_builtin_mcp_servers(db) -> List[str]: + """Delete stale global built-ins whose runtime is absent in this edition. + + CE physically removes commercial MCP packages and port registrations. An + upgraded database can still carry rows seeded by an older/full build; keep + the persisted catalog aligned with the packaged runtime instead of merely + hiding those rows at read time. Plugin-owned and user-owned MCP rows are + never touched. + """ + from mcp_servers._ports import PORTS + + removed_ids = { + str(spec["server_id"]) + for spec in BUILTIN_MCP_SERVERS + if str(spec["server_id"]) not in PORTS + } + if not removed_ids: + return [] + + rows = ( + db.query(AdminMcpServer) + .filter( + AdminMcpServer.server_id.in_(removed_ids), + AdminMcpServer.owner_user_id.is_(None), + AdminMcpServer.source_plugin.is_(None), + ) + .all() + ) + pruned = sorted(row.server_id for row in rows) + if not pruned: + return [] + for row in rows: + db.delete(row) + db.commit() + McpServerConfigService.get_instance().invalidate_cache() + return pruned + + def seed_builtin_mcp_servers_if_empty(db) -> List[str]: """Seed the built-in global MCP catalog when it is entirely absent. diff --git a/src/backend/core/services/oa_sso_service.py b/src/backend/core/services/oa_sso_service.py deleted file mode 100644 index 65cffcec..00000000 --- a/src/backend/core/services/oa_sso_service.py +++ /dev/null @@ -1,202 +0,0 @@ -"""OA single sign-on (server-side direct-push provisioning). - -Integration model (a separate path from the ticket verification in core/auth/sso.py): - - The OA backend has already authenticated the user on its side → pushes - ``user_id`` + ``dept_id`` + signature to this platform - → platform verifies the signature (HMAC, shared secret between OA and platform) - → auto-creates a local account (username = user_id, random strong password; - idempotent: reused if it already exists) - → binds a team by dept_id (default member role) - → issues a session token in the response; OA uses it for the login redirect - -The trust anchor is on the OA side. The platform does not trust the user_id in the -request body; server-to-server trust is established by HMAC signature verification -with the shared secret agreed between OA and the platform — otherwise "knowing a -valid employee ID = becoming that person". Verification is enforced once -``OA_SSO_SIGN_SECRET`` is configured; leaving it empty is for intranet integration -testing only (a warning is logged). -""" - -from __future__ import annotations - -import hashlib -import hmac -import time -import uuid -from datetime import datetime -from typing import Optional, Tuple - -from sqlalchemy.orm import Session - -from core.config.settings import settings -from core.db.models import Team, TeamMember, UserShadow -from core.infra.logging import get_logger -from core.services.local_user_service import LocalUserService - -logger = get_logger(__name__) - - -class OASsoError(Exception): - """OA SSO provisioning failure. Carries the HTTP status code and business code for the route layer to render the envelope.""" - - def __init__(self, message: str, *, status_code: int = 401, code: int = 30002): - self.message = message - self.status_code = status_code - self.code = code - super().__init__(message) - - -def verify_signature( - *, - user_id: str, - dept_id: str, - timestamp: str, - nonce: str, - signature: str, -) -> None: - """Verify the HMAC-SHA256 signature of an OA server-side request (with timestamp replay protection). - - Signature base string (canonical, newline-joined, fixed order):: - - user_id \n dept_id \n timestamp \n nonce - - HMAC-SHA256(secret, base) → hex; the OA side and platform side use the same - algorithm. When ``OA_SSO_SIGN_SECRET`` is empty, verification is skipped - (intranet integration testing only) and a warning reminds that production - must configure the secret. - """ - secret = settings.oa_sso.sign_secret - if not secret: - logger.warning( - "oa_sso_signature_skipped", - reason="OA_SSO_SIGN_SECRET 未配置——生产环境必须配密钥，否则任意调用方可冒充任意工号", - user_id=user_id, - ) - return - - if not signature or not timestamp: - raise OASsoError("Missing signature or timestamp", status_code=401, code=30005) - - # Timestamp replay protection: reject outright when outside the tolerance window - try: - ts = int(timestamp) - except (TypeError, ValueError): - raise OASsoError("Invalid timestamp", status_code=401, code=30005) - skew = abs(int(time.time()) - ts) - if skew > settings.oa_sso.sign_ttl_seconds: - raise OASsoError("Signature expired (timestamp out of window)", status_code=401, code=30005) - - base = "\n".join([user_id, dept_id or "", str(timestamp), nonce or ""]) - expected = hmac.new(secret.encode(), base.encode(), hashlib.sha256).hexdigest() - if not hmac.compare_digest(expected, signature.strip().lower()): - logger.warning("oa_sso_signature_mismatch", user_id=user_id) - raise OASsoError("Signature verification failed", status_code=401, code=30005) - - -class OASsoService: - """OA direct-push login: auto-create a local account + bind a team by dept_id.""" - - def __init__(self, db: Session): - self.db = db - self.local_service = LocalUserService(db) - - def _bind_team(self, *, user_id: str, dept_id: Optional[str]) -> Optional[str]: - """Bind the org team by dept_id and return the team_id. The team is auto-created when it does not exist. - - Uses dept_id as the team identifier (stored in ``Team.sso_department``). - Idempotent: repeated calls only ensure membership; they neither re-create the - team nor change existing roles. The member's role within the team is decided - by ``OA_SSO_DEFAULT_ROLE``; on the **first auto-creation of a team**, all - default roles with ``Role.is_team_default`` are also attached to that team - (members inherit their capability bits in real time), aligned with the - behavior of ``sso_sync.sync_user_department``. - """ - key = (dept_id or "").strip() - if not key: - return None - - role = settings.oa_sso.default_role - team = self.db.query(Team).filter(Team.sso_department == key).first() - if team is None: - team = Team( - team_id=f"team_{uuid.uuid4().hex[:16]}", - name=key, - description=f"由 OA 机构「{key}」自动创建", - sso_department=key, - source="sso_auto", - ) - self.db.add(team) - self.db.flush() - logger.info("oa_sso_team_auto_created", team_id=team.team_id, dept_id=key) - # Auto-attach the "new team default" roles (members inherit in real time) - # — aligned with sso_sync.sync_user_department; the OA direct-push path - # previously missed this step, so configured default roles never took - # effect for OA users. flush-only, keeping the "caller commits" - # convention; CE without role tables / any exception is skipped safely. - try: - from core.db.models import Role, RoleAssignment - - default_ids = [ - rid - for (rid,) in self.db.query(Role.role_id) - .filter(Role.is_team_default.is_(True)) - .all() - ] - for rid in default_ids: - self.db.add( - RoleAssignment( - role_id=rid, principal_type="team", principal_id=team.team_id - ) - ) - if default_ids: - self.db.flush() - logger.info( - "oa_sso_team_default_roles_applied", - team_id=team.team_id, - count=len(default_ids), - ) - except Exception as exc: # noqa: BLE001 - logger.warning("oa_sso_team_default_roles_failed", error=str(exc)) - - existing = ( - self.db.query(TeamMember) - .filter(TeamMember.team_id == team.team_id, TeamMember.user_id == user_id) - .first() - ) - if existing is None: - self.db.add( - TeamMember( - team_id=team.team_id, - user_id=user_id, - role=role, - joined_at=datetime.utcnow(), - ) - ) - self.db.flush() - logger.info("oa_sso_user_joined_team", team_id=team.team_id, user_id=user_id, role=role) - return team.team_id - - def provision( - self, - *, - oa_user_id: str, - dept_id: Optional[str], - ) -> Tuple[UserShadow, Optional[str], bool]: - """End to end: auto-create/fetch the local account → bind the org team. - - Returns ``(user_shadow, team_id, created)``, where ``created`` indicates - whether an account was newly created this time. The transaction is - committed by the caller. - """ - oa_user_id = (oa_user_id or "").strip() - if not oa_user_id: - raise OASsoError("Missing user_id", status_code=400, code=30001) - - user, created = self.local_service.get_or_create_external_account( - external_id=oa_user_id, - username=oa_user_id, - source="oa_sso", - ) - team_id = self._bind_team(user_id=user.user_id, dept_id=dept_id) - return user, team_id, created diff --git a/src/backend/core/services/ontology_service.py b/src/backend/core/services/ontology_service.py index 13d477ae..be808ef8 100644 --- a/src/backend/core/services/ontology_service.py +++ b/src/backend/core/services/ontology_service.py @@ -46,6 +46,35 @@ def disabled_ontology_runtime() -> dict[str, Any]: return {"enabled": False, "packs": [], "review_level": "none"} +def build_ontology_runtime_for_preference( + *, + enabled: bool, + task: str, + db: Session, + pack_ids: list[str] | None = None, +) -> tuple[bool, dict[str, Any]]: + """Resolve the effective ontology switch and compile its runtime policy. + + A user's stored opt-in is subordinate to the administrator-controlled pack + availability. Disabling the selected/default packs is therefore an + intentional global off switch, not a runtime-policy failure. Once an + active pack exists, compilation remains fail-closed so malformed policy + data cannot silently bypass validation. + """ + if not enabled: + return False, disabled_ontology_runtime() + + service = OntologyService(db) + selected = pack_ids or None + if not service.repo.get_active_versions(selected): + return False, disabled_ontology_runtime() + + runtime = service.build_runtime(task=task, pack_ids=selected) + if not runtime.get("enabled"): + raise ServiceUnavailableError("本体校验运行时策略编译失败") + return True, runtime + + class OntologyService: def __init__(self, db: Session): self.db = db @@ -561,13 +590,12 @@ def build_user_ontology_runtime( if isinstance(pack_ids, list) else None ) - runtime = OntologyService(db).build_runtime( + return build_ontology_runtime_for_preference( + enabled=opted_in, task=task, + db=db, pack_ids=selected or None, ) - if not runtime.get("enabled"): - raise ServiceUnavailableError("本体校验已开启，但当前没有可用的已激活 Domain Pack") - return True, runtime finally: if owns_session: db.close() diff --git a/src/backend/core/services/persona_distillation_service.py b/src/backend/core/services/persona_distillation_service.py deleted file mode 100644 index 24d8e819..00000000 --- a/src/backend/core/services/persona_distillation_service.py +++ /dev/null @@ -1,900 +0,0 @@ -"""Persona distillation service — lifecycle of persona-level distillation jobs (colleague skills / personal skills). - -create_job → asyncio background run_job: - map (per-session assemble_trajectory → session digest) - → memory collection (colleague: all workspaces; personal: projects linked to the selected sessions) - → reduce (synthesize SKILL.md) - → colleague mirrors into admin_skill_drafts; personal waits for the user to confirm save. - -Budget: each job consumes 1 daily run slot; every LLM call passes through the daily cost gate first; -once the per-job cost cap trips, we go into reduce early with the digests gathered so far (result marked partial). -""" - -from __future__ import annotations - -import asyncio -import json -import logging -import re -import uuid -from datetime import datetime -from typing import Any, Dict, List, Optional, Tuple - -import sqlalchemy as sa -from sqlalchemy import func, or_ -from sqlalchemy.orm import Session - -from core.config.distillation import DistillationConfig, get_config -from core.db.engine import SessionLocal -from core.db.models import ( - AdminSkill, - AdminSkillDraft, - ChatSession, - MemoryAudit, - PersonaDistillJob, - Project, - TeamMember, -) -from core.infra.distillation_budget import ( - check_and_reserve_run, - check_cost_budget, - record_cost, -) -from core.infra.exceptions import BadRequestError, ResourceNotFoundError -from core.llm._distill_shared import skill_to_markdown -from core.llm.persona_distiller import distill_persona, summarize_session -from core.ontology.build_validator import ensure_ontology_build_valid -from core.services.distillation_service import assemble_trajectory - -logger = logging.getLogger(__name__) - -VALID_KINDS = ("colleague", "personal") - -# High-confidentiality memories never enter the distillation corpus -_CONFIDENTIAL_VALUES = {"high", "secret", "confidential", "机密", "保密", "内部"} - -_COLLEAGUE_NAME_RE = re.compile(r"^数字同事-([A-Z]+)$") - - -# ─────────────────────── Code-name based naming (A..Z, AA..) ──────────────────────── - - -def _letters_to_num(letters: str) -> int: - n = 0 - for ch in letters: - n = n * 26 + (ord(ch) - ord("A") + 1) - return n - - -def _num_to_letters(n: int) -> str: - out = "" - while n > 0: - n, rem = divmod(n - 1, 26) - out = chr(ord("A") + rem) + out - return out - - -def allocate_colleague_identity(db: Session) -> Tuple[str, str]: - """Allocate the next digital-colleague code name. Returns (display_name, skill_id). - - Scans display_name of existing skills and drafts, takes max ordinal +1 (carries over to AA after Z). - """ - names: List[str] = [] - for (name,) in db.query(AdminSkill.display_name).all(): - if name: - names.append(name) - for (name,) in db.query(AdminSkillDraft.display_name).all(): - if name: - names.append(name) - max_n = 0 - for name in names: - m = _COLLEAGUE_NAME_RE.match(name.strip()) - if m: - max_n = max(max_n, _letters_to_num(m.group(1))) - code = _num_to_letters(max_n + 1) - return f"数字同事-{code}", f"colleague-{code.lower()}" - - -# ─────────────────────── Job creation / query ──────────────────────── - - -def validate_chat_ids(db: Session, chat_ids: List[str], user_id: str) -> List[str]: - """Verify that all chat_ids belong to user_id and are not deleted. Returns the list of invalid ids.""" - if not chat_ids: - return [] - rows = ( - db.query(ChatSession.chat_id) - .filter( - ChatSession.chat_id.in_(chat_ids), - ChatSession.user_id == user_id, - ChatSession.deleted_at.is_(None), - ) - .all() - ) - valid = {r[0] for r in rows} - return [c for c in chat_ids if c not in valid] - - -def create_job( - db: Session, - *, - kind: str, - target_user_id: str, - requested_by: str, - scope: Dict[str, Any], -) -> PersonaDistillJob: - cfg = get_config() - if not cfg.persona_enabled: - raise BadRequestError("人物技能蒸馏功能未启用（DISTILL_PERSONA_ENABLED=false）") - if kind not in VALID_KINDS: - raise BadRequestError(f"无效的蒸馏类型: {kind}") - - job = PersonaDistillJob( - job_id=f"pdj_{uuid.uuid4().hex[:16]}", - kind=kind, - target_user_id=target_user_id, - requested_by=requested_by, - scope=scope or {}, - status="queued", - created_at=datetime.utcnow(), - ) - db.add(job) - db.commit() - db.refresh(job) - return job - - -def start_job_background(job_id: str) -> None: - """Fire-and-forget background execution (routes call this from an async context).""" - asyncio.create_task(run_job(job_id)) - - -def job_to_dict( - job: PersonaDistillJob, - *, - include_result: bool = False, - admin_view: bool = False, - draft: Optional[AdminSkillDraft] = None, -) -> dict: - """PersonaDistillJob → API dict (shared by the lab and config routes). - - admin_view adds admin-console fields (target_user / mirrored-draft status); include_result carries - the full SKILL.md text — a colleague draft may have been edited in AdminApp, so the draft's current value wins. - """ - d = { - "job_id": job.job_id, - "kind": job.kind, - "status": job.status, - "progress_done": job.progress_done or 0, - "progress_total": job.progress_total or 0, - "cost_usd": float(job.cost_usd or 0), - "scope": job.scope or {}, - "result_meta": job.result_meta or {}, - "saved_skill_id": job.saved_skill_id, - "error": job.error, - "created_at": job.created_at.isoformat() if job.created_at else None, - "started_at": job.started_at.isoformat() if job.started_at else None, - "finished_at": job.finished_at.isoformat() if job.finished_at else None, - } - if admin_view: - d["target_user_id"] = job.target_user_id - d["result_draft_id"] = job.result_draft_id - d["draft_review_status"] = draft.review_status if draft else None - if include_result: - d["result_skill_content"] = draft.skill_content if draft else job.result_skill_content - return d - - -def get_job(db: Session, job_id: str) -> Optional[PersonaDistillJob]: - return ( - db.query(PersonaDistillJob) - .filter(PersonaDistillJob.job_id == job_id) - .first() - ) - - -def list_jobs( - db: Session, - *, - kind: Optional[str] = None, - target_user_id: Optional[str] = None, - requested_by: Optional[str] = None, - limit: int = 50, -) -> List[PersonaDistillJob]: - q = db.query(PersonaDistillJob) - if kind: - q = q.filter(PersonaDistillJob.kind == kind) - if target_user_id: - q = q.filter(PersonaDistillJob.target_user_id == target_user_id) - if requested_by: - q = q.filter(PersonaDistillJob.requested_by == requested_by) - return q.order_by(PersonaDistillJob.created_at.desc()).limit(limit).all() - - -def cancel_job(db: Session, job_id: str) -> PersonaDistillJob: - job = get_job(db, job_id) - if job is None: - raise ResourceNotFoundError("persona_distill_job", job_id) - if job.status not in ("queued", "running"): - raise BadRequestError(f"作业已结束（{job.status}），无法取消") - job.status = "cancelled" - job.finished_at = datetime.utcnow() - db.commit() - db.refresh(job) - return job - - -def delete_job(db: Session, job_id: str) -> None: - job = get_job(db, job_id) - if job is None: - raise ResourceNotFoundError("persona_distill_job", job_id) - if job.status == "running": - raise BadRequestError("作业运行中，请先取消") - db.delete(job) - db.commit() - - -def recover_orphan_jobs() -> int: - """After a process restart, mark orphaned running/queued jobs as failed (called from the startup hook).""" - db = SessionLocal() - try: - rows = ( - db.query(PersonaDistillJob) - .filter(PersonaDistillJob.status.in_(("queued", "running"))) - .all() - ) - for job in rows: - job.status = "failed" - job.error = "backend restarted while job was in flight" - job.finished_at = datetime.utcnow() - if rows: - db.commit() - return len(rows) - finally: - db.close() - - -# ─────────────────────── Session-set resolution and sampling ──────────────────────── - - -def _resolve_chat_ids(db: Session, job: PersonaDistillJob, cfg: DistillationConfig) -> Tuple[List[str], int]: - """Resolve the session set covered by the job. Returns (chat_ids, total_candidates). - - Excludes automation virtual sessions and code_exec sessions; 'all' mode takes up to - persona_max_sessions ordered by last_message_at descending (recent-first sampling). - """ - scope = job.scope or {} - chat_ids = scope.get("chat_ids") - - if isinstance(chat_ids, list) and chat_ids: - ids = [str(c) for c in chat_ids] - total = len(ids) - if len(ids) > cfg.persona_max_sessions: - ids = ids[: cfg.persona_max_sessions] - return ids, total - - q = db.query(ChatSession.chat_id).filter( - ChatSession.user_id == job.target_user_id, - ChatSession.deleted_at.is_(None), - or_( - ChatSession.extra_data.is_(None), - sa.and_( - ~func.cast(ChatSession.extra_data, sa.Text).contains('"automation_run"'), - ~func.cast(ChatSession.extra_data, sa.Text).contains('"code_exec_chat"'), - ), - ), - ) - date_from = scope.get("date_from") - date_to = scope.get("date_to") - if date_from: - q = q.filter(ChatSession.last_message_at >= date_from) - if date_to: - q = q.filter(ChatSession.last_message_at < date_to) - - total = q.count() - rows = ( - q.order_by(ChatSession.last_message_at.desc().nullslast()) - .limit(cfg.persona_max_sessions) - .all() - ) - return [r[0] for r in rows], total - - -# ─────────────────────── Memory collection ──────────────────────── - - -def _fact_confidential(item: dict) -> bool: - meta = item.get("metadata") or {} - conf = str(meta.get("confidentiality") or "").strip().lower() - return conf in _CONFIDENTIAL_VALUES - - -def _slim_fact(item: dict) -> Dict[str, Any]: - meta = item.get("metadata") or {} - return { - "memory": str(item.get("memory") or item.get("text") or "")[:500], - "tags": meta.get("tags") or [], - } - - -def _audit_memory_read(db: Session, *, actor: str, user_id: str, workspace_id: str, job_id: str) -> None: - db.add( - MemoryAudit( - actor=actor, - action="read", - layer="batch", - user_id=user_id, - workspace_id=workspace_id[:64], - reason=f"persona_distill:{job_id}", - ) - ) - - -async def _fetch_workspace_memories( - scope_user_id: str, - workspace_id: Optional[str], - profile_user_id: Optional[str] = None, -) -> Tuple[str, List[dict]]: - """Read one workspace's L1 profile (optional) and raw L2 entries; either failure is treated as empty.""" - from core.memory.profile import get as profile_get - from core.memory.service import get_all_memories - - profile = "" - if profile_user_id: - try: - profile = await profile_get(profile_user_id, workspace_id or "default") - except Exception: - profile = "" - try: - raw = await get_all_memories(scope_user_id, workspace_id=workspace_id) - except Exception: - raw = [] - return profile, raw - - -def _slim_facts(raw: List[dict], author_user_id: Optional[str] = None) -> List[dict]: - """Confidentiality filtering + (optional) author filtering + slimming.""" - return [ - _slim_fact(it) - for it in raw - if not _fact_confidential(it) - and ( - author_user_id is None - or ((it.get("metadata") or {}).get("author_user_id")) == author_user_id - ) - ] - - -async def collect_colleague_memories( - db: Session, - *, - target_user_id: str, - job_id: str, - actor: str, - include_personal: bool = True, - include_project: bool = True, -) -> Dict[str, Any]: - """Collect the target user's memories across all workspaces, grouped by workspace. - - - Default space: L1 profile + L2 facts (legacy data missing the workspace tag is treated as default) - - Personal projects: per-project L1 + L2 - - Team projects: shared scope team:, keep only entries with author_user_id == target user; - the team-shared L1 profile serves only as a project-background note - Project workspaces are fetched in parallel; each workspace read writes one memory_audit row. - """ - groups: Dict[str, Any] = {} - - if include_personal: - profile, raw = await _fetch_workspace_memories( - target_user_id, None, profile_user_id=target_user_id - ) - facts = _slim_facts([ - it for it in raw - if ((it.get("metadata") or {}).get("workspace_id") or "default") == "default" - ]) - if profile or facts: - groups["个人默认空间"] = {"profile": (profile or "")[:4000], "facts": facts[:80]} - _audit_memory_read(db, actor=actor, user_id=target_user_id, workspace_id="default", job_id=job_id) - - if include_project: - personal_projects = ( - db.query(Project) - .filter( - Project.kind == "personal", - Project.owner_user_id == target_user_id, - Project.deleted_at.is_(None), - ) - .all() - ) - team_ids = [ - r[0] - for r in db.query(TeamMember.team_id) - .filter(TeamMember.user_id == target_user_id) - .all() - ] - team_projects = ( - db.query(Project) - .filter( - Project.kind == "team", - Project.team_id.in_(team_ids), - Project.deleted_at.is_(None), - ) - .all() - ) if team_ids else [] - - # Each workspace is independent I/O; fetch in parallel - results = await asyncio.gather(*( - [ - _fetch_workspace_memories( - target_user_id, f"project:{p.project_id}", profile_user_id=target_user_id - ) - for p in personal_projects - ] - + [ - _fetch_workspace_memories(f"team:{p.team_id}", f"project:{p.project_id}") - for p in team_projects - ] - )) - - for p, (profile, raw) in zip(personal_projects, results[: len(personal_projects)]): - facts = _slim_facts(raw) - if profile or facts: - groups[f"个人项目「{p.name}」"] = { - "profile": (profile or "")[:2000], - "facts": facts[:50], - } - _audit_memory_read( - db, actor=actor, user_id=target_user_id, - workspace_id=f"project:{p.project_id}", job_id=job_id, - ) - for p, (_, raw) in zip(team_projects, results[len(personal_projects):]): - facts = _slim_facts(raw, author_user_id=target_user_id) - if facts: - groups[f"团队项目「{p.name}」（仅本人撰写条目）"] = {"facts": facts[:50]} - _audit_memory_read( - db, actor=actor, user_id=target_user_id, - workspace_id=f"project:{p.project_id}", job_id=job_id, - ) - - db.commit() - return groups - - -async def collect_personal_project_memories( - db: Session, - *, - user_id: str, - chat_ids: List[str], - job_id: str, -) -> Dict[str, Any]: - """personal mode: L2 facts authored by the user, from projects the selected sessions are attached to, as auxiliary corpus.""" - if not chat_ids: - return {} - project_ids = { - r[0] - for r in db.query(ChatSession.project_id) - .filter(ChatSession.chat_id.in_(chat_ids), ChatSession.project_id.isnot(None)) - .all() - if r[0] - } - if not project_ids: - return {} - - groups: Dict[str, Any] = {} - projects = ( - db.query(Project) - .filter(Project.project_id.in_(project_ids), Project.deleted_at.is_(None)) - .all() - ) - results = await asyncio.gather(*( - _fetch_workspace_memories( - f"team:{p.team_id}" if (p.kind == "team" and p.team_id) else user_id, - f"project:{p.project_id}", - ) - for p in projects - )) - for p, (_, raw) in zip(projects, results): - facts = _slim_facts(raw, author_user_id=user_id if p.kind == "team" else None) - if facts: - groups[f"项目「{p.name}」"] = {"facts": facts[:50]} - _audit_memory_read( - db, actor=user_id, user_id=user_id, - workspace_id=f"project:{p.project_id}", job_id=job_id, - ) - db.commit() - return groups - - -# ─────────────────────── Background execution ──────────────────────── - - -def _job_cancelled(db: Session, job_id: str) -> bool: - db.expire_all() - row = ( - db.query(PersonaDistillJob.status) - .filter(PersonaDistillJob.job_id == job_id) - .first() - ) - return bool(row and row[0] == "cancelled") - - -async def run_job(job_id: str) -> None: - db = SessionLocal() - try: - await _run_job_inner(db, job_id) - except Exception as exc: # safety net: any uncaught exception lands the job in failed - logger.exception("persona_distill: job %s crashed", job_id) - try: - db.rollback() - job = get_job(db, job_id) - if job and job.status in ("queued", "running"): - job.status = "failed" - job.error = str(exc)[:2000] - job.finished_at = datetime.utcnow() - db.commit() - except Exception: - logger.exception("persona_distill: job %s failed to persist error", job_id) - finally: - db.close() - - -async def _run_job_inner(db: Session, job_id: str) -> None: - cfg = get_config() - job = get_job(db, job_id) - if job is None or job.status != "queued": - return - job.status = "running" - job.started_at = datetime.utcnow() - db.commit() - - # Each job consumes 1 daily run slot (map's many lightweight calls are not counted again) - allowed, reason = await check_and_reserve_run(cfg) - if not allowed: - job.status = "failed" - job.error = f"budget: {reason}" - job.finished_at = datetime.utcnow() - db.commit() - return - - scope = job.scope or {} - chat_ids, total_candidates = _resolve_chat_ids(db, job, cfg) - if not chat_ids: - job.status = "failed" - job.error = "没有可蒸馏的会话" - job.finished_at = datetime.utcnow() - db.commit() - return - - sampled_ratio = (len(chat_ids) / total_candidates) if total_candidates else 1.0 - job.progress_total = len(chat_ids) - db.commit() - - # ── map: per-session digest (concurrency bounded by persona_map_concurrency) ── - hint = str(scope.get("hint") or "") - total_cost = 0.0 - partial = False - digests: List[Dict[str, Any]] = [] - sem = asyncio.Semaphore(max(1, cfg.persona_map_concurrency)) - cost_lock = asyncio.Lock() - - async def _one(chat_id: str) -> Optional[Dict[str, Any]]: - nonlocal total_cost, partial - async with sem: - async with cost_lock: - if partial: - return None - if total_cost >= cfg.persona_job_max_cost_usd: - partial = True - return None - ok, _reason = await check_cost_budget(cfg) - if not ok: - async with cost_lock: - partial = True - return None - # assemble + LLM (assemble is a synchronous DB read; the volume is small enough to accept) - traj = assemble_trajectory(db, chat_id, cfg) - if traj is None or not traj.turns: - return None - try: - digest, cost = await summarize_session(traj, cfg, hint=hint) - except Exception as exc: - logger.warning("persona_distill: digest failed chat=%s (%s)", chat_id, exc) - return None - async with cost_lock: - total_cost += cost - await record_cost(cost) - return digest - - done = 0 - # Run in batches, so progress persists to DB and cancellation checks can happen - batch_size = max(1, cfg.persona_map_concurrency) * 2 - for i in range(0, len(chat_ids), batch_size): - if _job_cancelled(db, job_id): - return - batch = chat_ids[i : i + batch_size] - results = await asyncio.gather(*(_one(c) for c in batch)) - digests.extend(d for d in results if d) - done += len(batch) - job = get_job(db, job_id) - if job is None or job.status == "cancelled": - return - job.progress_done = min(done, job.progress_total) - job.cost_usd = round(total_cost, 4) - job.intermediate = list(digests) # only a new object triggers JSON column dirty detection - db.commit() - if partial: - break - - useful = [d for d in digests if not d.get("low_value")] - if not useful: - job = get_job(db, job_id) - if job and job.status == "running": - job.status = "failed" - job.error = "所选会话无有效信息量（摘要全部为空/低价值）" - job.cost_usd = round(total_cost, 4) - job.finished_at = datetime.utcnow() - db.commit() - return - - # ── Memory collection ── - memories: Dict[str, Any] = {} - try: - if job.kind == "colleague": - memories = await collect_colleague_memories( - db, - target_user_id=job.target_user_id, - job_id=job_id, - actor=job.requested_by, - include_personal=bool(scope.get("include_memories", True)), - include_project=bool(scope.get("include_project_memories", True)), - ) - else: - if scope.get("include_project_memories", True): - memories = await collect_personal_project_memories( - db, user_id=job.target_user_id, chat_ids=chat_ids, job_id=job_id - ) - except Exception as exc: - logger.warning("persona_distill: memory collection failed for %s (%s)", job_id, exc) - - # ── Naming ── - if job.kind == "colleague": - display_name, skill_id = allocate_colleague_identity(db) - assigned_identity = f"display_name: {display_name}\nskill_id: {skill_id}" - else: - display_name, skill_id = "", "" - assigned_identity = "" - - if _job_cancelled(db, job_id): - return - - # ── reduce ── - try: - from core.ontology.validator import render_runtime_prompt - from core.services.ontology_service import build_user_ontology_runtime - - ontology_task = "\n".join( - filter( - None, - [hint, json.dumps(digests[:5], ensure_ascii=False, default=str)[:8000]], - ) - ) - _, ontology_runtime = build_user_ontology_runtime( - user_id=str(job.target_user_id), - task=ontology_task, - db=db, - ) - ontology_context = render_runtime_prompt(ontology_runtime) - if ontology_context: - hint = "\n\n".join( - filter( - None, - [ - hint, - "生成的技能草稿必须使用规范领域术语并满足完整工作流：\n" - + ontology_context, - ], - ) - ) - except Exception as exc: # The materialization gate still validates the draft. - logger.debug("persona_distill: ontology bootstrap unavailable (%s)", exc) - - effective_ratio = sampled_ratio * (len(digests) / len(chat_ids) if chat_ids else 1.0) - out = await distill_persona( - job.kind, - digests, - memories or None, - assigned_identity, - hint, - round(effective_ratio, 2), - cfg, - ) - total_cost += out.cost_usd - await record_cost(out.cost_usd) - - job = get_job(db, job_id) - if job is None or job.status == "cancelled": - return - job.cost_usd = round(total_cost, 4) - - if out.skill is None: - job.status = "failed" - job.error = f"reduce 阶段失败: {out.error}" - job.finished_at = datetime.utcnow() - db.commit() - return - - # Naming finalization: colleague is forced to use the allocated code name (model output is not trusted). - # The code name is allocated before reduce (the prompt needs it), and reduce takes tens of seconds — - # a concurrent job may have consumed the same code in the meantime, so re-check before persisting; - # if taken, advance to the next code and replace the old code throughout the body text. - fm = out.skill.get("frontmatter") or {} - if job.kind == "colleague": - final_display, final_sid = allocate_colleague_identity(db) - if final_sid != skill_id: - body = out.skill.get("instructions_md") or "" - out.skill["instructions_md"] = ( - body.replace(display_name, final_display).replace(skill_id, final_sid) - ) - out.digest_text = out.digest_text.replace(display_name, final_display) - display_name, skill_id = final_display, final_sid - out.skill["id"] = skill_id - fm["name"] = skill_id - fm["display_name"] = display_name - else: - sid = str(out.skill.get("id") or fm.get("name") or f"personal-{uuid.uuid4().hex[:8]}") - sid = re.sub(r"[^a-z0-9_-]+", "-", sid.lower()).strip("-") or f"personal-{uuid.uuid4().hex[:8]}" - out.skill["id"] = sid - fm["name"] = sid - fm.setdefault("display_name", sid) - out.skill["frontmatter"] = fm - - content = skill_to_markdown(out.skill) - job.result_skill_content = content - job.result_meta = { - "proposed_skill_id": out.skill["id"], - "display_name": str(fm.get("display_name") or out.skill["id"]), - "description": str(fm.get("description") or ""), - "tags": list(fm.get("tags") or []), - "confidence": out.confidence, - "digest_text": out.digest_text, - "sampled_ratio": round(effective_ratio, 2), - "partial": partial, - "session_count": len(chat_ids), - "useful_digests": len(useful), - } - - # colleague: mirror the draft into admin_skill_drafts (unified auditing/review) - if job.kind == "colleague": - draft_id = f"dsk_{uuid.uuid4().hex[:16]}" - db.add( - AdminSkillDraft( - draft_id=draft_id, - proposed_skill_id=out.skill["id"], - decision="new_skill", - display_name=str(fm.get("display_name") or out.skill["id"]), - description=str(fm.get("description") or ""), - tags=list(fm.get("tags") or []), - allowed_tools=list(fm.get("allowed_tools") or []), - version=str(fm.get("version") or "0.1.0"), - skill_content=content, - extra_files={}, - source_chat_id=chat_ids[0] if chat_ids else job.job_id, - source_user_id=job.target_user_id, - source_trace_ids=chat_ids, - trajectory_digest=( - f"人物蒸馏（{len(chat_ids)} 个会话，采样比例 {effective_ratio:.2f}）\n" - f"画像摘要：{out.digest_text}" - ), - distillation_cost_usd=round(total_cost, 4), - draft_kind="colleague", - source_job_id=job.job_id, - review_status="pending", - created_at=datetime.utcnow(), - updated_at=datetime.utcnow(), - ) - ) - job.result_draft_id = draft_id - - job.status = "completed" - job.finished_at = datetime.utcnow() - db.commit() - logger.info( - "persona_distill: job %s completed kind=%s sessions=%d cost=%.4f", - job_id, job.kind, len(chat_ids), total_cost, - ) - - -# ─────────────────────── Persisting personal-job output ──────────────────────── - - -def save_personal_skill( - db: Session, - job: PersonaDistillJob, - *, - edited_content: Optional[str] = None, - enable: bool = True, -) -> AdminSkill: - """Persist a personal job's output as the user's private skill (owner=target_user_id). - - On skill_id collision a numeric suffix is appended automatically; colliding with a public - or someone else's skill likewise switches the id instead of overwriting. - """ - from core.agent_skills.registry import _load_skill_metadata_from_str - - if job.status != "completed" or not job.result_skill_content: - raise BadRequestError("作业尚未完成或无产物") - if job.saved_skill_id: - raise BadRequestError(f"产物已保存为技能 {job.saved_skill_id}") - - content = (edited_content or job.result_skill_content).strip() - try: - meta = _load_skill_metadata_from_str( - content, (job.result_meta or {}).get("proposed_skill_id") or "personal-skill" - ) - except Exception as exc: - raise BadRequestError(f"SKILL.md 解析失败：{exc}") - - base_id = meta.id - taken = { - r[0] - for r in db.query(AdminSkill.skill_id) - .filter(or_(AdminSkill.skill_id == base_id, AdminSkill.skill_id.like(f"{base_id}-%"))) - .all() - } - skill_id = base_id - for i in range(2, 100): - if skill_id not in taken: - break - skill_id = f"{base_id}-{i}" - else: - raise BadRequestError("无法分配可用的技能 ID") - - if skill_id != meta.id: - # Also rewrite the name in the content frontmatter to keep it consistent with the id - content = re.sub( - rf"^name:\s*{re.escape(meta.id)}\s*$", - f"name: {skill_id}", - content, - count=1, - flags=re.MULTILINE, - ) - - ensure_ontology_build_valid( - db, - asset_type="skill", - name=meta.name or skill_id, - description=meta.description or "", - instructions=content, - tool_names=list(meta.allowed_tools or []), - ontology_tags=list(meta.tags or []), - ) - - skill = AdminSkill( - skill_id=skill_id, - skill_content=content, - display_name=meta.name, - description=meta.description, - version=meta.version or "0.1.0", - tags=list(meta.tags or []), - allowed_tools=list(meta.allowed_tools or []), - extra_files={}, - dependencies={}, - is_enabled=bool(enable), - owner_user_id=str(job.target_user_id), - created_by="personal_distill", - created_at=datetime.utcnow(), - updated_at=datetime.utcnow(), - ) - db.add(skill) - job.saved_skill_id = skill_id - if edited_content: - job.result_skill_content = content - db.commit() - db.refresh(skill) - - try: - from core.agent_skills.cache_refresh import refresh_skill_caches - refresh_skill_caches() - except Exception as exc: - logger.warning("persona_distill: cache refresh failed after save (%s)", exc) - return skill diff --git a/src/backend/core/services/plugin_service.py b/src/backend/core/services/plugin_service.py index 2f03c4d4..9f40d481 100644 --- a/src/backend/core/services/plugin_service.py +++ b/src/backend/core/services/plugin_service.py @@ -33,9 +33,6 @@ from pathlib import Path from typing import Any, Dict, Iterator, List, Optional, Tuple -from sqlalchemy.orm import Session -from sqlalchemy.orm.attributes import flag_modified - from core.agent_skills.binary_files import is_binary_value from core.agent_skills.cache_refresh import refresh_skill_caches from core.agent_skills.deps_detector import detect_dependencies @@ -44,6 +41,7 @@ from core.db.models import ( AdminMcpServer, AdminSkill, + ContentBlock, InstalledPlugin, PluginMarketPackage, PluginMarketSkillExclusion, @@ -62,6 +60,8 @@ _rewrite_path_vars, normalize_plugin_dir, ) +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified logger = logging.getLogger(__name__) @@ -75,6 +75,17 @@ MAX_ZIP_BYTES = 50 * 1024 * 1024 # pre-extraction cap for uploaded/imported plugin zips +# Credential-free plugins that form the CE out-of-box experience. Desktop and +# no-Docker installs bootstrap the same slugs from ``cli.py`` before the API +# starts; CE Compose uses the DB-backed marker below because its database volume +# outlives backend container rebuilds. +DEFAULT_BOOTSTRAP_PLUGIN_SLUGS: Tuple[str, ...] = ( + "automation", + "skill-manager", + "sites", +) +DEFAULT_BOOTSTRAP_MARKER_ID = "default_plugins_bootstrap_v1" + def _iter_plugin_dirs(): """Iterate over all plugin bundle directories containing plugin.json under default + marketplace.""" @@ -99,6 +110,7 @@ def _resolve_plugin_dir(slug: str) -> Optional[Path]: # ── id generation (namespaced) ─────────────────────────────────────────────── + def _make_plugin_install_id(slug: str, owner_user_id: Optional[str]) -> str: return f"{slug}@{owner_user_id or 'global'}" @@ -111,7 +123,9 @@ def _sanitize_id(value: str, maxlen: int) -> str: def _make_skill_id(slug: str, skill_name: str, owner_user_id: Optional[str]) -> str: """Namespaced skill id: {slug}-{skill} (+ user fingerprint). Constrained by _ID_RE (<=63).""" base = _sanitize_id(f"{slug}-{skill_name}", 50) - return compute_install_id(base, owner_user_id) # appends -<6-char fingerprint> when owner is non-empty + return compute_install_id( + base, owner_user_id + ) # appends -<6-char fingerprint> when owner is non-empty def _make_server_id(slug: str, server_name: str, owner_user_id: Optional[str]) -> str: @@ -121,6 +135,7 @@ def _make_server_id(slug: str, server_name: str, owner_user_id: Optional[str]) - # ── Rewriting inter-skill relative references ──────────────────────────────── + def _rewrite_sibling_refs(text: str, sibling_ids: Dict[str, str]) -> str: """Rewrite inter-skill relative references ``../`` into ``../``. @@ -155,6 +170,7 @@ def _rewrite_sibling_refs(text: str, sibling_ids: Dict[str, str]) -> str: # ── Persistence: single component ──────────────────────────────────────────── + def _apply_skill( db: Session, sk: NormalizedSkill, @@ -174,7 +190,9 @@ def _apply_skill( content = _rewrite_path_vars(sk.skill_content, skill_sandbox_dir=sandbox_dir) extra_files: Dict[str, str] = {} for k, v in sk.extra_files.items(): - extra_files[k] = v if is_binary_value(v) else _rewrite_path_vars(str(v), skill_sandbox_dir=sandbox_dir) + extra_files[k] = ( + v if is_binary_value(v) else _rewrite_path_vars(str(v), skill_sandbox_dir=sandbox_dir) + ) # Inter-skill relative-reference rewrite: ../ → ../ (body + text attachments) if sibling_ids: @@ -200,9 +218,7 @@ def _apply_skill( except Exception as exc: # noqa: BLE001 raise BadRequestError(message=f"技能 {sk.name!r} 的 SKILL.md 不合法：{exc}") - deps = detect_dependencies( - {fn: c for fn, c in extra_files.items() if not is_binary_value(c)} - ) + deps = detect_dependencies({fn: c for fn, c in extra_files.items() if not is_binary_value(c)}) ensure_ontology_build_valid( db, asset_type="skill", @@ -298,6 +314,7 @@ def _apply_mcp( # ── Persistence: whole plugin ──────────────────────────────────────────────── + def _apply_normalized( db: Session, np: NormalizedPlugin, @@ -329,8 +346,12 @@ def _apply_normalized( for sk in install_skills: sid = _apply_skill( - db, sk, slug=np.slug, owner_user_id=owner_user_id, - secrets=secrets, required_secrets=np.required_secrets, + db, + sk, + slug=np.slug, + owner_user_id=owner_user_id, + secrets=secrets, + required_secrets=np.required_secrets, enabled=(sk.name in de_skills) or not de_skills, sibling_ids=sibling_ids, ) @@ -339,13 +360,22 @@ def _apply_normalized( for mc in np.mcp: sid = _apply_mcp( - db, mc, slug=np.slug, owner_user_id=owner_user_id, + db, + mc, + slug=np.slug, + owner_user_id=owner_user_id, enabled=(mc.name in de_mcp), ) server_ids.append(sid) if mc.needs_runtime: - adapted.append({"type": "mcp", "id": sid, "name": mc.name, - "note": mc.note or "stdio MCP 已装上但禁用，需运行时"}) + adapted.append( + { + "type": "mcp", + "id": sid, + "name": mc.name, + "note": mc.note or "stdio MCP 已装上但禁用，需运行时", + } + ) else: imported.append({"type": "mcp", "id": sid, "name": mc.name}) @@ -355,9 +385,16 @@ def _apply_normalized( now = datetime.utcnow() existing = db.query(InstalledPlugin).filter(InstalledPlugin.install_id == install_id).first() fields = dict( - slug=np.slug, name=np.name, version=np.version, description=np.description, - category=np.category, icon=np.icon, owner_user_id=owner_user_id, - source=source, component_ids=component_ids, import_report=import_report, + slug=np.slug, + name=np.name, + version=np.version, + description=np.description, + category=np.category, + icon=np.icon, + owner_user_id=owner_user_id, + source=source, + component_ids=component_ids, + import_report=import_report, updated_at=now, ) if existing is not None: @@ -367,15 +404,22 @@ def _apply_normalized( flag_modified(existing, col) action = "updated" else: - db.add(InstalledPlugin(install_id=install_id, created_at=now, created_by=created_by, **fields)) + db.add( + InstalledPlugin(install_id=install_id, created_at=now, created_by=created_by, **fields) + ) action = "installed" db.commit() _refresh_after_change(owner_user_id) logger.info( "plugin_%s: slug=%s kind=%s owner=%s skills=%d mcp=%d dropped=%d", - action, np.slug, np.kind, owner_user_id or "global", - len(skill_ids), len(server_ids), len(np.dropped), + action, + np.slug, + np.kind, + owner_user_id or "global", + len(skill_ids), + len(server_ids), + len(np.dropped), ) return { "install_id": install_id, @@ -402,6 +446,7 @@ def _refresh_after_change(owner_user_id: Optional[str]) -> None: logger.debug("invalidate_capability_cache failed: %s", exc) try: from core.services.mcp_service import McpServerConfigService + McpServerConfigService.get_instance().invalidate_cache() except Exception as exc: # noqa: BLE001 logger.debug("mcp cache invalidate failed: %s", exc) @@ -442,9 +487,11 @@ def builtin_plugin_component_ids() -> Tuple[set, set]: # ── Public API ──────────────────────────────────────────────────────────────── + def _scan_native_manifest(plugin_dir: Path) -> Optional[Dict[str, Any]]: """Lightweight read of a builtin plugin bundle's display metadata (no full normalize).""" import json + mp = plugin_dir / "plugin.json" if not mp.is_file(): return None @@ -457,7 +504,8 @@ def _scan_native_manifest(plugin_dir: Path) -> Optional[Dict[str, Any]]: skills_root = plugin_dir / "skills" skills_count = ( sum(1 for c in skills_root.iterdir() if c.is_dir() and (c / "SKILL.md").is_file()) - if skills_root.is_dir() else 0 + if skills_root.is_dir() + else 0 ) return { "slug": _sanitize_id(str(m.get("name")), 100), @@ -510,11 +558,14 @@ def list_plugins( # Annotate installed status if items: - install_ids = {it["slug"]: _make_plugin_install_id(it["slug"], owner_user_id) for it in items} + install_ids = { + it["slug"]: _make_plugin_install_id(it["slug"], owner_user_id) for it in items + } present = { - row[0] for row in db.query(InstalledPlugin.install_id).filter( - InstalledPlugin.install_id.in_(list(install_ids.values())) - ).all() + row[0] + for row in db.query(InstalledPlugin.install_id) + .filter(InstalledPlugin.install_id.in_(list(install_ids.values()))) + .all() } for it in items: it["installed"] = install_ids[it["slug"]] in present @@ -524,9 +575,14 @@ def list_plugins( # annotations. For user-side callers owner_user_id is the current browsing # user, reused directly as the viewer. from core.services import marketplace_listing as ml + items = ml.annotate_and_filter( - db, ml.KIND_PLUGIN, items, id_key="slug", - include_disabled=include_disabled, viewer_user_id=owner_user_id, + db, + ml.KIND_PLUGIN, + items, + id_key="slug", + include_disabled=include_disabled, + viewer_user_id=owner_user_id, ) return items @@ -536,6 +592,7 @@ def set_plugin_market_enabled( ) -> Dict[str, Any]: """Publish/unpublish a marketplace plugin (controls display in the plugin marketplace; does not affect installed instances).""" from core.services import marketplace_listing as ml + res = ml.set_listing_enabled(db, ml.KIND_PLUGIN, slug, enabled, updated_by=updated_by) logger.info("plugin_market_listing: slug=%s enabled=%s", slug, enabled) return res @@ -558,10 +615,12 @@ def list_installed( if owner_user_id is None: q = q.filter(InstalledPlugin.owner_user_id.is_(None)) elif include_global: - q = q.filter(or_( - InstalledPlugin.owner_user_id == owner_user_id, - InstalledPlugin.owner_user_id.is_(None), - )) + q = q.filter( + or_( + InstalledPlugin.owner_user_id == owner_user_id, + InstalledPlugin.owner_user_id.is_(None), + ) + ) else: q = q.filter(InstalledPlugin.owner_user_id == owner_user_id) rows = q.order_by(InstalledPlugin.created_at.desc()).all() @@ -573,10 +632,7 @@ def list_installed( # the list). if owner_user_id is not None and include_global: global_slugs = {r.slug for r in rows if r.owner_user_id is None} - rows = [ - r for r in rows - if r.owner_user_id is None or r.slug not in global_slugs - ] + rows = [r for r in rows if r.owner_user_id is None or r.slug not in global_slugs] # Enabled state: # - user view (owner_user_id non-empty): determined by the user's # "effectively enabled" set (including per-user overrides), so the user's @@ -584,6 +640,7 @@ def list_installed( # - admin view (owner_user_id empty): determined by components' global is_enabled. if owner_user_id is not None: from core.config.catalog_resolver import resolve_all_runtime_enabled + eff_skills, _eff_agents, eff_mcps = resolve_all_runtime_enabled(db, owner_user_id) enabled_skills = set(eff_skills or []) enabled_mcps = set(eff_mcps or []) @@ -594,22 +651,35 @@ def list_installed( cids = r.component_ids or {} all_skill_ids.update(cids.get("skills") or []) all_mcp_ids.update(cids.get("mcp") or []) - enabled_skills = { - row[0] for row in db.query(AdminSkill.skill_id).filter( - AdminSkill.skill_id.in_(all_skill_ids), AdminSkill.is_enabled.is_(True) - ).all() - } if all_skill_ids else set() - enabled_mcps = { - row[0] for row in db.query(AdminMcpServer.server_id).filter( - AdminMcpServer.server_id.in_(all_mcp_ids), AdminMcpServer.is_enabled.is_(True) - ).all() - } if all_mcp_ids else set() + enabled_skills = ( + { + row[0] + for row in db.query(AdminSkill.skill_id) + .filter(AdminSkill.skill_id.in_(all_skill_ids), AdminSkill.is_enabled.is_(True)) + .all() + } + if all_skill_ids + else set() + ) + enabled_mcps = ( + { + row[0] + for row in db.query(AdminMcpServer.server_id) + .filter( + AdminMcpServer.server_id.in_(all_mcp_ids), AdminMcpServer.is_enabled.is_(True) + ) + .all() + } + if all_mcp_ids + else set() + ) out: List[Dict[str, Any]] = [] for r in rows: cids = r.component_ids or {} - enabled = any(s in enabled_skills for s in (cids.get("skills") or [])) or \ - any(m in enabled_mcps for m in (cids.get("mcp") or [])) + enabled = any(s in enabled_skills for s in (cids.get("skills") or [])) or any( + m in enabled_mcps for m in (cids.get("mcp") or []) + ) out.append(_installed_to_dict(r, enabled=enabled)) return out @@ -659,6 +729,7 @@ def get_installed_detail( eff_mcps: Optional[set] = None if owner_user_id is not None: from core.config.catalog_resolver import resolve_all_runtime_enabled + es, _ea, em = resolve_all_runtime_enabled(db, owner_user_id) eff_skills, eff_mcps = set(es or []), set(em or []) @@ -668,38 +739,48 @@ def get_installed_detail( extra = s.extra_files or {} # secrets.json is not shown as an ordinary file files = sorted(k for k in extra.keys() if k != "secrets.json") - sk_enabled = (s.skill_id in eff_skills) if eff_skills is not None else bool(s.is_enabled) - skills_out.append({ - "skill_id": s.skill_id, - "name": s.display_name or s.skill_id, - "description": s.description or "", - "version": s.version or "", - "tags": list(s.tags or []), - "enabled": sk_enabled, - "instructions": _strip_frontmatter(s.skill_content), - "files": files, - "has_secrets": "secrets.json" in extra, - }) + sk_enabled = ( + (s.skill_id in eff_skills) if eff_skills is not None else bool(s.is_enabled) + ) + skills_out.append( + { + "skill_id": s.skill_id, + "name": s.display_name or s.skill_id, + "description": s.description or "", + "version": s.version or "", + "tags": list(s.tags or []), + "enabled": sk_enabled, + "instructions": _strip_frontmatter(s.skill_content), + "files": files, + "has_secrets": "secrets.json" in extra, + } + ) mcp_out: List[Dict[str, Any]] = [] if server_ids: for m in db.query(AdminMcpServer).filter(AdminMcpServer.server_id.in_(server_ids)).all(): raw_tools = m.tools_json or [] tools = [ - {"name": str(tdef.get("name") or ""), "description": str(tdef.get("description") or "")} - for tdef in raw_tools if isinstance(tdef, dict) + { + "name": str(tdef.get("name") or ""), + "description": str(tdef.get("description") or ""), + } + for tdef in raw_tools + if isinstance(tdef, dict) ] mc_enabled = (m.server_id in eff_mcps) if eff_mcps is not None else bool(m.is_enabled) - mcp_out.append({ - "server_id": m.server_id, - "name": m.display_name or m.server_id, - "description": m.description or "", - "transport": m.transport, - "url": m.url, - "enabled": mc_enabled, - "needs_runtime": m.transport == "stdio", - "tools": tools, - }) + mcp_out.append( + { + "server_id": m.server_id, + "name": m.display_name or m.server_id, + "description": m.description or "", + "transport": m.transport, + "url": m.url, + "enabled": mc_enabled, + "needs_runtime": m.transport == "stdio", + "tools": tools, + } + ) return { "install_id": row.install_id, @@ -727,6 +808,7 @@ def get_installed_detail( # field keys the plugin declared (prevents privilege escalation into writing # arbitrary system config). + def _admin_config_for_slug(slug: str) -> Optional[Dict[str, Any]]: """Locate the plugin bundle by slug and read its admin_config declaration; None if absent.""" plugin_dir = _resolve_plugin_dir(slug) @@ -750,6 +832,7 @@ def _connection_for_slug(slug: str) -> Optional[str]: return None try: import json + m = json.loads((plugin_dir / "plugin.json").read_text(encoding="utf-8")) conn = m.get("connection") return str(conn).strip() if conn else None @@ -764,6 +847,7 @@ def _has_admin_config_for_slug(slug: str) -> bool: return False try: import json + m = json.loads((plugin_dir / "plugin.json").read_text(encoding="utf-8")) ac = m.get("admin_config") return isinstance(ac, dict) and bool(ac.get("fields")) @@ -782,7 +866,9 @@ def _admin_config_configured(mode: str, sets: List[bool]) -> bool: return any(sets) if mode == "any" else all(sets) -def _admin_config_view(admin_config: Optional[Dict[str, Any]], *, with_values: bool) -> Optional[Dict[str, Any]]: +def _admin_config_view( + admin_config: Optional[Dict[str, Any]], *, with_values: bool +) -> Optional[Dict[str, Any]]: """Compute the current admin_config state. with_values=False (user side, read-only): returns only is_set + @@ -802,10 +888,15 @@ def _admin_config_view(admin_config: Optional[Dict[str, Any]], *, with_values: b raw = svc.get(f["key"]) s = _is_set(raw) sets.append(s) - item = {"key": f["key"], "label": f["label"], "secret": f["secret"], - "description": f["description"], "is_set": s} + item = { + "key": f["key"], + "label": f["label"], + "secret": f["secret"], + "description": f["description"], + "is_set": s, + } if with_values: - item["value"] = (("****" if s else "") if f["secret"] else (raw or "")) + item["value"] = ("****" if s else "") if f["secret"] else (raw or "") fields.append(item) mode = admin_config.get("mode") or "all" return { @@ -885,7 +976,7 @@ def get_plugin_detail(slug: str, db: Optional[Session] = None) -> Dict[str, Any] def exclude_market_skill( db: Session, slug: str, skill_name: str, *, created_by: Optional[str] = None ) -> Dict[str, Any]: - """"Remove" a skill from a marketplace plugin: record one exclusion (idempotent). + """ "Remove" a skill from a marketplace plugin: record one exclusion (idempotent). Validates the skill actually belongs to this plugin (guards against typos); afterwards the marketplace list/detail/install no longer include @@ -912,6 +1003,7 @@ def exclude_market_skill( def _skill_component_preview(sk: NormalizedSkill) -> Dict[str, Any]: """Preview component for a not-yet-installed builtin/external skill: includes body instructions + file list for pre-install inspection.""" from core.agent_skills.registry import _split_frontmatter + desc = "" tags: List[str] = [] try: @@ -929,14 +1021,16 @@ def _skill_component_preview(sk: NormalizedSkill) -> Dict[str, Any]: "description": desc, "version": "", "tags": tags, - "enabled": True, # enabled by default after install (preview semantics) + "enabled": True, # enabled by default after install (preview semantics) "instructions": _strip_frontmatter(sk.skill_content), "files": files, "has_secrets": False, } -def _normalized_to_detail(np: NormalizedPlugin, *, excluded: Optional[set] = None) -> Dict[str, Any]: +def _normalized_to_detail( + np: NormalizedPlugin, *, excluded: Optional[set] = None +) -> Dict[str, Any]: excluded = excluded or set() return { "slug": np.slug, @@ -952,9 +1046,14 @@ def _normalized_to_detail(np: NormalizedPlugin, *, excluded: Optional[set] = Non "skills": [_skill_component_preview(s) for s in np.skills if s.name not in excluded], "mcp": [ { - "server_id": m.name, "name": m.name, "description": m.description, - "transport": m.transport, "url": m.url, "enabled": not m.needs_runtime, - "needs_runtime": m.needs_runtime, "note": m.note, + "server_id": m.name, + "name": m.name, + "description": m.description, + "transport": m.transport, + "url": m.url, + "enabled": not m.needs_runtime, + "needs_runtime": m.needs_runtime, + "note": m.note, "tools": list(getattr(m, "tools", None) or []), } for m in np.mcp @@ -976,20 +1075,66 @@ def install_plugin( if plugin_dir is not None: np = normalize_plugin_dir(plugin_dir) return _apply_normalized( - db, np, owner_user_id=owner_user_id, secrets=secrets or {}, - source="builtin", created_by=created_by, + db, + np, + owner_user_id=owner_user_id, + secrets=secrets or {}, + source="builtin", + created_by=created_by, ) # DB-published marketplace package: extract the original zip and go through the same path as import row = _market_row(db, slug) if row is not None: with _extract_plugin_zip(base64.b64decode(row.package_b64)) as plugin_root: return import_plugin( - db, plugin_root, owner_user_id=owner_user_id, - secrets=secrets or {}, created_by=created_by, + db, + plugin_root, + owner_user_id=owner_user_id, + secrets=secrets or {}, + created_by=created_by, ) raise ResourceNotFoundError("plugin", slug) +def ensure_default_plugins_bootstrapped(db: Session) -> bool: + """Install the CE default plugin set exactly once for a persistent DB. + + The marker is deliberately independent from the current installation rows: + after the first successful bootstrap, uninstalling a default plugin is a + user choice and must survive backend/container restarts. A partial failure + writes no marker, so the next startup idempotently retries the complete set. + + Returns ``True`` only when this call completes the first bootstrap. + """ + marker = db.query(ContentBlock).filter(ContentBlock.id == DEFAULT_BOOTSTRAP_MARKER_ID).first() + if marker is not None: + return False + + installed: List[str] = [] + try: + for slug in DEFAULT_BOOTSTRAP_PLUGIN_SLUGS: + install_plugin( + db, + slug, + owner_user_id=None, + created_by="system_bootstrap", + ) + installed.append(slug) + except Exception: + db.rollback() + raise + + db.add( + ContentBlock( + id=DEFAULT_BOOTSTRAP_MARKER_ID, + payload={"version": 1, "plugins": installed}, + updated_by="system_bootstrap", + ) + ) + db.commit() + return True + + def import_plugin( db: Session, plugin_dir: Path, @@ -1002,13 +1147,18 @@ def import_plugin( np = normalize_plugin_dir(plugin_dir) source = {"claude": "imported_claude", "codex": "imported_codex"}.get(np.kind, "builtin") return _apply_normalized( - db, np, owner_user_id=owner_user_id, secrets=secrets or {}, - source=source, created_by=created_by, + db, + np, + owner_user_id=owner_user_id, + secrets=secrets or {}, + source=source, + created_by=created_by, ) def _locate_plugin_root(extract_dir: Path) -> Path: """Locate the plugin root in the extraction dir: prefer the directory containing a manifest; zips often wrap an extra directory level.""" + def _has_manifest(d: Path) -> bool: return ( (d / "plugin.json").is_file() @@ -1024,7 +1174,9 @@ def _has_manifest(d: Path) -> bool: for d in sorted(extract_dir.rglob("*")): if d.is_dir() and _has_manifest(d): return d - raise BadRequestError(message="zip 内未找到插件清单（plugin.json / .claude-plugin / .codex-plugin）") + raise BadRequestError( + message="zip 内未找到插件清单（plugin.json / .claude-plugin / .codex-plugin）" + ) @contextmanager @@ -1060,13 +1212,17 @@ def import_plugin_from_zip( """Import a plugin from uploaded zip bytes (extract → locate root → import_plugin). Shared by user and admin uploads.""" with _extract_plugin_zip(raw) as plugin_root: return import_plugin( - db, plugin_root, owner_user_id=owner_user_id, - secrets=secrets, created_by=created_by, + db, + plugin_root, + owner_user_id=owner_user_id, + secrets=secrets, + created_by=created_by, ) # ── Plugin marketplace DB publishing (admin upload → persisted as an installable source; not installed, not globally in effect) ────────── + def _market_row(db: Session, slug: str) -> Optional[PluginMarketPackage]: if not slug: return None @@ -1098,16 +1254,25 @@ def publish_plugin_zip_to_market( explicitly installed by admins/users; re-uploading the same slug updates it. """ with _extract_plugin_zip(raw) as plugin_root: - np = normalize_plugin_dir(plugin_root) # parse/validate; invalid input raises immediately, nothing persisted + np = normalize_plugin_dir( + plugin_root + ) # parse/validate; invalid input raises immediately, nothing persisted package_b64 = base64.b64encode(raw).decode("ascii") has_admin_config = bool(np.admin_config and (np.admin_config.get("fields"))) now = datetime.utcnow() existing = _market_row(db, np.slug) fields = dict( - name=np.name, version=np.version, description=np.description or "", - category=np.category or "", icon=np.icon, kind=np.kind, - skills_count=len(np.skills), required_secrets=list(np.required_secrets or []), - has_admin_config=has_admin_config, package_b64=package_b64, updated_at=now, + name=np.name, + version=np.version, + description=np.description or "", + category=np.category or "", + icon=np.icon, + kind=np.kind, + skills_count=len(np.skills), + required_secrets=list(np.required_secrets or []), + has_admin_config=has_admin_config, + package_b64=package_b64, + updated_at=now, ) if existing is not None: for key, val in fields.items(): @@ -1118,7 +1283,9 @@ def publish_plugin_zip_to_market( db.add(PluginMarketPackage(slug=np.slug, created_at=now, created_by=created_by, **fields)) action = "published" db.commit() - logger.info("plugin_market_%s: slug=%s kind=%s skills=%d", action, np.slug, np.kind, len(np.skills)) + logger.info( + "plugin_market_%s: slug=%s kind=%s skills=%d", action, np.slug, np.kind, len(np.skills) + ) return { "slug": np.slug, "name": np.name, @@ -1140,7 +1307,9 @@ def delete_market_package(db: Session, slug: str) -> Dict[str, Any]: return {"slug": slug, "deleted": True} -def uninstall_plugin(db: Session, install_id: str, *, owner_user_id: Optional[str]) -> Dict[str, Any]: +def uninstall_plugin( + db: Session, install_id: str, *, owner_user_id: Optional[str] +) -> Dict[str, Any]: """Uninstall a plugin: precisely reverse-delete skills/MCP by source_plugin + owner, then delete the installation record.""" row = db.query(InstalledPlugin).filter(InstalledPlugin.install_id == install_id).first() if row is None: @@ -1158,12 +1327,16 @@ def _owner_filter(model): else model.owner_user_id.is_(None) ) - n_sk = db.query(AdminSkill).filter( - AdminSkill.source_plugin == slug, _owner_filter(AdminSkill) - ).delete(synchronize_session=False) - n_mcp = db.query(AdminMcpServer).filter( - AdminMcpServer.source_plugin == slug, _owner_filter(AdminMcpServer) - ).delete(synchronize_session=False) + n_sk = ( + db.query(AdminSkill) + .filter(AdminSkill.source_plugin == slug, _owner_filter(AdminSkill)) + .delete(synchronize_session=False) + ) + n_mcp = ( + db.query(AdminMcpServer) + .filter(AdminMcpServer.source_plugin == slug, _owner_filter(AdminMcpServer)) + .delete(synchronize_session=False) + ) db.delete(row) db.commit() @@ -1248,7 +1421,12 @@ def set_plugin_component_enabled( effective = srv.is_enabled db.commit() _refresh_after_change(owner_user_id) - return {"install_id": install_id, "kind": kind, "component_id": component_id, "enabled": effective} + return { + "install_id": install_id, + "kind": kind, + "component_id": component_id, + "enabled": effective, + } def set_plugin_enabled_for_user( @@ -1271,6 +1449,7 @@ def set_plugin_enabled_for_user( raise BadRequestError(message="无权操作该插件") from core.services.catalog_service import CatalogService + svc = CatalogService(db) cids = row.component_ids or {} for sid in cids.get("skills") or []: diff --git a/src/backend/core/services/project_file_service.py b/src/backend/core/services/project_file_service.py index da203b6e..918d01d9 100644 --- a/src/backend/core/services/project_file_service.py +++ b/src/backend/core/services/project_file_service.py @@ -1,250 +1,155 @@ -"""Project files = artifacts under the subtree of a linked folder (personal / team). - -Design principles: -- A project itself does not "own" files; it is merely a view of some MySpace folder. -- Upload / delete / create-subfolder all go entirely through MySpace's existing - services (``UserFolderService`` / ``TeamFolderService`` + the file_upload route). -- This service is only responsible for browsing (listing the subtree), a convenience - wrapper for creating subfolders, and capacity statistics. -""" +"""Personal project-file service for Community Edition.""" from __future__ import annotations -import logging import os import uuid -from datetime import datetime from typing import Any, Dict, List, Optional, Tuple +from core.db.models import Artifact, Project, UserFolder +from core.storage import get_storage +from fastapi import HTTPException from sqlalchemy import func from sqlalchemy.orm import Session -from core.db.models import ( - Artifact, - Project, - TeamFolder, - UserFolder, -) -from core.db.repository import AuditLogRepository -from core.storage import get_storage - -logger = logging.getLogger(__name__) - _MAX_UPLOAD_BYTES = 50 * 1024 * 1024 -_DEFAULT_PROJECT_CAPACITY_BYTES = 200 * 1024 * 1024 # 200 MB +_DEFAULT_PROJECT_CAPACITY_BYTES = 200 * 1024 * 1024 def _capacity_limit() -> int: - raw = os.getenv("PROJECT_FILE_CAPACITY_BYTES") - if not raw: - return _DEFAULT_PROJECT_CAPACITY_BYTES try: - n = int(raw) - return max(n, 1) + return max(int(os.getenv("PROJECT_FILE_CAPACITY_BYTES", "")), 1) except (TypeError, ValueError): return _DEFAULT_PROJECT_CAPACITY_BYTES -def _artifact_to_dict(art: Artifact, folder_path: str) -> Dict[str, Any]: +def _artifact_to_dict(artifact: Artifact, folder_path: str) -> Dict[str, Any]: return { - "id": art.artifact_id, - "artifact_id": art.artifact_id, - # Relative path within the project: subfolder/file.ext or file.ext - "name": (folder_path + "/" + art.filename) if folder_path else art.filename, - "title": art.title, - "mime_type": art.mime_type, - "size_bytes": int(art.size_bytes or 0), - "download_url": f"/files/{art.artifact_id}", - "type": art.type, + "id": artifact.artifact_id, + "artifact_id": artifact.artifact_id, + "name": f"{folder_path}/{artifact.filename}" if folder_path else artifact.filename, + "title": artifact.title, + "mime_type": artifact.mime_type, + "size_bytes": int(artifact.size_bytes or 0), + "download_url": f"/files/{artifact.artifact_id}", + "type": artifact.type, "folder_path": folder_path, - "created_at": (art.updated_at or art.created_at).isoformat() if (art.updated_at or art.created_at) else None, + "created_at": ( + (artifact.updated_at or artifact.created_at).isoformat() + if artifact.updated_at or artifact.created_at + else None + ), } class ProjectFileService: - """Project file browsing / creating subfolders / capacity statistics.""" - def __init__(self, db: Session): self.db = db - self.audit_repo = AuditLogRepository(db) - - # ── Read ────────────────────────────────────────────────────────── - def list_files(self, project: Project) -> List[Dict[str, Any]]: - """Recursively traverse all live artifacts under the linked folder's subtree, - returned flattened by ``folder_path``. - ``folder_path`` looks like ``"q1"`` or ``"q1/sub"``, relative to the linked - folder root. The frontend can aggregate by the first path segment to form - folder cards. - """ - if project.kind == "personal": - if not project.linked_folder_id: - return [] - return self._list_user_subtree(project.owner_user_id, project.linked_folder_id) - if project.kind == "team": - if not project.linked_team_folder_id or not project.team_id: - return [] - return self._list_team_subtree(project.team_id, project.linked_team_folder_id) - return [] - - def _list_user_subtree(self, user_id: str, root_id: str) -> List[Dict[str, Any]]: - out: List[Dict[str, Any]] = [] - stack: List[Tuple[str, str]] = [(root_id, "")] # (folder_id, path_rel_to_root) - guard = 0 - while stack and guard < 5000: - guard += 1 - fid, path = stack.pop() - arts = ( - self.db.query(Artifact) - .filter( - Artifact.user_id == user_id, - Artifact.team_id.is_(None), - Artifact.user_folder_id == fid, - Artifact.deleted_at.is_(None), - ) - .order_by(Artifact.created_at.desc()) - .all() - ) - for a in arts: - if a.filename: - out.append(_artifact_to_dict(a, path)) + def _user_subtree_ids(self, user_id: str, root: str) -> List[str]: + out: List[str] = [] + stack = [root] + seen: set[str] = set() + while stack and len(seen) < 5000: + folder_id = stack.pop() + if folder_id in seen: + continue + seen.add(folder_id) + out.append(folder_id) children = ( - self.db.query(UserFolder.folder_id, UserFolder.name) + self.db.query(UserFolder.folder_id) .filter( UserFolder.user_id == user_id, - UserFolder.parent_folder_id == fid, + UserFolder.parent_folder_id == folder_id, UserFolder.deleted_at.is_(None), ) .all() ) - for child_id, child_name in children: - sub = (path + "/" + child_name) if path else child_name - stack.append((child_id, sub)) + stack.extend(row[0] for row in children) return out - def _list_team_subtree(self, team_id: str, root_id: str) -> List[Dict[str, Any]]: + def list_files(self, project: Project) -> List[Dict[str, Any]]: + if project.kind != "personal" or not project.linked_folder_id: + return [] out: List[Dict[str, Any]] = [] - stack: List[Tuple[str, str]] = [(root_id, "")] - guard = 0 - while stack and guard < 5000: - guard += 1 - fid, path = stack.pop() - arts = ( + stack: List[Tuple[str, str]] = [(project.linked_folder_id, "")] + while stack: + folder_id, path = stack.pop() + artifacts = ( self.db.query(Artifact) .filter( - Artifact.team_id == team_id, - Artifact.team_folder_id == fid, + Artifact.user_id == project.owner_user_id, + Artifact.user_folder_id == folder_id, Artifact.deleted_at.is_(None), ) .order_by(Artifact.created_at.desc()) .all() ) - for a in arts: - if a.filename: - out.append(_artifact_to_dict(a, path)) + out.extend(_artifact_to_dict(item, path) for item in artifacts if item.filename) children = ( - self.db.query(TeamFolder.folder_id, TeamFolder.name) + self.db.query(UserFolder.folder_id, UserFolder.name) .filter( - TeamFolder.team_id == team_id, - TeamFolder.parent_folder_id == fid, - TeamFolder.deleted_at.is_(None), + UserFolder.user_id == project.owner_user_id, + UserFolder.parent_folder_id == folder_id, + UserFolder.deleted_at.is_(None), ) .all() ) for child_id, child_name in children: - sub = (path + "/" + child_name) if path else child_name - stack.append((child_id, sub)) + stack.append((child_id, f"{path}/{child_name}" if path else child_name)) return out - # ── Capacity (byte total accumulated over the whole linked-folder subtree) ── def capacity_used(self, project: Project) -> int: - if project.kind == "personal": - if not project.linked_folder_id: - return 0 - ids = self._user_subtree_ids(project.owner_user_id, project.linked_folder_id) - if not ids: - return 0 - total = ( - self.db.query(func.coalesce(func.sum(Artifact.size_bytes), 0)) - .filter( - Artifact.user_id == project.owner_user_id, - Artifact.team_id.is_(None), - Artifact.user_folder_id.in_(ids), - Artifact.deleted_at.is_(None), - ) - .scalar() + if project.kind != "personal" or not project.linked_folder_id: + return 0 + folder_ids = self._user_subtree_ids(project.owner_user_id, project.linked_folder_id) + return int( + self.db.query(func.coalesce(func.sum(Artifact.size_bytes), 0)) + .filter( + Artifact.user_id == project.owner_user_id, + Artifact.user_folder_id.in_(folder_ids), + Artifact.deleted_at.is_(None), ) - return int(total or 0) - if project.kind == "team": - if not project.linked_team_folder_id or not project.team_id: - return 0 - ids = self._team_subtree_ids(project.team_id, project.linked_team_folder_id) - if not ids: - return 0 - total = ( - self.db.query(func.coalesce(func.sum(Artifact.size_bytes), 0)) - .filter( - Artifact.team_id == project.team_id, - Artifact.team_folder_id.in_(ids), - Artifact.deleted_at.is_(None), - ) - .scalar() - ) - return int(total or 0) - return 0 + .scalar() + or 0 + ) def capacity_limit(self) -> int: return _capacity_limit() - def _user_subtree_ids(self, user_id: str, root: str) -> List[str]: - out: List[str] = [] - stack = [root] - seen: set[str] = set() - guard = 0 - while stack and guard < 5000: - guard += 1 - fid = stack.pop() - if fid in seen: - continue - seen.add(fid) - out.append(fid) - children = ( - self.db.query(UserFolder.folder_id) + def _ensure_user_subfolder_chain( + self, + user_id: str, + root_folder_id: Optional[str], + names: List[str], + *, + actor: str, + ) -> Optional[str]: + from core.services.user_folder_service import UserFolderService + + parent_id = root_folder_id + for name in names: + existing = ( + self.db.query(UserFolder) .filter( UserFolder.user_id == user_id, - UserFolder.parent_folder_id == fid, + UserFolder.parent_folder_id == parent_id, + UserFolder.name == name, UserFolder.deleted_at.is_(None), ) - .all() + .first() ) - stack.extend(r[0] for r in children) - return out - - def _team_subtree_ids(self, team_id: str, root: str) -> List[str]: - out: List[str] = [] - stack = [root] - seen: set[str] = set() - guard = 0 - while stack and guard < 5000: - guard += 1 - fid = stack.pop() - if fid in seen: + if existing is not None: + parent_id = existing.folder_id continue - seen.add(fid) - out.append(fid) - children = ( - self.db.query(TeamFolder.folder_id) - .filter( - TeamFolder.team_id == team_id, - TeamFolder.parent_folder_id == fid, - TeamFolder.deleted_at.is_(None), - ) - .all() + result = UserFolderService(self.db).create_folder( + user_id=user_id, parent_folder_id=parent_id, name=name, actor=actor ) - stack.extend(r[0] for r in children) - return out + if not result.ok or not result.folder_id: + raise HTTPException(status_code=400, detail=result.message or "新建子文件夹失败") + parent_id = result.folder_id + return parent_id - # ── Upload (write under the linked folder, optionally creating subfolders by subpath) ── def upload( self, project: Project, @@ -253,178 +158,48 @@ def upload( filename: str, mime_type: Optional[str], ) -> Dict[str, Any]: - """Write bytes directly to the linked folder (or the subfolder resolved from the - ``filename`` path). - - When ``filename`` looks like ``"q1.xlsx"``, write to the linked folder root; - when ``"folder/sub/file.ext"``, mkdir the subfolders as needed and then write - the file. - """ - from fastapi import HTTPException - - if not filename: - raise HTTPException(status_code=400, detail="文件名不能为空") - if not file_bytes: - raise HTTPException(status_code=400, detail="文件内容为空") + if project.kind != "personal": + raise HTTPException(status_code=404, detail="项目不存在") + if not filename or not file_bytes: + raise HTTPException(status_code=400, detail="文件名或内容为空") if len(file_bytes) > _MAX_UPLOAD_BYTES: raise HTTPException(status_code=413, detail="单文件最大 50 MB") - - used = self.capacity_used(project) - limit = self.capacity_limit() - if used + len(file_bytes) > limit: - raise HTTPException( - status_code=400, - detail=f"项目容量不足（已用 {used} / 上限 {limit} 字节）", - ) - - # Split the filename path - rel = filename.replace("\\", "/").strip("/") - parts = [p for p in rel.split("/") if p and p not in (".", "..")] + if self.capacity_used(project) + len(file_bytes) > self.capacity_limit(): + raise HTTPException(status_code=400, detail="项目容量不足") + parts = [ + part + for part in filename.replace("\\", "/").strip("/").split("/") + if part not in ("", ".", "..") + ] if not parts: raise HTTPException(status_code=400, detail="文件名不合法") - leaf = parts[-1] - dirs = parts[:-1] - - if project.kind == "personal": - target_folder_id = self._ensure_user_subfolder_chain( - project.owner_user_id, project.linked_folder_id, dirs, actor=user_id - ) - else: - target_folder_id = self._ensure_team_subfolder_chain( - project.team_id, project.linked_team_folder_id, dirs, actor=user_id - ) - - env = os.getenv("ENVIRONMENT", "dev") - artifact_id = f"pj_{uuid.uuid4().hex[:16]}" - owner_user_id = ( - project.owner_user_id if project.kind == "personal" else user_id + leaf, directories = parts[-1], parts[:-1] + folder_id = self._ensure_user_subfolder_chain( + project.owner_user_id, + project.linked_folder_id, + directories, + actor=user_id, ) - storage_prefix = ( - f"{env}/{owner_user_id}/user_uploads/{artifact_id}/{leaf}" - if project.kind == "personal" - else f"{env}/teams/{project.team_id}/{artifact_id}/{leaf}" - ) - try: - storage_url = get_storage().upload_bytes(file_bytes, storage_prefix) - except Exception as exc: - logger.warning("[project_file] upload_bytes 失败 key=%s: %s", storage_prefix, exc) - raise HTTPException(status_code=500, detail=f"文件上传失败: {exc}") - + artifact_id = f"pj_{uuid.uuid4().hex[:16]}" + storage_key = f"{os.getenv('ENVIRONMENT', 'dev')}/{project.owner_user_id}/user_uploads/{artifact_id}/{leaf}" + storage_url = get_storage().upload_bytes(file_bytes, storage_key) artifact = Artifact( artifact_id=artifact_id, - user_id=owner_user_id, - team_id=project.team_id if project.kind == "team" else None, - team_folder_id=target_folder_id if project.kind == "team" else None, - user_folder_id=target_folder_id if project.kind == "personal" else None, + user_id=project.owner_user_id, + user_folder_id=folder_id, type="other", title=leaf, filename=leaf, size_bytes=len(file_bytes), mime_type=mime_type or "application/octet-stream", - storage_key=storage_prefix, + storage_key=storage_key, storage_url=storage_url, - extra_data={ - "source": "user_upload", - "via_project": project.project_id, - }, + extra_data={"source": "user_upload", "via_project": project.project_id}, ) self.db.add(artifact) self.db.commit() self.db.refresh(artifact) + return _artifact_to_dict(artifact, "/".join(directories)) - self.audit_repo.create({ - "user_id": user_id, - "action": "project_file.uploaded", - "resource_type": "project", - "resource_id": project.project_id, - "details": { - "artifact_id": artifact_id, - "filename": filename, - "size": len(file_bytes), - "target_folder_id": target_folder_id, - }, - "status": "success", - }) - folder_path = "/".join(dirs) - return _artifact_to_dict(artifact, folder_path) - - def _ensure_user_subfolder_chain( - self, - user_id: str, - root_folder_id: Optional[str], - names: List[str], - *, - actor: str, - ) -> Optional[str]: - from core.services.user_folder_service import UserFolderService - - svc = UserFolderService(self.db) - parent_id = root_folder_id - for name in names: - existing = ( - self.db.query(UserFolder) - .filter( - UserFolder.user_id == user_id, - UserFolder.parent_folder_id == parent_id, - UserFolder.name == name, - UserFolder.deleted_at.is_(None), - ) - .first() - ) - if existing is not None: - parent_id = existing.folder_id - continue - res = svc.create_folder( - user_id=user_id, - parent_folder_id=parent_id, - name=name, - actor=actor, - ) - if not res.ok or not res.folder_id: - from fastapi import HTTPException - raise HTTPException(status_code=400, detail=res.message or "新建子文件夹失败") - parent_id = res.folder_id - return parent_id - - def _ensure_team_subfolder_chain( - self, - team_id: str, - root_folder_id: Optional[str], - names: List[str], - *, - actor: str, - ) -> Optional[str]: - try: - from core.services.team_folder_service import TeamFolderService - except ModuleNotFoundError: # CE: no team folders, the team path is unreachable - from fastapi import HTTPException - raise HTTPException(status_code=404, detail="团队功能在当前版本不可用") - - svc = TeamFolderService(self.db) - parent_id = root_folder_id - for name in names: - existing = ( - self.db.query(TeamFolder) - .filter( - TeamFolder.team_id == team_id, - TeamFolder.parent_folder_id == parent_id, - TeamFolder.name == name, - TeamFolder.deleted_at.is_(None), - ) - .first() - ) - if existing is not None: - parent_id = existing.folder_id - continue - res = svc.create_folder( - team_id=team_id, - parent_folder_id=parent_id, - name=name, - actor=actor, - ) - if not res.ok or not res.folder_id: - from fastapi import HTTPException - raise HTTPException(status_code=400, detail=res.message or "新建团队子文件夹失败") - parent_id = res.folder_id - return parent_id +__all__ = ["ProjectFileService"] diff --git a/src/backend/core/services/project_scope.py b/src/backend/core/services/project_scope.py index 950b7f5b..942f7af3 100644 --- a/src/backend/core/services/project_scope.py +++ b/src/backend/core/services/project_scope.py @@ -1,22 +1,9 @@ -"""ProjectScope — the unified type for project-chat scope. - -Constructed once at the chats.py entry point and passed **explicitly** down the call chain to -everywhere that needs it (agent_factory → register_* tool closures → myspace_vfs.py helpers → -_persist_artifacts). - -Compared with the previous ContextVar model: -- No set/reset timing window; a missing scope is a parameter/type-level issue and no longer - silently falls back to the personal root (which once caused a bug where AI-generated files - under a team project leaked into the personal MySpace root). -- The scope follows the call chain rather than being thread-local, so it remains valid across - async generator finally boundaries. -- In tests just construct one directly; no need to monkeypatch a contextvar. -""" +"""Personal project scope for Community Edition.""" from __future__ import annotations from dataclasses import dataclass -from typing import Any, Literal, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Optional if TYPE_CHECKING: from sqlalchemy.orm import Session @@ -24,82 +11,38 @@ @dataclass(frozen=True) class ProjectScope: - """Immutable description of a project-chat scope. - - Attributes: - project_id: Project ID. - kind: ``"personal"`` or ``"team"`` — decides whether to use UserFolder or TeamFolder. - root_folder_id: Linked folder ID (personal=UserFolder.folder_id; - team=TeamFolder.folder_id). All /myspace paths are confined to this subtree. - folder_name: Name of the linked folder. Used to auto-complete a "bare relative path" - like ``/myspace/foo.txt`` into ``/myspace//foo.txt``. - team_id: Only valid for team kind; points to the team ID so artifact queries use - team_id rather than user_id (visible across members). - """ - project_id: str - kind: Literal["personal", "team"] + kind: str root_folder_id: str folder_name: str - team_id: Optional[str] = None @property - def is_team(self) -> bool: - return self.kind == "team" + def is_personal(self) -> bool: + return True @property - def is_personal(self) -> bool: - return self.kind == "personal" + def is_team(self) -> bool: + return False def project_scope_from_context(ctx: dict[str, Any]) -> Optional[ProjectScope]: - """Extract a ProjectScope from the workflow context dict built by chats.py. - - Missing any of project_id / project_folder_id / project_folder_kind → return None - (meaning a non-project chat; never construct a dummy scope, so downstream code can - reliably check for None). - - team kind must carry project_team_id, otherwise also return None (defensive: avoids - constructing a team scope missing team_id that would misdirect downstream queries). - """ - if not ctx.get("project_id"): + if not ctx.get("project_id") or ctx.get("project_folder_kind") != "personal": return None folder_id = ctx.get("project_folder_id") if not folder_id: return None - kind = ctx.get("project_folder_kind") - if kind not in ("personal", "team"): - return None - team_id: Optional[str] = None - if kind == "team": - team_id = ctx.get("project_team_id") or None - if not team_id: - return None return ProjectScope( project_id=str(ctx["project_id"]), - kind=kind, # type: ignore[arg-type] + kind="personal", root_folder_id=str(folder_id), folder_name=str(ctx.get("project_folder_name") or ""), - team_id=team_id, ) -def project_scope_from_chat_id( - db: "Session", chat_id: Optional[str] -) -> Optional[ProjectScope]: - """Look up the owning project by chat_id and construct a ProjectScope. - - For entry points that don't hold the workflow context dict (e.g. the plan-execute - background worker, cancel/cleanup paths). Returns None when the chat isn't in a project / - the chat doesn't exist / the project's linked folder is missing. - - Guarantees field semantics consistent with :func:`project_scope_from_context`: - team kind must carry team_id; none may be missing. - """ +def project_scope_from_chat_id(db: "Session", chat_id: Optional[str]) -> Optional[ProjectScope]: if not chat_id: return None - # Deferred import to avoid a top-level circular dependency (models depends on db.engine ↔ services) - from core.db.models import ChatSession, Project, TeamFolder, UserFolder + from core.db.models import ChatSession, Project, UserFolder chat = ( db.query(ChatSession) @@ -110,110 +53,68 @@ def project_scope_from_chat_id( return None project = ( db.query(Project) - .filter(Project.project_id == chat.project_id, Project.deleted_at.is_(None)) + .filter( + Project.project_id == chat.project_id, + Project.kind == "personal", + Project.deleted_at.is_(None), + ) .first() ) - if project is None: + if project is None or not project.linked_folder_id: return None - if project.kind == "personal" and project.linked_folder_id: - row = ( - db.query(UserFolder.name) - .filter(UserFolder.folder_id == project.linked_folder_id) - .first() - ) - return ProjectScope( - project_id=project.project_id, - kind="personal", - root_folder_id=project.linked_folder_id, - folder_name=row[0] if row else "", - team_id=None, - ) - if project.kind == "team" and project.linked_team_folder_id and project.team_id: - row = ( - db.query(TeamFolder.name) - .filter(TeamFolder.folder_id == project.linked_team_folder_id) - .first() - ) - return ProjectScope( - project_id=project.project_id, - kind="team", - root_folder_id=project.linked_team_folder_id, - folder_name=row[0] if row else "", - team_id=project.team_id, - ) - return None + row = db.query(UserFolder.name).filter(UserFolder.folder_id == project.linked_folder_id).first() + return ProjectScope( + project_id=project.project_id, + kind="personal", + root_folder_id=project.linked_folder_id, + folder_name=row[0] if row else "", + ) def build_project_ctx(db: "Session", project_id: Optional[str]) -> Optional[dict]: - """Build the project context dict (for injecting into the agent system prompt + resolving ProjectScope). - - Returned fields align with ``routing.workflow._PROJECT_CTX_KEYS``; returns None when - ``project_id`` is missing or the project doesn't exist. Reused by entry points that - **don't go through ``chats._build_ctx``** (e.g. plan mode), avoiding duplication of that - project metadata query logic. - """ if not project_id: return None - # Deferred import to avoid a top-level circular dependency. - from core.db.models import Project, TeamFolder, UserFolder + from core.db.models import Project, UserFolder from core.services.project_file_service import ProjectFileService project = ( db.query(Project) - .filter(Project.project_id == project_id, Project.deleted_at.is_(None)) + .filter( + Project.project_id == project_id, + Project.kind == "personal", + Project.deleted_at.is_(None), + ) .first() ) if project is None: return None - - folder_name: Optional[str] = None - folder_kind: Optional[str] = None - folder_id: Optional[str] = None - team_id: Optional[str] = None - if project.kind == "personal" and project.linked_folder_id: + folder_name = None + if project.linked_folder_id: row = ( db.query(UserFolder.name) .filter(UserFolder.folder_id == project.linked_folder_id) .first() ) folder_name = row[0] if row else None - folder_kind = "personal" - folder_id = project.linked_folder_id - elif project.kind == "team" and project.linked_team_folder_id: - row = ( - db.query(TeamFolder.name) - .filter(TeamFolder.folder_id == project.linked_team_folder_id) - .first() - ) - folder_name = row[0] if row else None - folder_kind = "team" - folder_id = project.linked_team_folder_id - team_id = project.team_id - try: - project_files = ProjectFileService(db).list_files(project) + files = ProjectFileService(db).list_files(project) except Exception: - project_files = [] - + files = [] return { "project_id": project_id, "project_name": project.name, "project_instructions": (project.instructions or "").strip() or None, "project_folder_name": folder_name, - "project_folder_kind": folder_kind, - "project_folder_id": folder_id, - "project_team_id": team_id, - "project_files": project_files, + "project_folder_kind": "personal" if project.linked_folder_id else None, + "project_folder_id": project.linked_folder_id, + "project_files": files, + "memory_scope_user_id": None, + "_memory_enabled": bool((project.extra_data or {}).get("memory_enabled", True)), + "_memory_write_enabled": bool((project.extra_data or {}).get("memory_write_enabled", True)), } def build_project_ctx_from_chat_id(db: "Session", chat_id: Optional[str]) -> Optional[dict]: - """Reverse lookup ``chat_id`` → ``ChatSession.project_id`` → :func:`build_project_ctx`. - - The plan-mode background worker holds no workflow context dict, only a chat_id, so it uses - this reverse-lookup path to build the project context (same source as - :func:`project_scope_from_chat_id`). - """ if not chat_id: return None from core.db.models import ChatSession @@ -223,15 +124,38 @@ def build_project_ctx_from_chat_id(db: "Session", chat_id: Optional[str]) -> Opt .filter(ChatSession.chat_id == chat_id, ChatSession.deleted_at.is_(None)) .first() ) - if chat is None or not chat.project_id: + return build_project_ctx(db, chat.project_id) if chat and chat.project_id else None + + +def project_memory_policy( + db: "Session", project_id: str, user_id: str +) -> Optional[tuple[bool, str]]: + from core.db.models import Project + + project = ( + db.query(Project) + .filter( + Project.project_id == project_id, + Project.kind == "personal", + Project.deleted_at.is_(None), + ) + .first() + ) + if project is None: return None - return build_project_ctx(db, chat.project_id) + return bool((project.extra_data or {}).get("memory_enabled", True)), user_id + + +def edition_project_context_keys() -> tuple[str, ...]: + return () __all__ = [ "ProjectScope", - "project_scope_from_context", - "project_scope_from_chat_id", "build_project_ctx", "build_project_ctx_from_chat_id", + "edition_project_context_keys", + "project_memory_policy", + "project_scope_from_chat_id", + "project_scope_from_context", ] diff --git a/src/backend/core/services/project_service.py b/src/backend/core/services/project_service.py index 7ecf80ce..677de7b0 100644 --- a/src/backend/core/services/project_service.py +++ b/src/backend/core/services/project_service.py @@ -1,95 +1,53 @@ -"""Project (Claude-style workspace) business layer. - -Handles project CRUD, listing (with mixed personal+team visibility), favorites, -and activity refresh. File-related operations live in -:mod:`core.services.project_file_service`. -""" +"""Personal-project service for Community Edition.""" from __future__ import annotations import uuid -from dataclasses import dataclass from datetime import datetime from typing import Any, Dict, List, Optional, Tuple -from sqlalchemy import and_, desc, func, or_ +from core.auth.permissions_iface import ProjectPermissionLevel, resolve_project_permission +from core.db.models import Artifact, ChatSession, Project, ProjectFavorite, UserFolder, UserShadow +from fastapi import HTTPException +from sqlalchemy import desc, func, or_ from sqlalchemy.orm import Session -from core.auth.permissions_iface import ( - ProjectPermissionLevel, - can_create_team_project, - resolve_project_permission, -) -from core.db.models import ( - Artifact, - ChatSession, - ChatSessionUserState, - Project, - ProjectFavorite, - Team, - TeamFolder, - TeamMember, - UserFolder, - UserShadow, -) -from core.db.repository import AuditLogRepository - - -@dataclass -class ProjectWithAccess: - project: Project - level: ProjectPermissionLevel - favorite: bool - def _next_available_folder_name(base: str, existing: List) -> str: - """Pick a non-conflicting folder name under the root (appending (2), (3) suffixes). - - ``existing`` is query rows of the shape ``[(name,), ...]``. - """ base = (base or "").strip() or "新项目" - used = {row[0] for row in (existing or [])} + used = {row[0] for row in existing or []} if base not in used: return base - n = 2 - while True: - candidate = f"{base} ({n})" - if candidate not in used: - return candidate - n += 1 + index = 2 + while f"{base} ({index})" in used: + index += 1 + return f"{base} ({index})" -def _project_to_summary( +def _summary( project: Project, *, - level: ProjectPermissionLevel, favorite: bool, - team_name: Optional[str] = None, - folder_name: Optional[str] = None, - file_count: int = 0, - chat_count: int = 0, + folder_name: Optional[str], + file_count: int, + chat_count: int, ) -> Dict[str, Any]: - """Unified serialization (shared by list / get).""" extra = project.extra_data or {} return { "project_id": project.project_id, "name": project.name, "description": project.description or "", - "kind": project.kind, + "kind": "personal", "owner_user_id": project.owner_user_id, - "team_id": project.team_id, - "team_name": team_name, "linked_folder_id": project.linked_folder_id, - "linked_team_folder_id": project.linked_team_folder_id, "folder_name": folder_name, "instructions": project.instructions or "", "icon_color": project.icon_color, "pinned": bool(project.pinned), "favorite": favorite, - # Project-level memory read/write switches: default ON when absent (both new and legacy projects count as enabled). "memory_enabled": bool(extra.get("memory_enabled", True)), "memory_write_enabled": bool(extra.get("memory_write_enabled", True)), - "permission": level, + "permission": "admin", "file_count": file_count, "chat_count": chat_count, "metadata": extra, @@ -104,18 +62,44 @@ def _project_to_summary( class ProjectService: def __init__(self, db: Session): self.db = db - self.audit_repo = AuditLogRepository(db) - # ── Visibility helpers ──────────────────────────────────────────── - def _visible_team_ids(self, user_id: str) -> List[str]: - rows = ( - self.db.query(TeamMember.team_id) - .filter(TeamMember.user_id == user_id) - .all() + def _subtree_ids(self, root_id: str, user_id: str) -> List[str]: + out: List[str] = [] + stack = [root_id] + seen: set[str] = set() + while stack and len(seen) < 5000: + folder_id = stack.pop() + if folder_id in seen: + continue + seen.add(folder_id) + out.append(folder_id) + children = ( + self.db.query(UserFolder.folder_id) + .filter( + UserFolder.user_id == user_id, + UserFolder.parent_folder_id == folder_id, + UserFolder.deleted_at.is_(None), + ) + .all() + ) + stack.extend(row[0] for row in children) + return out + + def _file_count(self, project: Project) -> int: + if not project.linked_folder_id: + return 0 + folder_ids = self._subtree_ids(project.linked_folder_id, project.owner_user_id) + return int( + self.db.query(func.count(Artifact.artifact_id)) + .filter( + Artifact.user_id == project.owner_user_id, + Artifact.user_folder_id.in_(folder_ids), + Artifact.deleted_at.is_(None), + ) + .scalar() + or 0 ) - return [r[0] for r in rows] - # ── List ────────────────────────────────────────────────────────── def list_visible( self, user_id: str, @@ -125,207 +109,77 @@ def list_visible( page: int = 1, page_size: int = 30, ) -> Tuple[List[Dict[str, Any]], int]: - """List projects visible to the current user (personal owner=me ∪ team where me∈team_members).""" - team_ids = self._visible_team_ids(user_id) - visibility = or_( - and_(Project.kind == "personal", Project.owner_user_id == user_id), - and_(Project.kind == "team", Project.team_id.in_(team_ids)) if team_ids - else and_(Project.kind == "team", Project.team_id == "__never__"), - ) - - base_q = self.db.query(Project).filter( + query = self.db.query(Project).filter( + Project.kind == "personal", + Project.owner_user_id == user_id, Project.deleted_at.is_(None), - visibility, ) if q: pattern = f"%{q.strip()}%" - base_q = base_q.filter( + query = query.filter( or_(Project.name.ilike(pattern), Project.description.ilike(pattern)) ) - - total = base_q.count() - - # sort + total = query.count() if sort == "name": - base_q = base_q.order_by(Project.name.asc()) + query = query.order_by(Project.name.asc()) elif sort == "created": - base_q = base_q.order_by(desc(Project.created_at)) - else: # default '-last_activity_at' - base_q = base_q.order_by(desc(Project.pinned), desc(Project.last_activity_at)) - - projects = ( - base_q.offset((page - 1) * page_size).limit(page_size).all() - ) - - # batch fetch: favorites, team names, chat counts (file_count is now computed per - # folder subtree — small scale, calculated separately in the loop) - project_ids = [p.project_id for p in projects] - fav_ids = set() - team_name_by_id: Dict[str, str] = {} - chat_count_by_id: Dict[str, int] = {} - folder_names_user: Dict[str, str] = {} - folder_names_team: Dict[str, str] = {} - if project_ids: - fav_rows = ( - self.db.query(ProjectFavorite.project_id) + query = query.order_by(desc(Project.created_at)) + else: + query = query.order_by(desc(Project.pinned), desc(Project.last_activity_at)) + projects = query.offset((page - 1) * page_size).limit(page_size).all() + ids = [project.project_id for project in projects] + favorite_ids = ( + { + row[0] + for row in self.db.query(ProjectFavorite.project_id) .filter( ProjectFavorite.user_id == user_id, - ProjectFavorite.project_id.in_(project_ids), + ProjectFavorite.project_id.in_(ids), ) .all() - ) - fav_ids = {r[0] for r in fav_rows} - - distinct_team_ids = [p.team_id for p in projects if p.team_id] - if distinct_team_ids: - t_rows = ( - self.db.query(Team.team_id, Team.name) - .filter(Team.team_id.in_(distinct_team_ids)) - .all() - ) - team_name_by_id = {tid: name for tid, name in t_rows} - - # chat_count: chats I own ∪ chats shared by others within team projects - team_project_ids = [p.project_id for p in projects if p.kind == "team"] - cc_or_terms = [ChatSession.user_id == user_id] - if team_project_ids: - cc_or_terms.append( - and_( - ChatSession.project_id.in_(team_project_ids), - ChatSession.share_scope.in_(("team_read", "team_edit")), - ) + } + if ids + else set() + ) + chat_counts = ( + { + project_id: int(count) + for project_id, count in self.db.query( + ChatSession.project_id, func.count(ChatSession.chat_id) ) - cc_filter = or_(*cc_or_terms) if len(cc_or_terms) > 1 else cc_or_terms[0] - cc_rows = ( - self.db.query(ChatSession.project_id, func.count(ChatSession.chat_id)) .filter( - ChatSession.project_id.in_(project_ids), + ChatSession.project_id.in_(ids), + ChatSession.user_id == user_id, ChatSession.deleted_at.is_(None), - cc_filter, ) .group_by(ChatSession.project_id) .all() - ) - chat_count_by_id = {pid: int(n) for pid, n in cc_rows} - - uf_ids = [p.linked_folder_id for p in projects if p.linked_folder_id] - if uf_ids: - folder_names_user = { - fid: name - for fid, name in self.db.query(UserFolder.folder_id, UserFolder.name) - .filter(UserFolder.folder_id.in_(uf_ids)) - .all() - } - tf_ids = [p.linked_team_folder_id for p in projects if p.linked_team_folder_id] - if tf_ids: - folder_names_team = { - fid: name - for fid, name in self.db.query(TeamFolder.folder_id, TeamFolder.name) - .filter(TeamFolder.folder_id.in_(tf_ids)) - .all() - } - - items = [ - _project_to_summary( - p, - level=resolve_project_permission(self.db, user_id, p), - favorite=(p.project_id in fav_ids), - team_name=team_name_by_id.get(p.team_id) if p.team_id else None, - folder_name=( - folder_names_user.get(p.linked_folder_id) - if p.kind == "personal" - else folder_names_team.get(p.linked_team_folder_id) - ), - file_count=self._count_files_in_subtree(p), - chat_count=chat_count_by_id.get(p.project_id, 0), - ) - for p in projects - ] - return items, total - - def _count_files_in_subtree(self, project: Project) -> int: - """Count live artifacts under the subtree of the folder linked to this project.""" - if project.kind == "personal": - if not project.linked_folder_id: - return 0 - ids = self._user_folder_subtree_ids(project.linked_folder_id, project.owner_user_id) - if not ids: - return 0 - return int( - self.db.query(func.count(Artifact.artifact_id)) - .filter( - Artifact.user_id == project.owner_user_id, - Artifact.team_id.is_(None), - Artifact.user_folder_id.in_(ids), - Artifact.deleted_at.is_(None), - ) - .scalar() or 0 - ) - # team - if not project.linked_team_folder_id or not project.team_id: - return 0 - ids = self._team_folder_subtree_ids(project.linked_team_folder_id, project.team_id) - if not ids: - return 0 - return int( - self.db.query(func.count(Artifact.artifact_id)) - .filter( - Artifact.team_id == project.team_id, - Artifact.team_folder_id.in_(ids), - Artifact.deleted_at.is_(None), - ) - .scalar() or 0 + } + if ids + else {} ) - - def _user_folder_subtree_ids(self, root_id: str, user_id: str) -> List[str]: - out: List[str] = [] - stack = [root_id] - seen: set[str] = set() - guard = 0 - while stack and guard < 5000: - guard += 1 - fid = stack.pop() - if fid in seen: - continue - seen.add(fid) - out.append(fid) - children = ( - self.db.query(UserFolder.folder_id) - .filter( - UserFolder.user_id == user_id, - UserFolder.parent_folder_id == fid, - UserFolder.deleted_at.is_(None), - ) - .all() - ) - stack.extend(r[0] for r in children) - return out - - def _team_folder_subtree_ids(self, root_id: str, team_id: str) -> List[str]: - out: List[str] = [] - stack = [root_id] - seen: set[str] = set() - guard = 0 - while stack and guard < 5000: - guard += 1 - fid = stack.pop() - if fid in seen: - continue - seen.add(fid) - out.append(fid) - children = ( - self.db.query(TeamFolder.folder_id) - .filter( - TeamFolder.team_id == team_id, - TeamFolder.parent_folder_id == fid, - TeamFolder.deleted_at.is_(None), - ) + folder_ids = [project.linked_folder_id for project in projects if project.linked_folder_id] + folder_names = ( + { + folder_id: name + for folder_id, name in self.db.query(UserFolder.folder_id, UserFolder.name) + .filter(UserFolder.folder_id.in_(folder_ids)) .all() + } + if folder_ids + else {} + ) + return [ + _summary( + project, + favorite=project.project_id in favorite_ids, + folder_name=folder_names.get(project.linked_folder_id), + file_count=self._file_count(project), + chat_count=chat_counts.get(project.project_id, 0), ) - stack.extend(r[0] for r in children) - return out + for project in projects + ], total - # ── Create ──────────────────────────────────────────────────────── def create_personal( self, user_id: str, @@ -334,255 +188,97 @@ def create_personal( *, linked_folder_id: Optional[str] = None, ) -> Project: - """Create a personal project. - - If ``linked_folder_id`` is given, link to that existing personal folder; - otherwise create a new user_folder named after the project at the personal - folder root as the link target. One folder may be linked to only one live - project (checked manually in the service layer to avoid relying on a - PG-only partial unique index). - """ - return self._create( - user_id=user_id, - kind="personal", - name=name, - description=description, - team_id=None, - linked_folder_id=linked_folder_id, - linked_team_folder_id=None, - ) - - def create_team( - self, - user_id: str, - team_id: str, - name: str, - description: Optional[str] = None, - *, - linked_team_folder_id: Optional[str] = None, - ) -> Project: - if not can_create_team_project(self.db, user_id, team_id): - from fastapi import HTTPException - self.audit_repo.log_denial( - user_id=user_id, - action="project.create_team", - reason="not_team_admin", - required="admin", - actual="member_or_none", - resource_type="team", - resource_id=team_id, - ) - raise HTTPException(status_code=403, detail="仅团队 owner / admin 可创建团队项目") - return self._create( - user_id=user_id, - kind="team", - name=name, - description=description, - team_id=team_id, - linked_folder_id=None, - linked_team_folder_id=linked_team_folder_id, - ) - - def _create( - self, - *, - user_id: str, - kind: str, - name: str, - description: Optional[str], - team_id: Optional[str], - linked_folder_id: Optional[str], - linked_team_folder_id: Optional[str], - ) -> Project: - from fastapi import HTTPException - clean = (name or "").strip() if not clean: raise HTTPException(status_code=400, detail="项目名不能为空") if len(clean) > 120: raise HTTPException(status_code=400, detail="项目名过长（≤120 字）") - - # Duplicate-name check: personal projects with the same owner / team projects within the same team may not share a name (live records) - dup_q = self.db.query(Project.project_id).filter( - Project.kind == kind, - Project.name == clean, - Project.deleted_at.is_(None), + duplicate = ( + self.db.query(Project.project_id) + .filter( + Project.kind == "personal", + Project.owner_user_id == user_id, + Project.name == clean, + Project.deleted_at.is_(None), + ) + .first() ) - if kind == "personal": - dup_q = dup_q.filter(Project.owner_user_id == user_id) - else: - dup_q = dup_q.filter(Project.team_id == team_id) - if dup_q.first() is not None: + if duplicate: raise HTTPException(status_code=400, detail="同名项目已存在") - - # ── Resolve or create the linked folder ──────────────── - if kind == "personal": - if linked_folder_id: - fld = ( - self.db.query(UserFolder) - .filter( - UserFolder.folder_id == linked_folder_id, - UserFolder.user_id == user_id, - UserFolder.deleted_at.is_(None), - ) - .first() - ) - if fld is None: - raise HTTPException(status_code=400, detail="目标文件夹不存在") - if self._user_folder_already_linked(linked_folder_id): - raise HTTPException(status_code=400, detail="该文件夹已被其它项目挂钩") - else: - # Auto-create a personal folder with the same name (appending a -N suffix on collision) - folder_name = self._unique_personal_folder_name(user_id, clean) - res = self._get_folder_service("personal").create_folder( - user_id=user_id, - parent_folder_id=None, - name=folder_name, - actor=user_id, - ) - if not res.ok or not res.folder_id: - raise HTTPException(status_code=400, detail=res.message or "新建项目文件夹失败") - linked_folder_id = res.folder_id - else: # team - if linked_team_folder_id: - tfld = ( - self.db.query(TeamFolder) - .filter( - TeamFolder.folder_id == linked_team_folder_id, - TeamFolder.team_id == team_id, - TeamFolder.deleted_at.is_(None), - ) - .first() + if linked_folder_id: + folder = ( + self.db.query(UserFolder) + .filter( + UserFolder.folder_id == linked_folder_id, + UserFolder.user_id == user_id, + UserFolder.deleted_at.is_(None), ) - if tfld is None: - raise HTTPException(status_code=400, detail="目标团队文件夹不存在") - if self._team_folder_already_linked(linked_team_folder_id): - raise HTTPException(status_code=400, detail="该团队文件夹已被其它项目挂钩") - else: - folder_name = self._unique_team_folder_name(team_id, clean) - res = self._get_folder_service("team").create_folder( - team_id=team_id, - parent_folder_id=None, - name=folder_name, - actor=user_id, + .first() + ) + if folder is None: + raise HTTPException(status_code=400, detail="目标文件夹不存在") + if ( + self.db.query(Project.project_id) + .filter( + Project.linked_folder_id == linked_folder_id, + Project.deleted_at.is_(None), ) - if not res.ok or not res.folder_id: - raise HTTPException(status_code=400, detail=res.message or "新建团队项目文件夹失败") - linked_team_folder_id = res.folder_id + .first() + ): + raise HTTPException(status_code=400, detail="该文件夹已被其它项目挂钩") + else: + from core.services.user_folder_service import UserFolderService + folder_name = _next_available_folder_name( + clean, + self.db.query(UserFolder.name) + .filter( + UserFolder.user_id == user_id, + UserFolder.parent_folder_id.is_(None), + UserFolder.deleted_at.is_(None), + ) + .all(), + ) + result = UserFolderService(self.db).create_folder( + user_id=user_id, parent_folder_id=None, name=folder_name, actor=user_id + ) + if not result.ok or not result.folder_id: + raise HTTPException(status_code=400, detail=result.message or "新建项目文件夹失败") + linked_folder_id = result.folder_id + now = datetime.utcnow() project = Project( project_id=f"prj_{uuid.uuid4().hex[:16]}", name=clean, description=(description or "").strip() or None, - kind=kind, + kind="personal", owner_user_id=user_id, - team_id=team_id, linked_folder_id=linked_folder_id, - linked_team_folder_id=linked_team_folder_id, - instructions=None, - icon_color=None, - pinned=False, extra_data={}, + pinned=False, + created_at=now, + updated_at=now, + last_activity_at=now, ) - now = datetime.utcnow() - project.created_at = now - project.updated_at = now - project.last_activity_at = now self.db.add(project) self.db.commit() self.db.refresh(project) - - self.audit_repo.create({ - "user_id": user_id, - "action": "project.created", - "resource_type": "project", - "resource_id": project.project_id, - "details": { - "kind": kind, - "team_id": team_id, - "name": clean, - "linked_folder_id": linked_folder_id, - "linked_team_folder_id": linked_team_folder_id, - }, - "status": "success", - }) return project - # ── Folder helpers ──────────────────────────────────────────────── - def _get_folder_service(self, kind: str): - """Folder service factory (seam C6). - - Imports are kept inside the branches: the CE derived tree does not contain - team_folder_service; in single-tenant mode kind is always personal, so the - team branch is never reached. - """ - if kind == "personal": - from core.services.user_folder_service import UserFolderService - return UserFolderService(self.db) - from core.services.team_folder_service import TeamFolderService - return TeamFolderService(self.db) - - def _user_folder_already_linked(self, folder_id: str) -> bool: - return ( - self.db.query(Project.project_id) - .filter( - Project.linked_folder_id == folder_id, - Project.deleted_at.is_(None), - ) - .first() - is not None - ) - - def _team_folder_already_linked(self, folder_id: str) -> bool: + def get_raw(self, project_id: str) -> Optional[Project]: return ( - self.db.query(Project.project_id) + self.db.query(Project) .filter( - Project.linked_team_folder_id == folder_id, + Project.project_id == project_id, + Project.kind == "personal", Project.deleted_at.is_(None), ) .first() - is not None - ) - - def _unique_personal_folder_name(self, user_id: str, base: str) -> str: - return _next_available_folder_name( - base, - existing=self.db.query(UserFolder.name) - .filter( - UserFolder.user_id == user_id, - UserFolder.parent_folder_id.is_(None), - UserFolder.deleted_at.is_(None), - ) - .all(), - ) - - def _unique_team_folder_name(self, team_id: str, base: str) -> str: - return _next_available_folder_name( - base, - existing=self.db.query(TeamFolder.name) - .filter( - TeamFolder.team_id == team_id, - TeamFolder.parent_folder_id.is_(None), - TeamFolder.deleted_at.is_(None), - ) - .all(), ) - # ── Read ────────────────────────────────────────────────────────── - def get( - self, project_id: str, user_id: str - ) -> Optional[Dict[str, Any]]: - project = ( - self.db.query(Project) - .filter(Project.project_id == project_id, Project.deleted_at.is_(None)) - .first() - ) - if project is None: - return None - level = resolve_project_permission(self.db, user_id, project) - if level == "none": + def get(self, project_id: str, user_id: str) -> Optional[Dict[str, Any]]: + project = self.get_raw(project_id) + if project is None or resolve_project_permission(self.db, user_id, project) == "none": return None - favorite = ( self.db.query(ProjectFavorite) .filter( @@ -592,61 +288,31 @@ def get( .first() is not None ) - team_name = None - if project.team_id: - t = self.db.query(Team.name).filter(Team.team_id == project.team_id).first() - team_name = t[0] if t else None - folder_name: Optional[str] = None - if project.kind == "personal" and project.linked_folder_id: - row = ( - self.db.query(UserFolder.name) - .filter(UserFolder.folder_id == project.linked_folder_id) - .first() - ) - folder_name = row[0] if row else None - elif project.kind == "team" and project.linked_team_folder_id: - row = ( - self.db.query(TeamFolder.name) - .filter(TeamFolder.folder_id == project.linked_team_folder_id) - .first() - ) - folder_name = row[0] if row else None - file_count = self._count_files_in_subtree(project) - # Team project: count includes my own ∪ chats in the project with share_scope ∈ (team_read, team_edit); - # personal project / non-project: only my own - chat_count_q = self.db.query(func.count(ChatSession.chat_id)).filter( - ChatSession.project_id == project_id, - ChatSession.deleted_at.is_(None), + folder = ( + self.db.query(UserFolder.name) + .filter(UserFolder.folder_id == project.linked_folder_id) + .first() + if project.linked_folder_id + else None ) - if project.kind == "team": - chat_count_q = chat_count_q.filter( - or_( - ChatSession.user_id == user_id, - ChatSession.share_scope.in_(("team_read", "team_edit")), - ) + chat_count = int( + self.db.query(func.count(ChatSession.chat_id)) + .filter( + ChatSession.project_id == project_id, + ChatSession.user_id == user_id, + ChatSession.deleted_at.is_(None), ) - else: - chat_count_q = chat_count_q.filter(ChatSession.user_id == user_id) - chat_count = chat_count_q.scalar() or 0 - return _project_to_summary( + .scalar() + or 0 + ) + return _summary( project, - level=level, favorite=favorite, - team_name=team_name, - folder_name=folder_name, - file_count=int(file_count), - chat_count=int(chat_count), + folder_name=folder[0] if folder else None, + file_count=self._file_count(project), + chat_count=chat_count, ) - def get_raw(self, project_id: str) -> Optional[Project]: - """Fetch the raw ORM row without authorization (for the permission layer / internal use only).""" - return ( - self.db.query(Project) - .filter(Project.project_id == project_id, Project.deleted_at.is_(None)) - .first() - ) - - # ── Update ──────────────────────────────────────────────────────── def update( self, project_id: str, @@ -655,80 +321,53 @@ def update( *, level: ProjectPermissionLevel, ) -> Optional[Dict[str, Any]]: - from fastapi import HTTPException - project = self.get_raw(project_id) if project is None: return None - - # admin-only fields - ADMIN_FIELDS = {"name", "pinned", "icon_color"} - EDIT_FIELDS = {"description", "instructions", "memory_enabled", "memory_write_enabled"} - for key in patch.keys(): - if key in ADMIN_FIELDS and level != "admin": + admin_fields = {"name", "pinned", "icon_color"} + edit_fields = {"description", "instructions", "memory_enabled", "memory_write_enabled"} + for key in patch: + if key in admin_fields and level != "admin": raise HTTPException(status_code=403, detail=f"仅项目管理员可修改 {key}") - if key not in ADMIN_FIELDS and key not in EDIT_FIELDS: + if key not in admin_fields | edit_fields: raise HTTPException(status_code=400, detail=f"不支持修改字段: {key}") - if "name" in patch: clean = (patch["name"] or "").strip() - if not clean: - raise HTTPException(status_code=400, detail="项目名不能为空") - if len(clean) > 120: - raise HTTPException(status_code=400, detail="项目名过长（≤120 字）") - # Duplicate-name check (excluding itself) - dup_q = self.db.query(Project.project_id).filter( - Project.kind == project.kind, - Project.name == clean, - Project.deleted_at.is_(None), - Project.project_id != project_id, + if not clean or len(clean) > 120: + raise HTTPException(status_code=400, detail="项目名不合法") + duplicate = ( + self.db.query(Project.project_id) + .filter( + Project.kind == "personal", + Project.owner_user_id == project.owner_user_id, + Project.name == clean, + Project.project_id != project_id, + Project.deleted_at.is_(None), + ) + .first() ) - if project.kind == "personal": - dup_q = dup_q.filter(Project.owner_user_id == project.owner_user_id) - else: - dup_q = dup_q.filter(Project.team_id == project.team_id) - if dup_q.first() is not None: + if duplicate: raise HTTPException(status_code=400, detail="同名项目已存在") project.name = clean - if "description" in patch: project.description = (patch["description"] or "").strip() or None if "instructions" in patch: - instructions = patch["instructions"] or "" - # Simple cap to keep the system prompt from getting too long - if len(instructions) > 8000: + value = patch["instructions"] or "" + if len(value) > 8000: raise HTTPException(status_code=400, detail="项目指令过长（≤8000 字符）") - project.instructions = instructions.strip() or None + project.instructions = value.strip() or None if "pinned" in patch: project.pinned = bool(patch["pinned"]) if "icon_color" in patch: - color = (patch["icon_color"] or "").strip() or None - if color and len(color) > 20: - raise HTTPException(status_code=400, detail="颜色字符串过长（≤20 字符）") - project.icon_color = color - # Memory switches are stored in extra_data (JSONB), updated only when explicitly - # passed; reads default to True when absent. Reassign the whole dict to trigger - # SQLAlchemy change detection (JSONType is not MutableDict). + project.icon_color = (patch["icon_color"] or "").strip() or None if "memory_enabled" in patch or "memory_write_enabled" in patch: - new_extra = dict(project.extra_data or {}) - if "memory_enabled" in patch: - new_extra["memory_enabled"] = bool(patch["memory_enabled"]) - if "memory_write_enabled" in patch: - new_extra["memory_write_enabled"] = bool(patch["memory_write_enabled"]) - project.extra_data = new_extra - + extra = dict(project.extra_data or {}) + for key in ("memory_enabled", "memory_write_enabled"): + if key in patch: + extra[key] = bool(patch[key]) + project.extra_data = extra project.updated_at = datetime.utcnow() self.db.commit() - self.db.refresh(project) - - self.audit_repo.create({ - "user_id": user_id, - "action": "project.updated", - "resource_type": "project", - "resource_id": project_id, - "details": {k: v for k, v in patch.items() if k != "instructions"}, # content excluded from audit - "status": "success", - }) return self.get(project_id, user_id) def soft_delete(self, project_id: str, user_id: str) -> bool: @@ -737,18 +376,10 @@ def soft_delete(self, project_id: str, user_id: str) -> bool: return False project.deleted_at = datetime.utcnow() self.db.commit() - self.audit_repo.create({ - "user_id": user_id, - "action": "project.deleted", - "resource_type": "project", - "resource_id": project_id, - "status": "success", - }) return True - # ── Favorite ────────────────────────────────────────────────────── def toggle_favorite(self, project_id: str, user_id: str, on: bool) -> bool: - existing = ( + row = ( self.db.query(ProjectFavorite) .filter( ProjectFavorite.project_id == project_id, @@ -756,23 +387,19 @@ def toggle_favorite(self, project_id: str, user_id: str, on: bool) -> bool: ) .first() ) - if on and existing is None: + if on and row is None: self.db.add(ProjectFavorite(project_id=project_id, user_id=user_id)) - self.db.commit() - elif (not on) and existing is not None: - self.db.delete(existing) - self.db.commit() + elif not on and row is not None: + self.db.delete(row) + self.db.commit() return on - # ── Activity ────────────────────────────────────────────────────── def touch_activity(self, project_id: str) -> None: project = self.get_raw(project_id) - if project is None: - return - project.last_activity_at = datetime.utcnow() - self.db.commit() + if project is not None: + project.last_activity_at = datetime.utcnow() + self.db.commit() - # ── Chats listing within a project ──────────────────────────────── def list_chats( self, project_id: str, @@ -781,166 +408,36 @@ def list_chats( page_size: int = 30, scope: str = "all", ) -> Tuple[List[Dict[str, Any]], int]: - """List chats within a project. - - - Personal project: only chats owned by the current user. - - Team project: my own chats, unioned with chats other members have shared by - setting ``share_scope`` to ``team_read`` / ``team_edit``; pin/favorite go - through the ``chat_session_user_states`` table (independent per user). - - ``scope`` filter: ``all`` / ``mine`` / ``shared`` (only meaningful for team projects). - """ - project = self.get_raw(project_id) - if project is None: - return [], 0 - - # Team projects always render with shared-list semantics - team_share = project.kind == "team" - - base = self.db.query(ChatSession).filter( + query = self.db.query(ChatSession).filter( ChatSession.project_id == project_id, + ChatSession.user_id == user_id, ChatSession.deleted_at.is_(None), ) - if team_share: - if scope == "mine": - base = base.filter(ChatSession.user_id == user_id) - elif scope == "shared": - base = base.filter( - ChatSession.user_id != user_id, - ChatSession.share_scope.in_(("team_read", "team_edit")), - ) - else: # 'all' - base = base.filter( - or_( - ChatSession.user_id == user_id, - ChatSession.share_scope.in_(("team_read", "team_edit")), - ) - ) - else: - base = base.filter(ChatSession.user_id == user_id) - - total = base.count() + total = query.count() rows = ( - base.order_by(desc(ChatSession.updated_at)) + query.order_by(desc(ChatSession.updated_at)) .offset((page - 1) * page_size) .limit(page_size) .all() ) - - # owner_id → username in one query - owner_ids = {s.user_id for s in rows} - owner_name_map: Dict[str, str] = {} - if owner_ids: - for uid, uname in ( - self.db.query(UserShadow.user_id, UserShadow.username) - .filter(UserShadow.user_id.in_(owner_ids)) - .all() - ): - owner_name_map[uid] = uname - - # The current user's per-user state for this batch of chat_ids (fetched only when team_share) - user_state_map: Dict[str, ChatSessionUserState] = {} - if team_share and rows: - chat_ids = [s.chat_id for s in rows] - for state in ( - self.db.query(ChatSessionUserState) - .filter( - ChatSessionUserState.user_id == user_id, - ChatSessionUserState.chat_id.in_(chat_ids), - ) - .all() - ): - user_state_map[state.chat_id] = state - - items: List[Dict[str, Any]] = [] - for s in rows: - is_owner = s.user_id == user_id - if team_share: - state = user_state_map.get(s.chat_id) - pinned = bool(state.pinned) if state is not None else False - favorite = bool(state.favorite) if state is not None else False - else: - pinned = bool(s.pinned) - favorite = bool(s.favorite) - items.append({ - "chat_id": s.chat_id, - "title": s.title, - "pinned": pinned, - "favorite": favorite, - "message_count": int(s.message_count or 0), - "last_message_at": s.last_message_at.isoformat() if s.last_message_at else None, - "updated_at": s.updated_at.isoformat() if s.updated_at else None, - "created_at": s.created_at.isoformat() if s.created_at else None, - "project_id": s.project_id, - "owner_user_id": s.user_id, - "owner_name": owner_name_map.get(s.user_id), - "share_scope": s.share_scope or "private", - "is_owner": is_owner, - }) - return items, total - - # ── Per-user chat state (team-share scenario) ───────────────────── - def upsert_chat_user_state( - self, - chat_id: str, - user_id: str, - *, - pinned: Optional[bool] = None, - favorite: Optional[bool] = None, - ) -> ChatSessionUserState: - """Upsert the current user's pin/favorite for a chat on ``chat_session_user_states``. - - - Only patch the fields explicitly passed in. - - If no row exists, insert a new one (defaults false) with the passed fields applied. - """ - state = ( - self.db.query(ChatSessionUserState) - .filter( - ChatSessionUserState.chat_id == chat_id, - ChatSessionUserState.user_id == user_id, - ) - .first() - ) - if state is None: - state = ChatSessionUserState( - chat_id=chat_id, - user_id=user_id, - pinned=bool(pinned) if pinned is not None else False, - favorite=bool(favorite) if favorite is not None else False, - ) - self.db.add(state) - else: - if pinned is not None: - state.pinned = bool(pinned) - if favorite is not None: - state.favorite = bool(favorite) - state.updated_at = datetime.utcnow() - self.db.commit() - self.db.refresh(state) - return state - - def get_chat_user_state( - self, chat_id: str, user_id: str - ) -> Optional[ChatSessionUserState]: - return ( - self.db.query(ChatSessionUserState) - .filter( - ChatSessionUserState.chat_id == chat_id, - ChatSessionUserState.user_id == user_id, - ) - .first() - ) - - # ── Helpers for cross-module use ────────────────────────────────── - def list_teams_user_can_create_in(self, user_id: str) -> List[Dict[str, Any]]: - """Frontend create-project modal: only show teams where the user is owner / admin.""" - rows = ( - self.db.query(Team, TeamMember.role) - .join(TeamMember, Team.team_id == TeamMember.team_id) - .filter( - TeamMember.user_id == user_id, - TeamMember.role.in_(("owner", "admin")), - ) - .order_by(Team.name) - .all() - ) - return [{"team_id": t.team_id, "name": t.name, "role": role} for t, role in rows] + owner = self.db.query(UserShadow.username).filter(UserShadow.user_id == user_id).first() + return [ + { + "chat_id": row.chat_id, + "title": row.title, + "pinned": bool(row.pinned), + "favorite": bool(row.favorite), + "message_count": int(row.message_count or 0), + "last_message_at": row.last_message_at.isoformat() if row.last_message_at else None, + "updated_at": row.updated_at.isoformat() if row.updated_at else None, + "created_at": row.created_at.isoformat() if row.created_at else None, + "project_id": row.project_id, + "owner_user_id": row.user_id, + "owner_name": owner[0] if owner else None, + "is_owner": True, + } + for row in rows + ], total + + +__all__ = ["ProjectService"] diff --git a/src/backend/core/services/role_service.py b/src/backend/core/services/role_service.py deleted file mode 100644 index 80fa29df..00000000 --- a/src/backend/core/services/role_service.py +++ /dev/null @@ -1,232 +0,0 @@ -"""Role permission business logic. - -A role = a reusable named capability grant bundle, assigned to a team (= department -default role, inherited by members in real time) or to an individual. -Capability bit normalization reuses ``normalize_role_permissions`` from -[[role_permissions]] (only granted bits are stored). -""" - -from __future__ import annotations - -import uuid -from dataclasses import dataclass -from typing import Any, Dict, List, Optional - -from sqlalchemy.orm import Session - -from core.auth.role_permissions import normalize_role_permissions -from core.db.models import Role -from core.db.repository import AuditLogRepository, RoleRepository - - -# Default built-in role seeds: auto-created on startup in a new environment (empty -# roles table), ready to use across platform deployments. -# Idempotent, deduplicated by "role name" — an existing role with the same name is -# skipped (does not overwrite an admin's changes). -DEFAULT_ROLES: List[dict] = [ - { - "role_id": "role_seed_dept_member", - "name": "部门成员", - "description": "部门普通成员的默认能力", - "is_team_default": True, # auto-attach this role when creating/syncing a team - "permissions": { - "can_add_skill": True, - "can_add_mcp": True, - "can_add_agent": True, - "can_use_api_key": True, - "can_import_plugin": True, - "can_create_private_kb": True, - "can_create_public_kb": True, - "can_create_channel_bot": True, - "allowed_apps": ["plan_mode", "automation", "batch_runner"], - }, - }, - { - "role_id": "role_seed_it_admin", - "name": "IT管理员", - "description": "部门 IT 管理员（含后台访问权限）", - "permissions": { - "lab_enabled": True, - "can_add_skill": True, - "can_add_mcp": True, - "can_add_agent": True, - "can_use_api_key": True, - "can_import_plugin": True, - "can_create_private_kb": True, - "can_create_public_kb": True, - "can_create_channel_bot": True, - "can_system_config": True, - "can_content_manage": True, - "allowed_apps": ["plan_mode", "automation", "batch_runner"], - }, - }, -] - - -def seed_default_roles(db: Session) -> List[str]: - """Seed default roles in a new environment (idempotent, deduplicated by name). - Returns the list of role names created this time. - - An existing role with the same name → skipped (does not overwrite the admin's - existing config); in CE, where there is no roles table, the caller's try/except - fallback degrades this to a no-op. - """ - repo = RoleRepository(db) - added: List[str] = [] - for spec in DEFAULT_ROLES: - if repo.get_by_name(spec["name"]) or repo.get(spec["role_id"]): - continue - repo.create( - { - "role_id": spec["role_id"], - "name": spec["name"], - "description": spec.get("description"), - "permissions": normalize_role_permissions(spec["permissions"]), - "is_system": False, - "is_team_default": bool(spec.get("is_team_default")), - } - ) - added.append(spec["name"]) - return added - - -def apply_team_default_roles(db: Session, team_id: str) -> int: - """Append all "new-team default" roles to a (newly created/synced) team, inherited - by members in real time. Returns the number of rows added. - - Idempotent: already-attached ones are skipped. In CE, where there is no roles table, - the caller's try/except degrades this to a no-op. - """ - repo = RoleRepository(db) - return repo.add_principal_roles("team", team_id, repo.list_team_default_role_ids()) - - -def serialize_role(role: Role, *, assignment_count: Optional[int] = None) -> dict: - """Role → brief structure for the frontend.""" - data: Dict[str, Any] = { - "role_id": role.role_id, - "name": role.name, - "description": role.description, - "permissions": dict(role.permissions or {}), - "is_system": bool(role.is_system), - "is_team_default": bool(role.is_team_default), - "created_at": role.created_at.isoformat() if role.created_at else None, - "updated_at": role.updated_at.isoformat() if role.updated_at else None, - } - if assignment_count is not None: - data["assignment_count"] = assignment_count - return data - - -@dataclass -class RoleResult: - ok: bool - message: str - role_id: Optional[str] = None - - -class RoleService: - def __init__(self, db: Session): - self.db = db - self.repo = RoleRepository(db) - self.audit_repo = AuditLogRepository(db) - - # ── List ───────────────────────────────────────────────── - def list_roles(self) -> List[dict]: - roles = self.repo.list_all() - counts = self.repo.assignment_counts_bulk([r.role_id for r in roles]) - return [serialize_role(r, assignment_count=counts.get(r.role_id, 0)) for r in roles] - - # ── CRUD ───────────────────────────────────────────────── - def create_role( - self, - name: str, - description: Optional[str] = None, - permissions: Optional[dict] = None, - is_team_default: bool = False, - actor: Optional[str] = None, - ) -> RoleResult: - name = (name or "").strip() - if not name: - return RoleResult(False, "角色名称不能为空") - if len(name) > 64: - return RoleResult(False, "角色名称过长（≤64）") - if self.repo.get_by_name(name): - return RoleResult(False, "角色名称已存在") - - role_id = f"role_{uuid.uuid4().hex[:16]}" - self.repo.create( - { - "role_id": role_id, - "name": name, - "description": description, - "permissions": normalize_role_permissions(permissions), - "is_system": False, - "is_team_default": bool(is_team_default), - } - ) - return RoleResult(True, "角色已创建", role_id) - - def update_role( - self, - role_id: str, - name: Optional[str] = None, - description: Optional[str] = None, - permissions: Optional[dict] = None, - is_team_default: Optional[bool] = None, - actor: Optional[str] = None, - ) -> RoleResult: - role = self.repo.get(role_id) - if not role: - return RoleResult(False, "角色不存在") - data: Dict[str, Any] = {} - if is_team_default is not None: - data["is_team_default"] = bool(is_team_default) - if name is not None: - name = name.strip() - if not name: - return RoleResult(False, "角色名称不能为空") - if len(name) > 64: - return RoleResult(False, "角色名称过长（≤64）") - existing = self.repo.get_by_name(name) - if existing and existing.role_id != role_id: - return RoleResult(False, "角色名称已存在") - data["name"] = name - if description is not None: - data["description"] = description - if permissions is not None: - data["permissions"] = normalize_role_permissions(permissions) - if data: - self.repo.update(role_id, data) - return RoleResult(True, "角色已更新", role_id) - - def delete_role(self, role_id: str, actor: Optional[str] = None) -> RoleResult: - role = self.repo.get(role_id) - if not role: - return RoleResult(False, "角色不存在") - if role.is_system: - return RoleResult(False, "内置角色不可删除") - self.repo.delete(role_id) # cascade-clear assignments - return RoleResult(True, "角色已删除", role_id) - - # ── Assignments ────────────────────────────────────────── - def list_assignments(self, role_id: str) -> Optional[List[dict]]: - """Which principals (users/teams) a given role is assigned to. Returns None if - the role does not exist.""" - if not self.repo.get(role_id): - return None - return [ - {"principal_type": a.principal_type, "principal_id": a.principal_id} - for a in self.repo.list_assignments(role_id) - ] - - def set_principal_roles( - self, principal_type: str, principal_id: str, role_ids: List[str] - ) -> int: - """Fully replace a principal's (user/team) role assignments; returns the number - of rows written.""" - return self.repo.set_principal_roles(principal_type, principal_id, role_ids) - - def get_principal_roles(self, principal_type: str, principal_id: str) -> List[dict]: - """Brief list of roles directly assigned to a principal (excluding inheritance).""" - return [serialize_role(r) for r in self.repo.list_roles_for_principal(principal_type, principal_id)] diff --git a/src/backend/core/services/service_probes.py b/src/backend/core/services/service_probes.py index 266ed1df..191a1f66 100644 --- a/src/backend/core/services/service_probes.py +++ b/src/backend/core/services/service_probes.py @@ -142,30 +142,6 @@ async def test_baidu(api_key: str) -> dict: return {"success": False, "latency_ms": latency, "error": str(exc)} -async def test_dify(base_url: str, api_key: str) -> dict: - """Test Dify KB connectivity by listing datasets.""" - url = f"{base_url.rstrip('/')}/datasets" - start = time.monotonic() - try: - async with httpx.AsyncClient(timeout=10) as client: - resp = await client.get( - url, - headers={"Authorization": f"Bearer {api_key}"}, - params={"limit": 1}, - ) - latency = int((time.monotonic() - start) * 1000) - if resp.status_code == 200: - return {"success": True, "latency_ms": latency, "error": None} - return { - "success": False, - "latency_ms": latency, - "error": f"HTTP {resp.status_code}: {resp.text[:200]}", - } - except Exception as exc: - latency = int((time.monotonic() - start) * 1000) - return {"success": False, "latency_ms": latency, "error": str(exc)} - - async def test_service_group(group_key: str) -> dict: """Run one connectivity test for a group (reading the current SystemConfigService config). @@ -177,11 +153,13 @@ async def test_service_group(group_key: str) -> dict: svc = SystemConfigService.get_instance() if group_key == "knowledge_base": + from core.services.edition_service_probe import test_external_knowledge + url = svc.get("knowledge_base.url") api_key = svc.get("knowledge_base.api_key") if not url: return {"success": False, "error": "URL 未配置", "latency_ms": 0} - return await test_dify(url, api_key or "") + return await test_external_knowledge(url, api_key or "") if group_key == "industry": url = svc.get("industry.url") diff --git a/src/backend/core/services/site_access_policy.py b/src/backend/core/services/site_access_policy.py new file mode 100644 index 00000000..fcdbc080 --- /dev/null +++ b/src/backend/core/services/site_access_policy.py @@ -0,0 +1,52 @@ +"""Single-owner site access policy for Community Edition.""" + +from typing import Optional + +from core.infra.exceptions import BadRequestError +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session + + +class SiteUpdateScopeFields(BaseModel): + visibility: Optional[str] = Field( + None, + description="可见性：public / private", + pattern="^(public|private)$", + ) + + +class SitePublishScopeFields(BaseModel): + visibility: str = Field( + "public", + description="可见性：public / private", + pattern="^(public|private)$", + ) + + +def resolve_site_scope( + _db: Session, + _user_id: str, + visibility: str, + _scope_id: Optional[str] = None, +) -> None: + if visibility not in ("public", "private"): + raise BadRequestError("visibility 仅支持 public / private") + return None + + +def site_scope_write_fields(_scope_id: Optional[str]) -> dict: + return {} + + +def site_scope_ref(fields) -> None: + return None + + +def serialize_site_scope(site) -> dict: + return {"visibility": site.visibility} + + +def can_view_site(_db: Session, site, viewer_user_id: Optional[str]) -> bool: + if site.visibility == "public": + return True + return bool(viewer_user_id and viewer_user_id == site.user_id) diff --git a/src/backend/core/services/site_service.py b/src/backend/core/services/site_service.py index c140e513..dc549ab1 100644 --- a/src/backend/core/services/site_service.py +++ b/src/backend/core/services/site_service.py @@ -20,21 +20,27 @@ from datetime import datetime from typing import Any, Dict, List, Optional, Tuple -from sqlalchemy.orm import Session - from core.db.models import Site from core.db.repository import SiteRepository from core.infra.exceptions import BadRequestError, ResourceNotFoundError +from core.services.site_access_policy import ( + can_view_site, + resolve_site_scope, + site_scope_write_fields, +) from core.storage import get_storage +from sqlalchemy.orm import Session logger = logging.getLogger(__name__) # ── Quotas ─────────────────────────────────────────────────────── MAX_SITE_FILES = 300 -MAX_SITE_TOTAL_BYTES = 30 * 1024 * 1024 # 30MB / site -MAX_SITE_FILE_BYTES = 10 * 1024 * 1024 # 10MB / file +MAX_SITE_TOTAL_BYTES = 30 * 1024 * 1024 # 30MB / site +MAX_SITE_FILE_BYTES = 10 * 1024 * 1024 # 10MB / file MAX_SITES_PER_USER = 50 -KEEP_VERSIONS = 3 # number of historical versions kept after publishing a new one in local mode (incl. current) +KEEP_VERSIONS = ( + 3 # number of historical versions kept after publishing a new one in local mode (incl. current) +) # Site-level KV / form-collection quotas (a minimal subset benchmarked against D1/R2) MAX_KV_KEYS_PER_SITE = 200 @@ -52,9 +58,26 @@ # under /site//, so in theory there's no conflict; this list guards # against a future move of sites to the root path + avoids misleading addresses) RESERVED_SLUGS = { - "api", "assets", "admin", "config", "docs", "files", "gateway", "health", - "home", "login", "logout", "mock-sso", "openapi", "redoc", "register", - "share", "site", "sites", "static", "www", + "api", + "assets", + "admin", + "config", + "docs", + "files", + "gateway", + "health", + "home", + "login", + "logout", + "mock-sso", + "openapi", + "redoc", + "register", + "share", + "site", + "sites", + "static", + "www", } # Fill in gaps in the default mimetypes table (/etc/mime.types inside the container may be incomplete) @@ -136,7 +159,7 @@ def publish( chat_id: Optional[str] = None, visibility: str = "public", description: str = "", - team_id: Optional[str] = None, + scope_id: Optional[str] = None, project_id: Optional[str] = None, build_info: Optional[dict] = None, ) -> Site: @@ -160,9 +183,7 @@ def publish( title = (title or "").strip() if not title: raise BadRequestError("站点标题不能为空") - if visibility not in ("public", "private", "team"): - raise BadRequestError("visibility 仅支持 public / private / team") - team_id = self._resolve_team(user_id, visibility, team_id) + resolved_scope_id = resolve_site_scope(self.db, user_id, visibility, scope_id) cleaned = self._validate_files(files) entry_file = self._pick_entry_file(cleaned) @@ -175,9 +196,7 @@ def publish( from core.db.models import ChatSession exists = ( - self.db.query(ChatSession.chat_id) - .filter(ChatSession.chat_id == chat_id) - .first() + self.db.query(ChatSession.chat_id).filter(ChatSession.chat_id == chat_id).first() ) if not exists: chat_id = None @@ -189,8 +208,13 @@ def publish( if site.user_id != user_id: raise BadRequestError("无权更新该站点（不属于当前用户）") return self._publish_new_version( - site, cleaned, title=title, entry_file=entry_file, - visibility=visibility, description=description, team_id=team_id, + site, + cleaned, + title=title, + entry_file=entry_file, + visibility=visibility, + description=description, + scope_id=resolved_scope_id, build_info=build_info, ) @@ -204,25 +228,27 @@ def publish( new_id = f"site_{uuid.uuid4().hex[:16]}" version = 1 total_size = self._write_version_files(new_id, version, cleaned) - return self.repo.create({ - "site_id": new_id, - "slug": final_slug, - "user_id": user_id, - "chat_id": chat_id, - "team_id": team_id, - "project_id": (project_id or None), - "title": title, - "description": description or None, - "visibility": visibility, - "entry_file": entry_file, - "current_version": version, - "file_count": len(cleaned), - "total_size_bytes": total_size, - "extra_data": { - "versions": [self._version_meta(version, cleaned, total_size)], - **({"build": build_info} if build_info else {}), - }, - }) + return self.repo.create( + { + "site_id": new_id, + "slug": final_slug, + "user_id": user_id, + "chat_id": chat_id, + **site_scope_write_fields(resolved_scope_id), + "project_id": (project_id or None), + "title": title, + "description": description or None, + "visibility": visibility, + "entry_file": entry_file, + "current_version": version, + "file_count": len(cleaned), + "total_size_bytes": total_size, + "extra_data": { + "versions": [self._version_meta(version, cleaned, total_size)], + **({"build": build_info} if build_info else {}), + }, + } + ) def _publish_new_version( self, @@ -233,7 +259,7 @@ def _publish_new_version( entry_file: str, visibility: str, description: str, - team_id: Optional[str] = None, + scope_id: Optional[str] = None, build_info: Optional[dict] = None, ) -> Site: meta = dict(site.extra_data or {}) @@ -242,8 +268,7 @@ def _publish_new_version( versions = list(meta.get("versions") or []) # New version number = historical max + 1 (after a rollback, current_version may be lower than the historical max) max_ver = max( - [int(site.current_version or 1)] - + [int(v.get("version") or 0) for v in versions] + [int(site.current_version or 1)] + [int(v.get("version") or 0) for v in versions] ) version = max_ver + 1 total_size = self._write_version_files(site.site_id, version, cleaned) @@ -251,38 +276,23 @@ def _publish_new_version( versions.append(self._version_meta(version, cleaned, total_size)) meta["versions"] = versions[-20:] # keep only the 20 most recent records - updated = self.repo.update(site.site_id, { - "title": title or site.title, - "description": description or site.description, - "visibility": visibility or site.visibility, - "team_id": team_id if team_id is not None else site.team_id, - "entry_file": entry_file, - "current_version": version, - "file_count": len(cleaned), - "total_size_bytes": total_size, - "extra_data": meta, - }) + updated = self.repo.update( + site.site_id, + { + "title": title or site.title, + "description": description or site.description, + "visibility": visibility or site.visibility, + **site_scope_write_fields(scope_id), + "entry_file": entry_file, + "current_version": version, + "file_count": len(cleaned), + "total_size_bytes": total_size, + "extra_data": meta, + }, + ) self._prune_old_versions(site.site_id, version) return updated - def _resolve_team( - self, user_id: str, visibility: str, team_id: Optional[str] - ) -> Optional[str]: - """When visibility=team, resolve and validate the authorized team; other visibility tiers return None.""" - if visibility != "team": - return None - from core.db.repository import TeamRepository - - teams = TeamRepository(self.db) - if team_id: - if not teams.get_member(team_id, user_id): - raise BadRequestError("你不是该团队成员，无法发布团队站点") - return team_id - mine = teams.list_for_user(user_id) - if not mine: - raise BadRequestError("你还没有加入任何团队，无法使用团队可见档") - return mine[0][0].team_id - @staticmethod def _version_meta( version: int, cleaned: List[Tuple[str, bytes]], total_size: int @@ -294,15 +304,11 @@ def _version_meta( "created_at": datetime.utcnow().isoformat(), } - def _validate_files( - self, files: List[Tuple[str, bytes]] - ) -> List[Tuple[str, bytes]]: + def _validate_files(self, files: List[Tuple[str, bytes]]) -> List[Tuple[str, bytes]]: if not files: raise BadRequestError("站点内容为空（目录里没有可发布的文件）") if len(files) > MAX_SITE_FILES: - raise BadRequestError( - f"站点文件数超限：{len(files)} > {MAX_SITE_FILES}" - ) + raise BadRequestError(f"站点文件数超限：{len(files)} > {MAX_SITE_FILES}") cleaned: List[Tuple[str, bytes]] = [] seen: set[str] = set() total = 0 @@ -334,22 +340,16 @@ def _pick_entry_file(cleaned: List[Tuple[str, bytes]]) -> str: paths = {p for p, _ in cleaned} if "index.html" in paths: return "index.html" - root_htmls = sorted( - p for p in paths if "/" not in p and p.endswith((".html", ".htm")) - ) + root_htmls = sorted(p for p in paths if "/" not in p and p.endswith((".html", ".htm"))) if len(root_htmls) == 1: return root_htmls[0] - raise BadRequestError( - "站点根目录必须有 index.html（或唯一的一个 .html 文件作为入口）" - ) + raise BadRequestError("站点根目录必须有 index.html（或唯一的一个 .html 文件作为入口）") def _resolve_slug(self, slug: str) -> str: slug = (slug or "").strip().lower() if slug: if not SLUG_RE.match(slug): - raise BadRequestError( - "slug 仅支持 3-50 位小写字母/数字/连字符，且首尾为字母数字" - ) + raise BadRequestError("slug 仅支持 3-50 位小写字母/数字/连字符，且首尾为字母数字") if slug in RESERVED_SLUGS: raise BadRequestError(f"slug '{slug}' 是保留字，请换一个") if self.repo.get_by_slug(slug): @@ -400,9 +400,7 @@ def _prune_old_versions(self, site_id: str, current_version: int) -> None: # ── Hosted file retrieval ──────────────────────────────────── - def resolve_site_file( - self, site: Site, path: str - ) -> Optional[Tuple[bytes, str]]: + def resolve_site_file(self, site: Site, path: str) -> Optional[Tuple[bytes, str]]: """Fetch a site file by the requested path; returns (bytes, content_type), or None if not found. Fallback order: exact path → directory index.html (``foo/`` or @@ -459,7 +457,7 @@ def update_site( visibility: Optional[str] = None, slug: Optional[str] = None, description: Optional[str] = None, - team_id: Optional[str] = None, + scope_id: Optional[str] = None, ) -> Site: site = self.get_owned(site_id, user_id) data: Dict[str, Any] = {} @@ -469,12 +467,9 @@ def update_site( raise BadRequestError("站点标题不能为空") data["title"] = title if visibility is not None: - if visibility not in ("public", "private", "team"): - raise BadRequestError("visibility 仅支持 public / private / team") + resolved_scope_id = resolve_site_scope(self.db, user_id, visibility, scope_id) data["visibility"] = visibility - data["team_id"] = self._resolve_team( - user_id, visibility, team_id or site.team_id - ) + data.update(site_scope_write_fields(resolved_scope_id)) if description is not None: data["description"] = description or None if slug is not None and slug != site.slug: @@ -490,8 +485,7 @@ def rollback(self, site_id: str, user_id: str, version: int) -> Site: if version == site.current_version: raise BadRequestError(f"v{version} 已是当前线上版本") versions = { - int(v.get("version") or 0) - for v in (site.extra_data or {}).get("versions") or [] + int(v.get("version") or 0) for v in (site.extra_data or {}).get("versions") or [] } if version not in versions: raise BadRequestError(f"版本 v{version} 不存在") @@ -508,28 +502,19 @@ def rollback(self, site_id: str, user_id: str, version: int) -> Site: "to": version, "at": datetime.utcnow().isoformat(), } - return self.repo.update(site.site_id, { - "current_version": version, - "extra_data": meta, - }) + return self.repo.update( + site.site_id, + { + "current_version": version, + "extra_data": meta, + }, + ) # ── View authorization (shared by the hosting route & site API) ─ def authorize_view(self, site: Site, viewer_user_id: Optional[str]) -> bool: - """Decide whether the viewer may access the site based on the visibility tier.""" - if site.visibility == "public": - return True - if not viewer_user_id: - return False - if viewer_user_id == site.user_id: - return True - if site.visibility == "team" and site.team_id: - from core.db.repository import TeamRepository - - return TeamRepository(self.db).get_member( - site.team_id, viewer_user_id - ) is not None - return False + """Decide whether the viewer may access the site under this edition's policy.""" + return can_view_site(self.db, site, viewer_user_id) # ── Site-level KV (a minimal subset benchmarked against D1) ── @@ -562,7 +547,10 @@ def kv_delete(self, site: Site, key: str) -> bool: # ── Form collection (export lands as an artifact) ──────────── def submit_form( - self, site: Site, form_key: str, payload: Dict[str, Any], + self, + site: Site, + form_key: str, + payload: Dict[str, Any], client_ip: Optional[str] = None, ) -> str: if not FORM_KEY_RE.match(form_key or ""): @@ -578,12 +566,12 @@ def submit_form( row = self.repo.submission_add(site.site_id, form_key, payload, client_ip) return row.id - def export_submissions_to_artifact( - self, site_id: str, user_id: str - ) -> Dict[str, Any]: + def export_submissions_to_artifact(self, site_id: str, user_id: str) -> Dict[str, Any]: """Export all form submissions as a CSV artifact (persisted; visible and downloadable in "My Space").""" site = self.get_owned(site_id, user_id) - rows, total = self.repo.submission_list(site.site_id, page=1, page_size=MAX_SUBMISSIONS_PER_SITE) + rows, total = self.repo.submission_list( + site.site_id, page=1, page_size=MAX_SUBMISSIONS_PER_SITE + ) if not rows: raise BadRequestError("该站点还没有表单数据") @@ -601,17 +589,23 @@ def export_submissions_to_artifact( writer.writerow(["提交时间", "表单", *field_names]) for r in reversed(rows): # export in chronological order payload = r.payload or {} - writer.writerow([ - r.created_at.isoformat() if r.created_at else "", - r.form_key, - *[ - _json.dumps(payload.get(k), ensure_ascii=False) - if isinstance(payload.get(k), (dict, list)) - else ("" if payload.get(k) is None else str(payload.get(k))) - for k in field_names - ], - ]) - content = buf.getvalue().encode("utf-8-sig") # BOM: opens directly in Excel without mojibake + writer.writerow( + [ + r.created_at.isoformat() if r.created_at else "", + r.form_key, + *[ + ( + _json.dumps(payload.get(k), ensure_ascii=False) + if isinstance(payload.get(k), (dict, list)) + else ("" if payload.get(k) is None else str(payload.get(k))) + ) + for k in field_names + ], + ] + ) + content = buf.getvalue().encode( + "utf-8-sig" + ) # BOM: opens directly in Excel without mojibake from core.artifacts.store import save_artifact_bytes from core.services.artifact_service import ArtifactService @@ -650,6 +644,4 @@ def delete_site(self, site_id: str, user_id: str) -> None: if (os.getenv("STORAGE_TYPE", "local").lower()) == "local": base = os.getenv("STORAGE_PATH", "./storage") - shutil.rmtree( - os.path.join(base, "sites", site.site_id), ignore_errors=True - ) + shutil.rmtree(os.path.join(base, "sites", site.site_id), ignore_errors=True) diff --git a/src/backend/core/services/system_config.py b/src/backend/core/services/system_config.py index 8b07b8c3..aba575de 100644 --- a/src/backend/core/services/system_config.py +++ b/src/backend/core/services/system_config.py @@ -17,6 +17,8 @@ from core.db.engine import SessionLocal from core.db.models import SystemConfig +from core.services.edition_system_config import CONFIG_KEY_TO_ENV as EDITION_CONFIG_KEY_TO_ENV +from core.services.edition_system_config import SEED_CONFIGS as EDITION_SEED_CONFIGS logger = logging.getLogger(__name__) @@ -52,32 +54,6 @@ "query_database", False, ), - # knowledge_base - ("knowledge_base.provider", "dify", "知识库后端", "知识库服务提供方", "knowledge_base", False), - ( - "knowledge_base.url", - None, - "知识库 API URL", - "Dify 或其他知识库服务地址", - "knowledge_base", - False, - ), - ( - "knowledge_base.api_key", - None, - "知识库 API Key", - "知识库服务鉴权密钥", - "knowledge_base", - True, - ), - ( - "knowledge_base.allowed_dataset_ids", - None, - "允许的数据集 ID", - "逗号分隔的数据集 ID 白名单，为空则全部允许", - "knowledge_base", - False, - ), ( "knowledge_base.detail_max_chars", "50000", @@ -240,7 +216,7 @@ "auth", False, ), -] +] + EDITION_SEED_CONFIGS # config_key → env var name mapping _CONFIG_KEY_TO_ENV: dict[str, str] = { @@ -248,10 +224,6 @@ "query_database.timeout": "QUERY_DATABASE_TIMEOUT_SECONDS", "query_database.retry_times": "QUERY_DATABASE_RETRY_TIMES", "query_database.max_output_tokens": "QUERY_DATABASE_MAX_OUTPUT_TOKENS", - "knowledge_base.provider": "KNOWLEDGE_BASE", - "knowledge_base.url": "DIFY_URL", - "knowledge_base.api_key": "DIFY_API_KEY", - "knowledge_base.allowed_dataset_ids": "DIFY_ALLOWED_DATASET_IDS", "knowledge_base.detail_max_chars": "KB_DETAIL_CONTENT_MAX_CHARS", "industry.url": "INDUSTRY_URL", "industry.auth_token": "INDUSTRY_AUTH_TOKEN", @@ -267,6 +239,7 @@ "internet_search.baidu_api_key": "BAIDU_API_KEY", # Note: sandbox.code_capability_enable deliberately does **not** map to an env var —— the admin toggle is the sole # authority (plan B), explicitly seeded to "true", and env CODE_CAPABILITY_ENABLED is retired. + **EDITION_CONFIG_KEY_TO_ENV, } # Reverse mapping for env-fallback lookups diff --git a/src/backend/core/services/user_agent_base.py b/src/backend/core/services/user_agent_base.py new file mode 100644 index 00000000..b6f2e741 --- /dev/null +++ b/src/backend/core/services/user_agent_base.py @@ -0,0 +1,796 @@ +"""Edition-neutral business logic for personal and administrator sub-agents.""" + +from __future__ import annotations + +import logging +import re +import uuid +from datetime import datetime +from decimal import Decimal +from typing import Any, Dict, List, Optional + +from core.db.models import UserAgent +from core.db.repository import AuditLogRepository, UserAgentRepository +from core.ontology.build_validator import ensure_ontology_build_valid +from sqlalchemy.orm import Session + +logger = logging.getLogger(__name__) + +MAX_USER_AGENTS = 20 +DEFAULT_AGENT_VERSION = "V1.0" +MAX_CHANGE_HISTORY = 30 +NON_VERSIONED_FIELDS = {"is_enabled"} +VERSIONED_FIELDS = { + "name": "名称", + "description": "简介", + "system_prompt": "角色设定", + "welcome_message": "开场白", + "suggested_questions": "推荐问题", + "mcp_server_ids": "绑定工具", + "skill_ids": "绑定技能", + "plugin_ids": "绑定插件", + "kb_ids": "绑定知识库", + "model_provider_id": "模型", + "temperature": "温度", + "max_tokens": "最大输出长度", + "max_iters": "最大推理轮次", + "timeout": "超时时间", + "is_enabled": "启用状态", +} + + +class UserAgentBaseService: + """Service for user agent CRUD and permission checks.""" + + def __init__(self, db: Session): + self.db = db + self.repo = UserAgentRepository(db) + + # ── Queries ────────────────────────────────────────────────────── + + def list_for_user(self, user_id: str) -> List[Dict[str, Any]]: + agents = self.repo.list_for_user(user_id) + return [self._serialize(a) for a in agents] + + def list_admin(self) -> List[Dict[str, Any]]: + agents = self.repo.list_admin() + return [self._serialize(a) for a in agents] + + def get_by_id(self, agent_id: str, user_id: Optional[str] = None) -> Dict[str, Any]: + agent = self.repo.get_by_id(agent_id) + if not agent: + raise LookupError(f"Agent {agent_id} not found") + if user_id and not self._is_accessible(agent, user_id): + raise PermissionError("No access to this agent") + return self._serialize(agent) + + def get_raw_by_id(self, agent_id: str, user_id: Optional[str] = None) -> UserAgent: + """Return the ORM object (for direct use by workflow/factory).""" + agent = self.repo.get_by_id(agent_id) + if not agent: + raise LookupError(f"Agent {agent_id} not found") + if user_id and not self._is_accessible(agent, user_id): + raise PermissionError("No access to this agent") + return agent + + # ── Mutations ──────────────────────────────────────────────────── + + def create( + self, + user_id: Optional[str], + operator_name: Optional[str], + owner_type: str, + data: Dict[str, Any], + scope_id: Optional[str] = None, + ) -> Dict[str, Any]: + data = dict(data) + incoming_extra = dict(data.get("extra_config") or {}) + ontology_tags = list( + data.pop("ontology_tags", incoming_extra.get("ontology_tags") or []) or [] + ) + incoming_extra["ontology_tags"] = ontology_tags + data["extra_config"] = incoming_extra + ensure_ontology_build_valid( + self.db, + asset_type="subagent", + name=str(data.get("name") or ""), + description=str(data.get("description") or ""), + instructions=str(data.get("system_prompt") or ""), + mcp_server_ids=list(data.get("mcp_server_ids") or []), + skill_ids=list(data.get("skill_ids") or []), + plugin_ids=list(data.get("plugin_ids") or []), + ontology_tags=ontology_tags, + ) + self._validate_create_scope(user_id, owner_type, scope_id) + + agent_id = f"ua_{uuid.uuid4().hex[:16]}" + created_at = self._now_iso() + creation_history = [ + { + "version": DEFAULT_AGENT_VERSION, + "timestamp": created_at, + "content": "创建了子智能体", + "operator_name": operator_name or user_id or "未知用户", + "details": [], + } + ] + extra_config = self._merge_extra_config( + current_extra=None, + incoming_extra=incoming_extra, + version=DEFAULT_AGENT_VERSION, + change_history=creation_history, + ) + + record = { + "agent_id": agent_id, + "owner_type": owner_type, + **self._owner_fields(user_id, owner_type, scope_id), + "created_by": user_id, + **data, + "extra_config": extra_config, + } + agent = self.repo.create(record) + self._audit( + user_id, + "agent.create", + agent_id, + {"owner_type": owner_type, "name": data.get("name")}, + ) + return self._serialize(agent) + + def update( + self, + agent_id: str, + user_id: Optional[str], + operator_name: Optional[str], + owner_type: str, + data: Dict[str, Any], + ) -> Dict[str, Any]: + agent = self.repo.get_by_id(agent_id) + if not agent: + raise LookupError(f"Agent {agent_id} not found") + self._check_ownership(agent, user_id, owner_type) + + data = dict(data) + current_extra = dict(agent.extra_config or {}) + incoming_extra = dict(data.get("extra_config") or {}) + ontology_tags = list( + data.pop( + "ontology_tags", + incoming_extra.get("ontology_tags", current_extra.get("ontology_tags") or []), + ) + or [] + ) + incoming_extra["ontology_tags"] = ontology_tags + if "extra_config" in data or ontology_tags != list( + current_extra.get("ontology_tags") or [] + ): + data["extra_config"] = incoming_extra + ensure_ontology_build_valid( + self.db, + asset_type="subagent", + name=str(data.get("name", agent.name) or ""), + description=str(data.get("description", agent.description) or ""), + instructions=str(data.get("system_prompt", agent.system_prompt) or ""), + mcp_server_ids=list(data.get("mcp_server_ids", agent.mcp_server_ids) or []), + skill_ids=list(data.get("skill_ids", agent.skill_ids) or []), + plugin_ids=list(data.get("plugin_ids", agent.plugin_ids) or []), + ontology_tags=ontology_tags, + ) + + changed_fields = self._collect_changed_fields(agent, data) + versioned_fields = [field for field in changed_fields if field not in NON_VERSIONED_FIELDS] + changed_labels = [VERSIONED_FIELDS[field] for field in changed_fields] + next_version = self._read_version(current_extra) + change_history = self._read_change_history(current_extra) + + if changed_labels: + change_summary = self._build_change_summary(changed_fields, data) + change_details = self._build_change_details(agent, changed_fields, data) + entry_version = next_version + if versioned_fields: + next_version = self._increment_version(next_version) + entry_version = next_version + change_history.append( + { + "version": entry_version, + "timestamp": self._now_iso(), + "content": change_summary, + "operator_name": operator_name or user_id or "未知用户", + "details": change_details, + } + ) + change_history = change_history[-MAX_CHANGE_HISTORY:] + + payload = dict(data) + payload["extra_config"] = self._merge_extra_config( + current_extra=current_extra, + incoming_extra=incoming_extra, + version=next_version, + change_history=change_history, + ) + + agent = self.repo.update(agent_id, payload) + audit_details = {"fields": list(data.keys())} + if changed_labels: + audit_details["change_summary"] = change_summary + audit_details["version"] = next_version + self._audit(user_id, "agent.update", agent_id, audit_details) + return self._serialize(agent) + + def delete( + self, + agent_id: str, + user_id: Optional[str], + owner_type: str, + ) -> bool: + agent = self.repo.get_by_id(agent_id) + if not agent: + raise LookupError(f"Agent {agent_id} not found") + self._check_ownership(agent, user_id, owner_type) + + ok = self.repo.delete(agent_id) + self._audit(user_id, "agent.delete", agent_id) + return ok + + def toggle_enabled(self, agent_id: str) -> Dict[str, Any]: + agent = self.repo.get_by_id(agent_id) + if not agent: + raise LookupError(f"Agent {agent_id} not found") + new_val = not agent.is_enabled + agent = self.repo.update(agent_id, {"is_enabled": new_val}) + return self._serialize(agent) + + # ── Available resources ────────────────────────────────────────── + + def list_available_resources(self, owner_user_id: Optional[str] = None) -> Dict[str, Any]: + """Return MCP servers, skills, plugins, and KB spaces bindable to agents. + + Plugin-sourced skills/MCP are removed from the skills/mcp_servers lists + and instead bound as a whole via the ``plugins`` list (plugin = + installable/removable unit, expanded at runtime into its skills+tools). + owner_user_id is used to include that user's private plugins and MCPs. + + The MCP list intentionally contains capabilities the user has personally + switched off. A sub-agent binding is an explicit, narrower capability + grant and therefore may opt into one of those tools without turning it + on for the user's main agent. Deployment-global MCPs disabled by an + administrator remain unavailable. + """ + from core.db.models import AdminMcpServer, InstalledPlugin, KBSpace + from sqlalchemy import or_ + + # ── Plugin list + their component id sets (used to strip plugin capabilities from the loose skills/tools) ── + # Query the ORM directly for component_ids (authoritative): global plugins + current user's private plugins. + try: + pq = self.db.query(InstalledPlugin) + if owner_user_id: + pq = pq.filter( + or_( + InstalledPlugin.owner_user_id == owner_user_id, + InstalledPlugin.owner_user_id.is_(None), + ) + ) + else: + pq = pq.filter(InstalledPlugin.owner_user_id.is_(None)) + plugin_rows = pq.order_by(InstalledPlugin.created_at.desc()).all() + except Exception as exc: # noqa: BLE001 + logger.debug("Failed to list installed plugins: %s", exc) + plugin_rows = [] + + plugin_skill_ids: set = set() + plugin_mcp_ids: set = set() + # The same plugin may exist both as a global version (install_id=slug@global) + # and as the user's private version (slug@) — same name → duplicate + # display (the user sees two "定时任务管理" entries). Dedupe by slug and + # show only one: prefer the user's private version (its components carry + # the user's fingerprint and match their sandbox credentials); fall back + # to the global version when there is no private one. + # Note: plugin_skill_ids / plugin_mcp_ids still accumulate from **all** + # rows (including the deduped-away one), so the loose skill/MCP lists + # can fully exclude both versions' components. + _plugin_by_slug: Dict[str, Dict[str, Any]] = {} + for p in plugin_rows: + cids = p.component_ids or {} + s_ids = list(cids.get("skills") or []) + m_ids = list(cids.get("mcp") or []) + plugin_skill_ids.update(s_ids) + plugin_mcp_ids.update(m_ids) + slug = (p.install_id or "").rsplit("@", 1)[0] + is_owned = bool(owner_user_id) and p.owner_user_id == owner_user_id + existing = _plugin_by_slug.get(slug) + if existing is None or (is_owned and not existing["_owned"]): + _plugin_by_slug[slug] = { + "id": p.install_id, + "name": p.name, + "description": p.description or "", + "skill_count": len(s_ids), + "mcp_count": len(m_ids), + "_owned": is_owned, + } + plugin_list: List[Dict[str, Any]] = [ + {k: v for k, v in item.items() if k != "_owned"} for item in _plugin_by_slug.values() + ] + + # Built-in plugin MCPs can also be present in the static catalog without + # a source_plugin DB row. They still belong under the plugin selector, + # not the loose MCP selector. + try: + from core.services.plugin_service import builtin_plugin_component_ids + + _, builtin_plugin_mcp_ids = builtin_plugin_component_ids() + plugin_mcp_ids.update(builtin_plugin_mcp_ids) + except Exception as exc: # noqa: BLE001 + logger.debug("Failed to load built-in plugin MCP ids: %s", exc) + + # Resolve the user's personal on/off layer once so each MCP option can + # explain whether it is already enabled for the main agent. This flag + # is display metadata only; disabled options remain bindable here. + enabled_mcp_ids: Optional[set[str]] = None + if owner_user_id: + try: + from core.config.catalog_resolver import resolve_all_runtime_enabled + + _skills, _agents, resolved_mcps = resolve_all_runtime_enabled( + self.db, owner_user_id + ) + if resolved_mcps is not None: + enabled_mcp_ids = set(resolved_mcps) + except Exception as exc: # noqa: BLE001 + logger.debug("Failed to resolve user MCP enablement: %s", exc) + + # ── MCP tools (exclude plugin-sourced + owner isolation) ── + # Owner isolation: private entries (owner_user_id non-null) are visible + # only to their owner. Otherwise private copies produced by other users + # installing the same plugin (e.g. automation-automation_task-) would all leak into this user's bindable list → + # duplicates of "定时任务" etc. Empty owner = globally shared entry, + # visible to everyone. + _mcp_owner_ok = ( + or_( + AdminMcpServer.owner_user_id.is_(None), + AdminMcpServer.owner_user_id == owner_user_id, + ) + if owner_user_id + else AdminMcpServer.owner_user_id.is_(None) + ) + mcp_servers = ( + self.db.query(AdminMcpServer) + .filter( + _mcp_owner_ok, + # Authoritative exclusion: any plugin-sourced MCP (source_plugin + # nonnull) never enters the loose list — it is bound as a whole via + # the plugins list instead. More robust than checking only + # plugin_mcp_ids (doesn't depend on the plugin row still + # existing/being visible), and also blocks orphaned plugin MCP rows. + AdminMcpServer.source_plugin.is_(None), + ) + .order_by(AdminMcpServer.sort_order) + .all() + ) + mcp_list: List[Dict[str, Any]] = [] + seen_mcp_ids: set = set() + for server in mcp_servers: + # A global false is an administrator lock. A private false is the + # owner's personal off state and remains eligible for an explicit + # sub-agent binding. + if server.owner_user_id is None and not server.is_enabled: + continue + if server.server_id in plugin_mcp_ids: + continue + mcp_list.append( + { + "id": server.server_id, + "name": server.display_name, + "description": server.description, + "enabled": ( + server.server_id in enabled_mcp_ids + if enabled_mcp_ids is not None + else bool(server.is_enabled) + ), + } + ) + seen_mcp_ids.add(server.server_id) + + # Some umbrella/built-in MCP entries are catalog-defined and do not + # necessarily have a same-named AdminMcpServer row. Include every + # administrator-enabled catalog item so the selector is complete. + try: + from core.config.catalog_runtime import get_runtime_catalog + + runtime_catalog = get_runtime_catalog(self.db, include_runtime_details=False) + for item in runtime_catalog.get("mcp") or []: + item_id = str(item.get("id") or "").strip() + if ( + not item_id + or item_id in seen_mcp_ids + or item_id in plugin_mcp_ids + or not bool(item.get("enabled", True)) + ): + continue + mcp_list.append( + { + "id": item_id, + "name": item.get("name") or item_id, + "description": item.get("description") or item.get("desc") or "", + "enabled": ( + item_id in enabled_mcp_ids + if enabled_mcp_ids is not None + else bool(item.get("enabled", True)) + ), + } + ) + seen_mcp_ids.add(item_id) + except Exception as exc: # noqa: BLE001 + logger.debug("Failed to load catalog MCP resources: %s", exc) + + # ── Skills: DB-managed + filesystem-discovered (both exclude plugin-sourced + owner isolation) ── + # ⚠️ Critical: the filesystem loader (load_all_metadata) scans **all** + # materialized skills on disk — including other users' private skills + # and plugin skills materialized by other users' plugin installs. + # Excluding via plugin_skill_ids alone (components of plugins visible to + # the current user only) would miss some, letting other users' plugin + # skills sneak into "bindable skills". So compute two **authoritative** + # exclusion sets directly from AdminSkill and filter both sources + # uniformly: + # - all_plugin_skill_ids: any plugin-sourced skill (source_plugin non-null, any owner); + # - foreign_private_skill_ids: other users' private skills (owner_user_id non-null and ≠ current user). + from core.db.models import AdminSkill + + all_plugin_skill_ids: set = set() + foreign_private_skill_ids: set = set() + try: + for row in self.db.query( + AdminSkill.skill_id, AdminSkill.source_plugin, AdminSkill.owner_user_id + ).all(): + if row.source_plugin: + all_plugin_skill_ids.add(row.skill_id) + if row.owner_user_id and row.owner_user_id != owner_user_id: + foreign_private_skill_ids.add(row.skill_id) + except Exception as exc: + logger.debug("Failed to precompute skill exclusion sets: %s", exc) + + _excluded_skill_ids = plugin_skill_ids | all_plugin_skill_ids | foreign_private_skill_ids + + skill_list: List[Dict[str, Any]] = [] + try: + _skill_owner_ok = ( + or_(AdminSkill.owner_user_id.is_(None), AdminSkill.owner_user_id == owner_user_id) + if owner_user_id + else AdminSkill.owner_user_id.is_(None) + ) + db_skills = ( + self.db.query(AdminSkill) + .filter( + AdminSkill.is_enabled == True, + _skill_owner_ok, + ) + .order_by(AdminSkill.updated_at.desc()) + .all() + ) + seen_ids = set() + for s in db_skills: + if s.skill_id in _excluded_skill_ids: + continue + skill_list.append( + {"id": s.skill_id, "name": s.display_name, "description": s.description or ""} + ) + seen_ids.add(s.skill_id) + except Exception: + seen_ids = set() + + try: + from core.agent_skills.loader import get_skill_loader + + loader = get_skill_loader() + for sid, meta in loader.load_all_metadata().items(): + if sid not in seen_ids and sid not in _excluded_skill_ids: + skill_list.append( + { + "id": sid, + "name": getattr(meta, "name", sid), + "description": getattr(meta, "description", ""), + } + ) + except Exception as exc: + logger.debug("Failed to load filesystem skills: %s", exc) + + # KB spaces + kb_list: List[Dict[str, Any]] = [] + try: + kb_spaces = ( + self.db.query(KBSpace) + .filter( + KBSpace.deleted_at.is_(None), + ) + .order_by(KBSpace.created_at.desc()) + .all() + ) + kb_list = [ + {"id": s.kb_id, "name": s.name, "description": s.description or ""} + for s in kb_spaces + ] + except Exception: + pass + + return { + "mcp_servers": mcp_list, + "skills": skill_list, + "plugins": plugin_list, + "kb_spaces": kb_list, + "ontology_tags": self._ontology_tag_options(), + } + + def _ontology_tag_options(self) -> List[Dict[str, Any]]: + """Controlled labels that activate a sub-agent ontology workflow at runtime.""" + from core.services.ontology_service import OntologyService + + return OntologyService(self.db).list_asset_tag_options("subagent") + + # ── Helpers ─────────────────────────────────────────────────────── + + def _validate_create_scope( + self, user_id: Optional[str], owner_type: str, scope_id: Optional[str] + ) -> None: + if owner_type == "user": + if not user_id: + raise ValueError("user_id required for user agents") + if self.repo.count_user_agents(user_id) >= MAX_USER_AGENTS: + raise ValueError(f"Maximum {MAX_USER_AGENTS} agents per user reached") + return + if owner_type == "admin": + return + raise ValueError(f"Unsupported agent owner type: {owner_type}") + + @staticmethod + def _owner_fields( + user_id: Optional[str], owner_type: str, scope_id: Optional[str] + ) -> Dict[str, Any]: + return {"user_id": user_id if owner_type == "user" else None} + + def _is_accessible(self, agent: UserAgent, user_id: str) -> bool: + if agent.owner_type == "admin" and agent.is_enabled: + return True + if agent.owner_type == "user" and agent.user_id == user_id: + return True + return False + + def _check_ownership(self, agent: UserAgent, user_id: Optional[str], owner_type: str) -> None: + # owner_type is the route context: 'admin' = Admin console routes, everything else = user-side routes + if owner_type == "admin": + if agent.owner_type != "admin": + raise PermissionError("Admin can only modify admin agents") + return + if agent.owner_type == "user" and agent.user_id == user_id: + return + raise PermissionError("You can only modify your own agents") + + @classmethod + def _serialize(cls, agent: UserAgent) -> Dict[str, Any]: + extra_config = agent.extra_config or {} + return { + "agent_id": agent.agent_id, + "owner_type": agent.owner_type, + "user_id": agent.user_id, + **cls._serialize_scope(agent), + "name": agent.name, + "avatar": agent.avatar, + "description": agent.description, + "system_prompt": agent.system_prompt, + "welcome_message": agent.welcome_message, + "suggested_questions": agent.suggested_questions or [], + "mcp_server_ids": agent.mcp_server_ids or [], + "skill_ids": agent.skill_ids or [], + "plugin_ids": agent.plugin_ids or [], + "kb_ids": agent.kb_ids or [], + "model_provider_id": agent.model_provider_id, + "temperature": float(agent.temperature) if agent.temperature is not None else None, + "max_tokens": agent.max_tokens, + "max_iters": agent.max_iters, + "timeout": agent.timeout, + "is_enabled": agent.is_enabled, + "sort_order": agent.sort_order, + "source_market_slug": agent.source_market_slug, + "ontology_tags": list(extra_config.get("ontology_tags") or []), + "extra_config": extra_config, + "version": cls._read_version(extra_config), + "change_history": cls._read_change_history(extra_config), + "created_at": agent.created_at.isoformat() if agent.created_at else None, + "updated_at": agent.updated_at.isoformat() if agent.updated_at else None, + "created_by": agent.created_by, + } + + @staticmethod + def _serialize_scope(agent: UserAgent) -> Dict[str, Any]: + return {} + + def _audit( + self, user_id: Optional[str], action: str, resource_id: str, details: Dict = None + ) -> None: + try: + audit_repo = AuditLogRepository(self.db) + audit_repo.create( + { + "user_id": user_id, + "action": action, + "resource_type": "user_agent", + "resource_id": resource_id, + "details": details or {}, + "status": "success", + } + ) + except Exception as exc: + logger.warning("Audit log failed: %s", exc) + + @staticmethod + def _normalize_value(value: Any) -> Any: + if isinstance(value, Decimal): + return float(value) + if isinstance(value, list): + return list(value) + return value + + @staticmethod + def _now_iso() -> str: + return datetime.now().replace(microsecond=0).isoformat() + + @classmethod + def _collect_changed_fields(cls, agent: UserAgent, data: Dict[str, Any]) -> List[str]: + fields: List[str] = [] + for field in VERSIONED_FIELDS: + if field not in data: + continue + old_value = cls._normalize_value(getattr(agent, field, None)) + new_value = cls._normalize_value(data.get(field)) + if old_value != new_value: + fields.append(field) + return fields + + @staticmethod + def _read_version(extra_config: Optional[Dict[str, Any]]) -> str: + if not isinstance(extra_config, dict): + return DEFAULT_AGENT_VERSION + raw = extra_config.get("version") + return UserAgentBaseService._normalize_version(raw if isinstance(raw, str) else "") + + @staticmethod + def _read_change_history(extra_config: Optional[Dict[str, Any]]) -> List[Dict[str, str]]: + if not isinstance(extra_config, dict): + return [] + raw_history = extra_config.get("change_history") + if not isinstance(raw_history, list): + return [] + + history: List[Dict[str, str]] = [] + for item in raw_history: + if not isinstance(item, dict): + continue + timestamp = item.get("timestamp") + content = item.get("content") + version = item.get("version") + operator_name = item.get("operator_name") + details = item.get("details") + if not isinstance(timestamp, str) or not isinstance(content, str): + continue + history.append( + { + "timestamp": timestamp, + "content": content, + "version": UserAgentBaseService._normalize_version( + version if isinstance(version, str) else "" + ), + "operator_name": ( + operator_name + if isinstance(operator_name, str) and operator_name.strip() + else "未知用户" + ), + "details": UserAgentBaseService._normalize_change_details(details), + } + ) + return history + + @staticmethod + def _increment_version(version: str) -> str: + normalized = UserAgentBaseService._normalize_version(version) + match = re.match(r"^[Vv](\d+)\.(\d+)$", normalized) + if not match: + return "V1.1" + major, minor = (int(part) for part in match.groups()) + return f"V{major}.{minor + 1}" + + @staticmethod + def _build_change_summary(changed_fields: List[str], data: Dict[str, Any]) -> str: + if changed_fields == ["is_enabled"]: + return "启用了子智能体" if bool(data.get("is_enabled")) else "停用了子智能体" + + changed_labels = [VERSIONED_FIELDS[field] for field in changed_fields] + if not changed_labels: + return "更新了智能体配置" + if len(changed_labels) <= 3: + return f"修改了{'、'.join(changed_labels)}" + preview = "、".join(changed_labels[:3]) + return f"修改了{preview}等{len(changed_labels)}项" + + @classmethod + def _build_change_details( + cls, agent: UserAgent, changed_fields: List[str], data: Dict[str, Any] + ) -> List[Dict[str, str]]: + details: List[Dict[str, str]] = [] + for field in changed_fields: + old_value = cls._stringify_detail_value(field, getattr(agent, field, None)) + new_value = cls._stringify_detail_value(field, data.get(field)) + details.append( + { + "field": VERSIONED_FIELDS[field], + "before": old_value, + "after": new_value, + } + ) + return details + + @staticmethod + def _stringify_detail_value(field: str, value: Any) -> str: + if field == "is_enabled": + return "启用" if bool(value) else "关闭" + if value is None: + return "未填写" + if isinstance(value, list): + return "、".join(str(item) for item in value) if value else "未填写" + if isinstance(value, bool): + return "是" if value else "否" + text = str(value).strip() + return text if text else "未填写" + + @staticmethod + def _normalize_change_details(details: Any) -> List[Dict[str, str]]: + if not isinstance(details, list): + return [] + normalized: List[Dict[str, str]] = [] + for item in details: + if not isinstance(item, dict): + continue + field = item.get("field") + before = item.get("before") + after = item.get("after") + if not isinstance(field, str): + continue + normalized.append( + { + "field": field, + "before": before if isinstance(before, str) else "未填写", + "after": after if isinstance(after, str) else "未填写", + } + ) + return normalized + + @staticmethod + def _normalize_version(version: str) -> str: + if not isinstance(version, str) or not version.strip(): + return DEFAULT_AGENT_VERSION + raw = version.strip() + if re.match(r"^[Vv]\d+\.\d+$", raw): + return f"V{raw[1:]}" + legacy_patch = re.match(r"^(\d+)\.(\d+)\.(\d+)$", raw) + if legacy_patch: + major, minor, patch = (int(part) for part in legacy_patch.groups()) + return f"V{major}.{minor + patch}" + legacy_minor = re.match(r"^(\d+)\.(\d+)$", raw) + if legacy_minor: + major, minor = (int(part) for part in legacy_minor.groups()) + return f"V{major}.{minor}" + return DEFAULT_AGENT_VERSION + + @staticmethod + def _merge_extra_config( + current_extra: Optional[Dict[str, Any]], + incoming_extra: Optional[Dict[str, Any]], + *, + version: str, + change_history: List[Dict[str, str]], + ) -> Dict[str, Any]: + merged = dict(current_extra or {}) + if isinstance(incoming_extra, dict): + merged.update(incoming_extra) + merged["version"] = version + merged["change_history"] = change_history + return merged diff --git a/src/backend/core/services/user_agent_service.py b/src/backend/core/services/user_agent_service.py index 2426f1d4..4c95be63 100644 --- a/src/backend/core/services/user_agent_service.py +++ b/src/backend/core/services/user_agent_service.py @@ -1,808 +1,7 @@ -"""Business logic for custom sub-agents (UserAgent).""" +"""Community custom sub-agent service.""" -from __future__ import annotations +from core.services.user_agent_base import UserAgentBaseService -import logging -import re -import uuid -from datetime import datetime -from decimal import Decimal -from typing import Any, Dict, List, Optional +UserAgentService = UserAgentBaseService -from core.db.models import UserAgent -from core.db.repository import AuditLogRepository, UserAgentRepository -from core.ontology.build_validator import ensure_ontology_build_valid -from sqlalchemy.orm import Session - -logger = logging.getLogger(__name__) - -MAX_USER_AGENTS = 20 -MAX_TEAM_AGENTS = 50 -DEFAULT_AGENT_VERSION = "V1.0" -MAX_CHANGE_HISTORY = 30 -NON_VERSIONED_FIELDS = {"is_enabled"} -VERSIONED_FIELDS = { - "name": "名称", - "description": "简介", - "system_prompt": "角色设定", - "welcome_message": "开场白", - "suggested_questions": "推荐问题", - "mcp_server_ids": "绑定工具", - "skill_ids": "绑定技能", - "plugin_ids": "绑定插件", - "kb_ids": "绑定知识库", - "model_provider_id": "模型", - "temperature": "温度", - "max_tokens": "最大输出长度", - "max_iters": "最大推理轮次", - "timeout": "超时时间", - "is_enabled": "启用状态", -} - - -class UserAgentService: - """Service for user agent CRUD and permission checks.""" - - def __init__(self, db: Session): - self.db = db - self.repo = UserAgentRepository(db) - - # ── Queries ────────────────────────────────────────────────────── - - def list_for_user(self, user_id: str) -> List[Dict[str, Any]]: - agents = self.repo.list_for_user(user_id) - return [self._serialize(a) for a in agents] - - def list_admin(self) -> List[Dict[str, Any]]: - agents = self.repo.list_admin() - return [self._serialize(a) for a in agents] - - def get_by_id(self, agent_id: str, user_id: Optional[str] = None) -> Dict[str, Any]: - agent = self.repo.get_by_id(agent_id) - if not agent: - raise LookupError(f"Agent {agent_id} not found") - if user_id and not self._is_accessible(agent, user_id): - raise PermissionError("No access to this agent") - return self._serialize(agent) - - def get_raw_by_id(self, agent_id: str, user_id: Optional[str] = None) -> UserAgent: - """Return the ORM object (for direct use by workflow/factory).""" - agent = self.repo.get_by_id(agent_id) - if not agent: - raise LookupError(f"Agent {agent_id} not found") - if user_id and not self._is_accessible(agent, user_id): - raise PermissionError("No access to this agent") - return agent - - # ── Mutations ──────────────────────────────────────────────────── - - def create( - self, - user_id: Optional[str], - operator_name: Optional[str], - owner_type: str, - data: Dict[str, Any], - team_id: Optional[str] = None, - ) -> Dict[str, Any]: - data = dict(data) - incoming_extra = dict(data.get("extra_config") or {}) - ontology_tags = list( - data.pop("ontology_tags", incoming_extra.get("ontology_tags") or []) or [] - ) - incoming_extra["ontology_tags"] = ontology_tags - data["extra_config"] = incoming_extra - ensure_ontology_build_valid( - self.db, - asset_type="subagent", - name=str(data.get("name") or ""), - description=str(data.get("description") or ""), - instructions=str(data.get("system_prompt") or ""), - mcp_server_ids=list(data.get("mcp_server_ids") or []), - skill_ids=list(data.get("skill_ids") or []), - plugin_ids=list(data.get("plugin_ids") or []), - ontology_tags=ontology_tags, - ) - if owner_type == "user": - if not user_id: - raise ValueError("user_id required for user agents") - count = self.repo.count_user_agents(user_id) - if count >= MAX_USER_AGENTS: - raise ValueError(f"Maximum {MAX_USER_AGENTS} agents per user reached") - elif owner_type == "team": - if not team_id: - raise ValueError("team_id required for team agents") - # Only the team owner/admin may create team sub-agents - if not self._is_team_manager(user_id, team_id): - raise PermissionError("Only team owner/admin can create team agents") - count = self.repo.count_team_agents(team_id) - if count >= MAX_TEAM_AGENTS: - raise ValueError(f"Maximum {MAX_TEAM_AGENTS} agents per team reached") - - agent_id = f"ua_{uuid.uuid4().hex[:16]}" - created_at = self._now_iso() - creation_history = [ - { - "version": DEFAULT_AGENT_VERSION, - "timestamp": created_at, - "content": "创建了子智能体", - "operator_name": operator_name or user_id or "未知用户", - "details": [], - } - ] - extra_config = self._merge_extra_config( - current_extra=None, - incoming_extra=incoming_extra, - version=DEFAULT_AGENT_VERSION, - change_history=creation_history, - ) - - record = { - "agent_id": agent_id, - "owner_type": owner_type, - "user_id": user_id if owner_type == "user" else None, - "team_id": team_id if owner_type == "team" else None, - "created_by": user_id, - **data, - "extra_config": extra_config, - } - agent = self.repo.create(record) - self._audit( - user_id, - "agent.create", - agent_id, - {"owner_type": owner_type, "team_id": team_id, "name": data.get("name")}, - ) - return self._serialize(agent) - - def update( - self, - agent_id: str, - user_id: Optional[str], - operator_name: Optional[str], - owner_type: str, - data: Dict[str, Any], - ) -> Dict[str, Any]: - agent = self.repo.get_by_id(agent_id) - if not agent: - raise LookupError(f"Agent {agent_id} not found") - self._check_ownership(agent, user_id, owner_type) - - data = dict(data) - current_extra = dict(agent.extra_config or {}) - incoming_extra = dict(data.get("extra_config") or {}) - ontology_tags = list( - data.pop( - "ontology_tags", - incoming_extra.get("ontology_tags", current_extra.get("ontology_tags") or []), - ) - or [] - ) - incoming_extra["ontology_tags"] = ontology_tags - if "extra_config" in data or ontology_tags != list( - current_extra.get("ontology_tags") or [] - ): - data["extra_config"] = incoming_extra - ensure_ontology_build_valid( - self.db, - asset_type="subagent", - name=str(data.get("name", agent.name) or ""), - description=str(data.get("description", agent.description) or ""), - instructions=str(data.get("system_prompt", agent.system_prompt) or ""), - mcp_server_ids=list(data.get("mcp_server_ids", agent.mcp_server_ids) or []), - skill_ids=list(data.get("skill_ids", agent.skill_ids) or []), - plugin_ids=list(data.get("plugin_ids", agent.plugin_ids) or []), - ontology_tags=ontology_tags, - ) - - changed_fields = self._collect_changed_fields(agent, data) - versioned_fields = [field for field in changed_fields if field not in NON_VERSIONED_FIELDS] - changed_labels = [VERSIONED_FIELDS[field] for field in changed_fields] - next_version = self._read_version(current_extra) - change_history = self._read_change_history(current_extra) - - if changed_labels: - change_summary = self._build_change_summary(changed_fields, data) - change_details = self._build_change_details(agent, changed_fields, data) - entry_version = next_version - if versioned_fields: - next_version = self._increment_version(next_version) - entry_version = next_version - change_history.append( - { - "version": entry_version, - "timestamp": self._now_iso(), - "content": change_summary, - "operator_name": operator_name or user_id or "未知用户", - "details": change_details, - } - ) - change_history = change_history[-MAX_CHANGE_HISTORY:] - - payload = dict(data) - payload["extra_config"] = self._merge_extra_config( - current_extra=current_extra, - incoming_extra=incoming_extra, - version=next_version, - change_history=change_history, - ) - - agent = self.repo.update(agent_id, payload) - audit_details = {"fields": list(data.keys())} - if changed_labels: - audit_details["change_summary"] = change_summary - audit_details["version"] = next_version - self._audit(user_id, "agent.update", agent_id, audit_details) - return self._serialize(agent) - - def delete( - self, - agent_id: str, - user_id: Optional[str], - owner_type: str, - ) -> bool: - agent = self.repo.get_by_id(agent_id) - if not agent: - raise LookupError(f"Agent {agent_id} not found") - self._check_ownership(agent, user_id, owner_type) - - ok = self.repo.delete(agent_id) - self._audit(user_id, "agent.delete", agent_id) - return ok - - def toggle_enabled(self, agent_id: str) -> Dict[str, Any]: - agent = self.repo.get_by_id(agent_id) - if not agent: - raise LookupError(f"Agent {agent_id} not found") - new_val = not agent.is_enabled - agent = self.repo.update(agent_id, {"is_enabled": new_val}) - return self._serialize(agent) - - # ── Available resources ────────────────────────────────────────── - - def list_available_resources(self, owner_user_id: Optional[str] = None) -> Dict[str, Any]: - """Return MCP servers, skills, plugins, and KB spaces bindable to agents. - - Plugin-sourced skills/MCP are removed from the skills/mcp_servers lists - and instead bound as a whole via the ``plugins`` list (plugin = - installable/removable unit, expanded at runtime into its skills+tools). - owner_user_id is used to include that user's private plugins and MCPs. - - The MCP list intentionally contains capabilities the user has personally - switched off. A sub-agent binding is an explicit, narrower capability - grant and therefore may opt into one of those tools without turning it - on for the user's main agent. Deployment-global MCPs disabled by an - administrator remain unavailable. - """ - from core.db.models import AdminMcpServer, InstalledPlugin, KBSpace - from sqlalchemy import or_ - - # ── Plugin list + their component id sets (used to strip plugin capabilities from the loose skills/tools) ── - # Query the ORM directly for component_ids (authoritative): global plugins + current user's private plugins. - try: - pq = self.db.query(InstalledPlugin) - if owner_user_id: - pq = pq.filter( - or_( - InstalledPlugin.owner_user_id == owner_user_id, - InstalledPlugin.owner_user_id.is_(None), - ) - ) - else: - pq = pq.filter(InstalledPlugin.owner_user_id.is_(None)) - plugin_rows = pq.order_by(InstalledPlugin.created_at.desc()).all() - except Exception as exc: # noqa: BLE001 - logger.debug("Failed to list installed plugins: %s", exc) - plugin_rows = [] - - plugin_skill_ids: set = set() - plugin_mcp_ids: set = set() - # The same plugin may exist both as a global version (install_id=slug@global) - # and as the user's private version (slug@) — same name → duplicate - # display (the user sees two "定时任务管理" entries). Dedupe by slug and - # show only one: prefer the user's private version (its components carry - # the user's fingerprint and match their sandbox credentials); fall back - # to the global version when there is no private one. - # Note: plugin_skill_ids / plugin_mcp_ids still accumulate from **all** - # rows (including the deduped-away one), so the loose skill/MCP lists - # can fully exclude both versions' components. - _plugin_by_slug: Dict[str, Dict[str, Any]] = {} - for p in plugin_rows: - cids = p.component_ids or {} - s_ids = list(cids.get("skills") or []) - m_ids = list(cids.get("mcp") or []) - plugin_skill_ids.update(s_ids) - plugin_mcp_ids.update(m_ids) - slug = (p.install_id or "").rsplit("@", 1)[0] - is_owned = bool(owner_user_id) and p.owner_user_id == owner_user_id - existing = _plugin_by_slug.get(slug) - if existing is None or (is_owned and not existing["_owned"]): - _plugin_by_slug[slug] = { - "id": p.install_id, - "name": p.name, - "description": p.description or "", - "skill_count": len(s_ids), - "mcp_count": len(m_ids), - "_owned": is_owned, - } - plugin_list: List[Dict[str, Any]] = [ - {k: v for k, v in item.items() if k != "_owned"} for item in _plugin_by_slug.values() - ] - - # Built-in plugin MCPs can also be present in the static catalog without - # a source_plugin DB row. They still belong under the plugin selector, - # not the loose MCP selector. - try: - from core.services.plugin_service import builtin_plugin_component_ids - - _, builtin_plugin_mcp_ids = builtin_plugin_component_ids() - plugin_mcp_ids.update(builtin_plugin_mcp_ids) - except Exception as exc: # noqa: BLE001 - logger.debug("Failed to load built-in plugin MCP ids: %s", exc) - - # Resolve the user's personal on/off layer once so each MCP option can - # explain whether it is already enabled for the main agent. This flag - # is display metadata only; disabled options remain bindable here. - enabled_mcp_ids: Optional[set[str]] = None - if owner_user_id: - try: - from core.config.catalog_resolver import resolve_all_runtime_enabled - - _skills, _agents, resolved_mcps = resolve_all_runtime_enabled( - self.db, owner_user_id - ) - if resolved_mcps is not None: - enabled_mcp_ids = set(resolved_mcps) - except Exception as exc: # noqa: BLE001 - logger.debug("Failed to resolve user MCP enablement: %s", exc) - - # ── MCP tools (exclude plugin-sourced + owner isolation) ── - # Owner isolation: private entries (owner_user_id non-null) are visible - # only to their owner. Otherwise private copies produced by other users - # installing the same plugin (e.g. automation-automation_task-) would all leak into this user's bindable list → - # duplicates of "定时任务" etc. Empty owner = globally shared entry, - # visible to everyone. - _mcp_owner_ok = ( - or_( - AdminMcpServer.owner_user_id.is_(None), - AdminMcpServer.owner_user_id == owner_user_id, - ) - if owner_user_id - else AdminMcpServer.owner_user_id.is_(None) - ) - mcp_servers = ( - self.db.query(AdminMcpServer) - .filter( - _mcp_owner_ok, - # Authoritative exclusion: any plugin-sourced MCP (source_plugin - # nonnull) never enters the loose list — it is bound as a whole via - # the plugins list instead. More robust than checking only - # plugin_mcp_ids (doesn't depend on the plugin row still - # existing/being visible), and also blocks orphaned plugin MCP rows. - AdminMcpServer.source_plugin.is_(None), - ) - .order_by(AdminMcpServer.sort_order) - .all() - ) - mcp_list: List[Dict[str, Any]] = [] - seen_mcp_ids: set = set() - for server in mcp_servers: - # A global false is an administrator lock. A private false is the - # owner's personal off state and remains eligible for an explicit - # sub-agent binding. - if server.owner_user_id is None and not server.is_enabled: - continue - if server.server_id in plugin_mcp_ids: - continue - mcp_list.append( - { - "id": server.server_id, - "name": server.display_name, - "description": server.description, - "enabled": ( - server.server_id in enabled_mcp_ids - if enabled_mcp_ids is not None - else bool(server.is_enabled) - ), - } - ) - seen_mcp_ids.add(server.server_id) - - # Some umbrella/built-in MCP entries are catalog-defined and do not - # necessarily have a same-named AdminMcpServer row. Include every - # administrator-enabled catalog item so the selector is complete. - try: - from core.config.catalog_runtime import get_runtime_catalog - - runtime_catalog = get_runtime_catalog(self.db, include_runtime_details=False) - for item in runtime_catalog.get("mcp") or []: - item_id = str(item.get("id") or "").strip() - if ( - not item_id - or item_id in seen_mcp_ids - or item_id in plugin_mcp_ids - or not bool(item.get("enabled", True)) - ): - continue - mcp_list.append( - { - "id": item_id, - "name": item.get("name") or item_id, - "description": item.get("description") or item.get("desc") or "", - "enabled": ( - item_id in enabled_mcp_ids - if enabled_mcp_ids is not None - else bool(item.get("enabled", True)) - ), - } - ) - seen_mcp_ids.add(item_id) - except Exception as exc: # noqa: BLE001 - logger.debug("Failed to load catalog MCP resources: %s", exc) - - # ── Skills: DB-managed + filesystem-discovered (both exclude plugin-sourced + owner isolation) ── - # ⚠️ Critical: the filesystem loader (load_all_metadata) scans **all** - # materialized skills on disk — including other users' private skills - # and plugin skills materialized by other users' plugin installs. - # Excluding via plugin_skill_ids alone (components of plugins visible to - # the current user only) would miss some, letting other users' plugin - # skills sneak into "bindable skills". So compute two **authoritative** - # exclusion sets directly from AdminSkill and filter both sources - # uniformly: - # - all_plugin_skill_ids: any plugin-sourced skill (source_plugin non-null, any owner); - # - foreign_private_skill_ids: other users' private skills (owner_user_id non-null and ≠ current user). - from core.db.models import AdminSkill - - all_plugin_skill_ids: set = set() - foreign_private_skill_ids: set = set() - try: - for row in self.db.query( - AdminSkill.skill_id, AdminSkill.source_plugin, AdminSkill.owner_user_id - ).all(): - if row.source_plugin: - all_plugin_skill_ids.add(row.skill_id) - if row.owner_user_id and row.owner_user_id != owner_user_id: - foreign_private_skill_ids.add(row.skill_id) - except Exception as exc: - logger.debug("Failed to precompute skill exclusion sets: %s", exc) - - _excluded_skill_ids = plugin_skill_ids | all_plugin_skill_ids | foreign_private_skill_ids - - skill_list: List[Dict[str, Any]] = [] - try: - _skill_owner_ok = ( - or_(AdminSkill.owner_user_id.is_(None), AdminSkill.owner_user_id == owner_user_id) - if owner_user_id - else AdminSkill.owner_user_id.is_(None) - ) - db_skills = ( - self.db.query(AdminSkill) - .filter( - AdminSkill.is_enabled == True, - _skill_owner_ok, - ) - .order_by(AdminSkill.updated_at.desc()) - .all() - ) - seen_ids = set() - for s in db_skills: - if s.skill_id in _excluded_skill_ids: - continue - skill_list.append( - {"id": s.skill_id, "name": s.display_name, "description": s.description or ""} - ) - seen_ids.add(s.skill_id) - except Exception: - seen_ids = set() - - try: - from core.agent_skills.loader import get_skill_loader - - loader = get_skill_loader() - for sid, meta in loader.load_all_metadata().items(): - if sid not in seen_ids and sid not in _excluded_skill_ids: - skill_list.append( - { - "id": sid, - "name": getattr(meta, "name", sid), - "description": getattr(meta, "description", ""), - } - ) - except Exception as exc: - logger.debug("Failed to load filesystem skills: %s", exc) - - # KB spaces - kb_list: List[Dict[str, Any]] = [] - try: - kb_spaces = ( - self.db.query(KBSpace) - .filter( - KBSpace.deleted_at.is_(None), - ) - .order_by(KBSpace.created_at.desc()) - .all() - ) - kb_list = [ - {"id": s.kb_id, "name": s.name, "description": s.description or ""} - for s in kb_spaces - ] - except Exception: - pass - - return { - "mcp_servers": mcp_list, - "skills": skill_list, - "plugins": plugin_list, - "kb_spaces": kb_list, - "ontology_tags": self._ontology_tag_options(), - } - - def _ontology_tag_options(self) -> List[Dict[str, Any]]: - """Controlled labels that activate a sub-agent ontology workflow at runtime.""" - from core.services.ontology_service import OntologyService - - return OntologyService(self.db).list_asset_tag_options("subagent") - - # ── Helpers ─────────────────────────────────────────────────────── - - def _team_role(self, user_id: Optional[str], team_id: Optional[str]) -> Optional[str]: - """The current user's role in a team (owner/admin/member); None for non-members.""" - if not user_id or not team_id: - return None - from core.db.repository import TeamRepository - - return TeamRepository(self.db).get_member_role(team_id, user_id) - - def _is_team_manager(self, user_id: Optional[str], team_id: Optional[str]) -> bool: - return self._team_role(user_id, team_id) in ("owner", "admin") - - def _is_accessible(self, agent: UserAgent, user_id: str) -> bool: - if agent.owner_type == "admin" and agent.is_enabled: - return True - if agent.owner_type == "user" and agent.user_id == user_id: - return True - if agent.owner_type == "team" and agent.team_id: - role = self._team_role(user_id, agent.team_id) - if role is None: - return False - # Enabled team agents are visible to all members; disabled ones only to owner/admin (for management) - return bool(agent.is_enabled) or role in ("owner", "admin") - return False - - def _check_ownership(self, agent: UserAgent, user_id: Optional[str], owner_type: str) -> None: - # owner_type is the route context: 'admin' = Admin console routes, everything else = user-side routes - if owner_type == "admin": - if agent.owner_type != "admin": - raise PermissionError("Admin can only modify admin agents") - return - if agent.owner_type == "user" and agent.user_id == user_id: - return - if agent.owner_type == "team" and self._is_team_manager(user_id, agent.team_id): - return - raise PermissionError("You can only modify your own or your team's agents") - - @staticmethod - def _serialize(agent: UserAgent) -> Dict[str, Any]: - extra_config = agent.extra_config or {} - return { - "agent_id": agent.agent_id, - "owner_type": agent.owner_type, - "user_id": agent.user_id, - "team_id": agent.team_id, - "name": agent.name, - "avatar": agent.avatar, - "description": agent.description, - "system_prompt": agent.system_prompt, - "welcome_message": agent.welcome_message, - "suggested_questions": agent.suggested_questions or [], - "mcp_server_ids": agent.mcp_server_ids or [], - "skill_ids": agent.skill_ids or [], - "plugin_ids": agent.plugin_ids or [], - "kb_ids": agent.kb_ids or [], - "model_provider_id": agent.model_provider_id, - "temperature": float(agent.temperature) if agent.temperature is not None else None, - "max_tokens": agent.max_tokens, - "max_iters": agent.max_iters, - "timeout": agent.timeout, - "is_enabled": agent.is_enabled, - "sort_order": agent.sort_order, - "source_market_slug": agent.source_market_slug, - "ontology_tags": list(extra_config.get("ontology_tags") or []), - "extra_config": extra_config, - "version": UserAgentService._read_version(extra_config), - "change_history": UserAgentService._read_change_history(extra_config), - "created_at": agent.created_at.isoformat() if agent.created_at else None, - "updated_at": agent.updated_at.isoformat() if agent.updated_at else None, - "created_by": agent.created_by, - } - - def _audit( - self, user_id: Optional[str], action: str, resource_id: str, details: Dict = None - ) -> None: - try: - audit_repo = AuditLogRepository(self.db) - audit_repo.create( - { - "user_id": user_id, - "action": action, - "resource_type": "user_agent", - "resource_id": resource_id, - "details": details or {}, - "status": "success", - } - ) - except Exception as exc: - logger.warning("Audit log failed: %s", exc) - - @staticmethod - def _normalize_value(value: Any) -> Any: - if isinstance(value, Decimal): - return float(value) - if isinstance(value, list): - return list(value) - return value - - @staticmethod - def _now_iso() -> str: - return datetime.now().replace(microsecond=0).isoformat() - - @classmethod - def _collect_changed_fields(cls, agent: UserAgent, data: Dict[str, Any]) -> List[str]: - fields: List[str] = [] - for field in VERSIONED_FIELDS: - if field not in data: - continue - old_value = cls._normalize_value(getattr(agent, field, None)) - new_value = cls._normalize_value(data.get(field)) - if old_value != new_value: - fields.append(field) - return fields - - @staticmethod - def _read_version(extra_config: Optional[Dict[str, Any]]) -> str: - if not isinstance(extra_config, dict): - return DEFAULT_AGENT_VERSION - raw = extra_config.get("version") - return UserAgentService._normalize_version(raw if isinstance(raw, str) else "") - - @staticmethod - def _read_change_history(extra_config: Optional[Dict[str, Any]]) -> List[Dict[str, str]]: - if not isinstance(extra_config, dict): - return [] - raw_history = extra_config.get("change_history") - if not isinstance(raw_history, list): - return [] - - history: List[Dict[str, str]] = [] - for item in raw_history: - if not isinstance(item, dict): - continue - timestamp = item.get("timestamp") - content = item.get("content") - version = item.get("version") - operator_name = item.get("operator_name") - details = item.get("details") - if not isinstance(timestamp, str) or not isinstance(content, str): - continue - history.append( - { - "timestamp": timestamp, - "content": content, - "version": UserAgentService._normalize_version( - version if isinstance(version, str) else "" - ), - "operator_name": ( - operator_name - if isinstance(operator_name, str) and operator_name.strip() - else "未知用户" - ), - "details": UserAgentService._normalize_change_details(details), - } - ) - return history - - @staticmethod - def _increment_version(version: str) -> str: - normalized = UserAgentService._normalize_version(version) - match = re.match(r"^[Vv](\d+)\.(\d+)$", normalized) - if not match: - return "V1.1" - major, minor = (int(part) for part in match.groups()) - return f"V{major}.{minor + 1}" - - @staticmethod - def _build_change_summary(changed_fields: List[str], data: Dict[str, Any]) -> str: - if changed_fields == ["is_enabled"]: - return "启用了子智能体" if bool(data.get("is_enabled")) else "停用了子智能体" - - changed_labels = [VERSIONED_FIELDS[field] for field in changed_fields] - if not changed_labels: - return "更新了智能体配置" - if len(changed_labels) <= 3: - return f"修改了{'、'.join(changed_labels)}" - preview = "、".join(changed_labels[:3]) - return f"修改了{preview}等{len(changed_labels)}项" - - @classmethod - def _build_change_details( - cls, agent: UserAgent, changed_fields: List[str], data: Dict[str, Any] - ) -> List[Dict[str, str]]: - details: List[Dict[str, str]] = [] - for field in changed_fields: - old_value = cls._stringify_detail_value(field, getattr(agent, field, None)) - new_value = cls._stringify_detail_value(field, data.get(field)) - details.append( - { - "field": VERSIONED_FIELDS[field], - "before": old_value, - "after": new_value, - } - ) - return details - - @staticmethod - def _stringify_detail_value(field: str, value: Any) -> str: - if field == "is_enabled": - return "启用" if bool(value) else "关闭" - if value is None: - return "未填写" - if isinstance(value, list): - return "、".join(str(item) for item in value) if value else "未填写" - if isinstance(value, bool): - return "是" if value else "否" - text = str(value).strip() - return text if text else "未填写" - - @staticmethod - def _normalize_change_details(details: Any) -> List[Dict[str, str]]: - if not isinstance(details, list): - return [] - normalized: List[Dict[str, str]] = [] - for item in details: - if not isinstance(item, dict): - continue - field = item.get("field") - before = item.get("before") - after = item.get("after") - if not isinstance(field, str): - continue - normalized.append( - { - "field": field, - "before": before if isinstance(before, str) else "未填写", - "after": after if isinstance(after, str) else "未填写", - } - ) - return normalized - - @staticmethod - def _normalize_version(version: str) -> str: - if not isinstance(version, str) or not version.strip(): - return DEFAULT_AGENT_VERSION - raw = version.strip() - if re.match(r"^[Vv]\d+\.\d+$", raw): - return f"V{raw[1:]}" - legacy_patch = re.match(r"^(\d+)\.(\d+)\.(\d+)$", raw) - if legacy_patch: - major, minor, patch = (int(part) for part in legacy_patch.groups()) - return f"V{major}.{minor + patch}" - legacy_minor = re.match(r"^(\d+)\.(\d+)$", raw) - if legacy_minor: - major, minor = (int(part) for part in legacy_minor.groups()) - return f"V{major}.{minor}" - return DEFAULT_AGENT_VERSION - - @staticmethod - def _merge_extra_config( - current_extra: Optional[Dict[str, Any]], - incoming_extra: Optional[Dict[str, Any]], - *, - version: str, - change_history: List[Dict[str, str]], - ) -> Dict[str, Any]: - merged = dict(current_extra or {}) - if isinstance(incoming_extra, dict): - merged.update(incoming_extra) - merged["version"] = version - merged["change_history"] = change_history - return merged +__all__ = ["UserAgentService"] diff --git a/src/backend/core/services/user_folder_service.py b/src/backend/core/services/user_folder_service.py index 4ed222c7..8ef4ccb6 100644 --- a/src/backend/core/services/user_folder_service.py +++ b/src/backend/core/services/user_folder_service.py @@ -1,17 +1,15 @@ """Personal folder (UserFolder) business logic. -Mirrors TeamFolderService's capabilities, acting only on the "My Space" personal -file hierarchy: +Manages only the "My Space" personal file hierarchy: - depth limit (MAX_FOLDER_DEPTH) - same-name check among siblings (with soft-delete fallback) - cycle detection on move - cascading soft delete (itself + descendant folders + associated personal artifacts) - audit persistence -Note: a personal artifact's ownership is determined by user_id + user_folder_id; -team_id / team_folder_id are both NULL. This service does not handle -"personal ↔ team" migration — that remains the job of the existing team_files -routes. +Personal artifact ownership is determined by ``user_id`` and +``user_folder_id``. Edition-specific ownership transitions are handled outside +this shared service. """ from __future__ import annotations @@ -22,11 +20,11 @@ from datetime import datetime from typing import Any, Dict, List, Optional, Tuple -from sqlalchemy.orm import Session - from core.db.models import Artifact, UserFolder from core.db.repository import ArtifactRepository, AuditLogRepository +from core.services.artifact_edition import is_personal_artifact, personal_artifact_create_fields from core.storage import get_storage +from sqlalchemy.orm import Session MAX_FOLDER_DEPTH = 8 @@ -151,9 +149,11 @@ def create_folder( self.db.query(UserFolder) .filter( UserFolder.user_id == user_id, - UserFolder.parent_folder_id.is_(parent_folder_id) - if parent_folder_id is None - else UserFolder.parent_folder_id == parent_folder_id, + ( + UserFolder.parent_folder_id.is_(parent_folder_id) + if parent_folder_id is None + else UserFolder.parent_folder_id == parent_folder_id + ), UserFolder.name == cleaned, UserFolder.deleted_at.is_(None), ) @@ -172,14 +172,16 @@ def create_folder( self.db.add(folder) self.db.commit() - self.audit.create({ - "user_id": actor, - "action": "user_folder.create", - "resource_type": "user_folder", - "resource_id": folder_id, - "details": {"user_id": user_id, "parent": parent_folder_id, "name": cleaned}, - "status": "success", - }) + self.audit.create( + { + "user_id": actor, + "action": "user_folder.create", + "resource_type": "user_folder", + "resource_id": folder_id, + "details": {"user_id": user_id, "parent": parent_folder_id, "name": cleaned}, + "status": "success", + } + ) return FolderResult(True, "文件夹已创建", folder_id=folder_id) def rename_folder(self, folder_id: str, name: str, actor: str) -> FolderResult: @@ -197,9 +199,11 @@ def rename_folder(self, folder_id: str, name: str, actor: str) -> FolderResult: self.db.query(UserFolder) .filter( UserFolder.user_id == folder.user_id, - UserFolder.parent_folder_id.is_(folder.parent_folder_id) - if folder.parent_folder_id is None - else UserFolder.parent_folder_id == folder.parent_folder_id, + ( + UserFolder.parent_folder_id.is_(folder.parent_folder_id) + if folder.parent_folder_id is None + else UserFolder.parent_folder_id == folder.parent_folder_id + ), UserFolder.name == cleaned, UserFolder.deleted_at.is_(None), UserFolder.folder_id != folder_id, @@ -213,14 +217,16 @@ def rename_folder(self, folder_id: str, name: str, actor: str) -> FolderResult: folder.updated_at = datetime.utcnow() self.db.commit() - self.audit.create({ - "user_id": actor, - "action": "user_folder.rename", - "resource_type": "user_folder", - "resource_id": folder_id, - "details": {"name": cleaned}, - "status": "success", - }) + self.audit.create( + { + "user_id": actor, + "action": "user_folder.rename", + "resource_type": "user_folder", + "resource_id": folder_id, + "details": {"name": cleaned}, + "status": "success", + } + ) return FolderResult(True, "已重命名", folder_id=folder_id) def move_folder( @@ -256,9 +262,11 @@ def move_folder( self.db.query(UserFolder) .filter( UserFolder.user_id == folder.user_id, - UserFolder.parent_folder_id.is_(new_parent_id) - if new_parent_id is None - else UserFolder.parent_folder_id == new_parent_id, + ( + UserFolder.parent_folder_id.is_(new_parent_id) + if new_parent_id is None + else UserFolder.parent_folder_id == new_parent_id + ), UserFolder.name == folder.name, UserFolder.deleted_at.is_(None), UserFolder.folder_id != folder_id, @@ -272,14 +280,16 @@ def move_folder( folder.updated_at = datetime.utcnow() self.db.commit() - self.audit.create({ - "user_id": actor, - "action": "user_folder.move", - "resource_type": "user_folder", - "resource_id": folder_id, - "details": {"new_parent_id": new_parent_id}, - "status": "success", - }) + self.audit.create( + { + "user_id": actor, + "action": "user_folder.move", + "resource_type": "user_folder", + "resource_id": folder_id, + "details": {"new_parent_id": new_parent_id}, + "status": "success", + } + ) return FolderResult(True, "已移动", folder_id=folder_id) def _collect_descendants(self, folder_id: str) -> List[str]: @@ -326,17 +336,19 @@ def delete_folder(self, folder_id: str, actor: str) -> Tuple[FolderResult, int]: self.db.commit() - self.audit.create({ - "user_id": actor, - "action": "user_folder.delete", - "resource_type": "user_folder", - "resource_id": folder_id, - "details": { - "cascaded_folder_ids": ids_to_delete, - "artifacts_affected": int(affected or 0), - }, - "status": "success", - }) + self.audit.create( + { + "user_id": actor, + "action": "user_folder.delete", + "resource_type": "user_folder", + "resource_id": folder_id, + "details": { + "cascaded_folder_ids": ids_to_delete, + "artifacts_affected": int(affected or 0), + }, + "status": "success", + } + ) return FolderResult(True, "文件夹已删除", folder_id=folder_id), int(affected or 0) def count_affected_artifacts(self, folder_id: str, user_id: str) -> int: @@ -366,8 +378,7 @@ def move_artifact( Validation: - the artifact exists and belongs to the actor - the target folder (if any) belongs to the actor - - the artifact must be a personal file (team_id is NULL). A team file must - first be transferred back to personal via the team API. + - the artifact must be a personal file. """ artifact = ( self.db.query(Artifact) @@ -376,8 +387,8 @@ def move_artifact( ) if artifact is None or artifact.user_id != actor: return FolderResult(False, "文件不存在") - if artifact.team_id is not None: - return FolderResult(False, "团队文件请通过团队 API 移动") + if not is_personal_artifact(artifact): + return FolderResult(False, "非个人文件不能在个人空间中移动") if target_folder_id is not None: target = self.get(target_folder_id) @@ -388,14 +399,16 @@ def move_artifact( artifact.updated_at = datetime.utcnow() self.db.commit() - self.audit.create({ - "user_id": actor, - "action": "user_folder.move_artifact", - "resource_type": "artifact", - "resource_id": artifact_id, - "details": {"target_folder_id": target_folder_id}, - "status": "success", - }) + self.audit.create( + { + "user_id": actor, + "action": "user_folder.move_artifact", + "resource_type": "artifact", + "resource_id": artifact_id, + "details": {"target_folder_id": target_folder_id}, + "status": "success", + } + ) return FolderResult(True, "已移动") def copy_artifact( @@ -410,7 +423,7 @@ def copy_artifact( field; source/storage untouched); copy creates a separate new artifact record + duplicates the storage object (``download_bytes`` from the old key → ``upload_bytes`` to a new key), leaving the source file intact. Same - validation as move: exists, belongs to the actor, not a team file, target + validation as move: exists, belongs to the actor, is personal, target folder belongs to the actor. """ artifact = ( @@ -420,8 +433,8 @@ def copy_artifact( ) if artifact is None or artifact.user_id != actor: return FolderResult(False, "文件不存在") - if artifact.team_id is not None: - return FolderResult(False, "团队文件请通过团队 API 复制") + if not is_personal_artifact(artifact): + return FolderResult(False, "非个人文件不能复制到个人空间") if target_folder_id is not None: target = self.get(target_folder_id) @@ -435,33 +448,39 @@ def copy_artifact( try: content = storage.download_bytes(artifact.storage_key) new_url = storage.upload_bytes(content, new_key) - except Exception as exc: # noqa: BLE001 — leave no half-written record when storage I/O fails + except ( + Exception + ) as exc: # noqa: BLE001 — leave no half-written record when storage I/O fails return FolderResult(False, f"复制失败：存储对象读写出错（{exc}）") extra = dict(artifact.extra_data or {}) extra.update({"source": "copy_personal", "copied_from": artifact.artifact_id}) - ArtifactRepository(self.db).create({ - "artifact_id": new_id, - "chat_id": None, - "user_id": actor, - "user_folder_id": target_folder_id, - "team_id": None, - "type": artifact.type, - "title": artifact.title, - "filename": artifact.filename, - "size_bytes": artifact.size_bytes, - "mime_type": artifact.mime_type, - "storage_key": new_key, - "storage_url": new_url, - "extra_data": extra, - }) - - self.audit.create({ - "user_id": actor, - "action": "user_folder.copy_artifact", - "resource_type": "artifact", - "resource_id": new_id, - "details": {"target_folder_id": target_folder_id, "copied_from": artifact_id}, - "status": "success", - }) + ArtifactRepository(self.db).create( + { + "artifact_id": new_id, + "chat_id": None, + "user_id": actor, + "user_folder_id": target_folder_id, + **personal_artifact_create_fields(), + "type": artifact.type, + "title": artifact.title, + "filename": artifact.filename, + "size_bytes": artifact.size_bytes, + "mime_type": artifact.mime_type, + "storage_key": new_key, + "storage_url": new_url, + "extra_data": extra, + } + ) + + self.audit.create( + { + "user_id": actor, + "action": "user_folder.copy_artifact", + "resource_type": "artifact", + "resource_id": new_id, + "details": {"target_folder_id": target_folder_id, "copied_from": artifact_id}, + "status": "success", + } + ) return FolderResult(True, "已复制", artifact_id=new_id) diff --git a/src/backend/core/services/user_service.py b/src/backend/core/services/user_service.py index fc839c16..d61b2213 100644 --- a/src/backend/core/services/user_service.py +++ b/src/backend/core/services/user_service.py @@ -1,14 +1,13 @@ """User-related business logic.""" -from typing import Optional, Dict, Any -from datetime import datetime import uuid -from sqlalchemy.orm import Session +from datetime import datetime +from typing import Any, Dict, Optional -from core.db.repository import UserRepository, AuditLogRepository +from core.auth.account_policy import AccountCapacityExceeded, account_capacity_block_reason from core.db.models import UserShadow -from core.licensing import SeatLimitExceeded -from core.licensing.seats import seat_block_reason +from core.db.repository import AuditLogRepository, UserRepository +from sqlalchemy.orm import Session class UserService: @@ -24,7 +23,7 @@ def get_or_create_user_shadow( user_center_id: str, username: str, email: Optional[str] = None, - avatar_url: Optional[str] = None + avatar_url: Optional[str] = None, ) -> UserShadow: """ Lazy load user shadow from user center. @@ -43,21 +42,16 @@ def get_or_create_user_shadow( # Note: avatar_url is only updated when SSO returns a non-empty value—— # the user may have set their own avatar in SettingsModal, and SSO returns # None in most scenarios; we must not overwrite the user's custom avatar with None. - update_data = { - "username": username, - "email": email, - "last_sync_at": datetime.utcnow() - } + update_data = {"username": username, "email": email, "last_sync_at": datetime.utcnow()} if avatar_url: update_data["avatar_url"] = avatar_url return self.repo.update(user.user_id, update_data) else: - # License seat cap (M4): SSO/external-auth auto account creation is also - # subject to the seat constraint—— only "new creation" is blocked; existing - # users' logins are unaffected. The domain exception is rendered as 402 by error_handler. - block_reason = seat_block_reason(self.db) + # Edition policy may cap account creation. Existing users are never + # blocked by this admission check. + block_reason = account_capacity_block_reason(self.db) if block_reason: - raise SeatLimitExceeded(block_reason) + raise AccountCapacityExceeded(block_reason) # Create new user shadow user_data = { @@ -66,22 +60,23 @@ def get_or_create_user_shadow( "username": username, "email": email, "avatar_url": avatar_url, - "last_sync_at": datetime.utcnow() + "last_sync_at": datetime.utcnow(), } user = self.repo.create(user_data) # Audit log - self.audit_repo.create({ - "user_id": user.user_id, - "action": "user.created", - "resource_type": "user", - "resource_id": user.user_id, - "status": "success" - }) + self.audit_repo.create( + { + "user_id": user.user_id, + "action": "user.created", + "resource_type": "user", + "resource_id": user.user_id, + "status": "success", + } + ) return user - def get_user_settings(self, user_id: str) -> Dict[str, Any]: """Read preferences and apply effective memory capability defaults.""" user = self.repo.get_by_id(user_id) diff --git a/src/backend/mcp_servers/_ports.py b/src/backend/mcp_servers/_ports.py index 546a4ef6..fcbf1e2a 100644 --- a/src/backend/mcp_servers/_ports.py +++ b/src/backend/mcp_servers/_ports.py @@ -1,4 +1,4 @@ -"""Single source of truth for MCP server → port mapping（社区版：8 个通用工具）. +"""Single source of truth for MCP server → port mapping（社区版：9 个通用工具）. Both ``core/config/mcp_config.py`` (which builds backend-side ``http://mcp:NNNN/mcp/`` URLs) and ``mcp_servers/_launcher.py`` (which @@ -7,21 +7,23 @@ Port assignments are stable; never reassign without updating both catalog/display_names and any deployed configs. """ + from __future__ import annotations # server_id (the catalog/display_names key) → port PORTS: dict[str, int] = { - "retrieve_dataset_content": 9100, # historical KB port + "retrieve_dataset_content": 9100, # historical KB port # 9101 reserved（行业数据库查询，商业版） - "internet_search": 9102, + "internet_search": 9102, # 9103 reserved（产业知识中心查询，商业版） - "generate_chart_tool": 9104, - "report_export_mcp": 9105, - "web_fetch": 9106, - "batch_runner": 9107, - "automation_task": 9108, + "generate_chart_tool": 9104, + "report_export_mcp": 9105, + "web_fetch": 9106, + "batch_runner": 9107, + "automation_task": 9108, # 9109-9111 reserved (excel/ppt/pdf 已转生为 skill_bundles 技能；9108 已复用) - "skill_manager": 9112, + "skill_manager": 9112, + "site_publish": 9113, } diff --git a/src/backend/mcp_servers/_serve.py b/src/backend/mcp_servers/_serve.py index 3b8461ee..039d4769 100644 --- a/src/backend/mcp_servers/_serve.py +++ b/src/backend/mcp_servers/_serve.py @@ -4,16 +4,22 @@ Picks transport from ``--transport`` (``stdio`` for local debug, the default; ``streamable-http`` for the dedicated mcp container). """ + from __future__ import annotations import argparse import asyncio +import os from typing import TYPE_CHECKING if TYPE_CHECKING: from mcp.server.fastmcp import FastMCP +def _streamable_http_bind_host() -> str: + return os.getenv("MCP_BIND_HOST", "0.0.0.0").strip() or "0.0.0.0" + + def run(mcp: "FastMCP", default_port: int) -> None: parser = argparse.ArgumentParser(description=mcp.name) parser.add_argument( @@ -26,7 +32,12 @@ def run(mcp: "FastMCP", default_port: int) -> None: if args.transport == "streamable-http": mcp.settings.port = args.port - mcp.settings.host = "0.0.0.0" # noqa: S104 — private docker network + # Compose needs all-interface binding inside its private container + # network. The no-Docker local/desktop launcher explicitly supplies + # MCP_BIND_HOST=127.0.0.1 so these internal control-plane ports are not + # exposed to the user's LAN (and do not trigger an avoidable Windows + # firewall prompt). + mcp.settings.host = _streamable_http_bind_host() # MCP's default DNS-rebinding allow-list is localhost only. Backend # reaches us via the docker DNS name (e.g. ``mcp:9108``); rebinding # protection isn't relevant on a private network with no browser diff --git a/src/backend/mcp_servers/generate_chart_tool_mcp/_selftest.py b/src/backend/mcp_servers/generate_chart_tool_mcp/_selftest.py index c63c08b5..fee2b45a 100755 --- a/src/backend/mcp_servers/generate_chart_tool_mcp/_selftest.py +++ b/src/backend/mcp_servers/generate_chart_tool_mcp/_selftest.py @@ -18,8 +18,6 @@ def _fail(msg: str) -> None: def main() -> None: # Import-time env guards for other modules. os.environ.setdefault("TAVILY_API_KEY", "DUMMY") - os.environ.setdefault("DIFY_API_KEY", "DUMMY") - os.environ.setdefault("DIFY_URL", "http://localhost") os.environ.setdefault("DATABASE_URL", "http://localhost") try: diff --git a/src/backend/mcp_servers/retrieve_dataset_content_mcp/_selftest.py b/src/backend/mcp_servers/retrieve_dataset_content_mcp/_selftest.py index 5f42c0e2..284e7a6f 100755 --- a/src/backend/mcp_servers/retrieve_dataset_content_mcp/_selftest.py +++ b/src/backend/mcp_servers/retrieve_dataset_content_mcp/_selftest.py @@ -17,8 +17,6 @@ def _fail(msg: str) -> None: def main() -> None: os.environ.setdefault("TAVILY_API_KEY", "DUMMY") - os.environ.setdefault("DIFY_API_KEY", "DUMMY") - os.environ.setdefault("DIFY_URL", "http://localhost") try: importlib.import_module("mcp_servers.retrieve_dataset_content_mcp.server") diff --git a/src/backend/mcp_servers/retrieve_dataset_content_mcp/impl.py b/src/backend/mcp_servers/retrieve_dataset_content_mcp/impl.py old mode 100755 new mode 100644 index 2d026efb..a14aa227 --- a/src/backend/mcp_servers/retrieve_dataset_content_mcp/impl.py +++ b/src/backend/mcp_servers/retrieve_dataset_content_mcp/impl.py @@ -1,605 +1,53 @@ -"""Implementation for MCP tools: retrieve_dataset_content & retrieve_local_kb.""" +"""Edition-neutral knowledge retrieval facade.""" from __future__ import annotations -import asyncio -import json import logging import os -import time from typing import Any, Dict, List -import httpx -import requests -from core.auth.kb_permissions import ( - get_accessible_local_kb_ids, - get_dataset_levels, - is_shared_visibility, +from core.auth.kb_permissions import get_accessible_local_kb_ids, is_shared_visibility +from core.kb.external_retrieval import ( + MAX_RETRIEVE_TOKENS, + RETRIEVE_MAX_CONCURRENCY, + RETRIEVE_REQUEST_TIMEOUT_SECONDS, + RETRIEVE_TOTAL_TIMEOUT_SECONDS, + DatasetRetrievalTimeoutError, + DatasetRetrievalUnavailableError, + list_external_datasets, + retrieve_dataset_content, + retrieve_dataset_content_async, +) +from mcp_servers.retrieve_dataset_content_mcp.local_impl import ( + LOCAL_RETRIEVE_STAGE_TIMEOUT_SECONDS, + LOCAL_RETRIEVE_TOTAL_TIMEOUT_SECONDS, + LocalKnowledgeBaseTimeoutError, + _build_runtime_local_kb_section, + retrieve_local_kb, ) -from core.config.runtime_env import get_runtime_value -from core.kb.dify_kb import get_allowed_dataset_ids -from dotenv import load_dotenv -from mcp_servers._retrieve_cleaning import clean_retrieve_document, truncate_records_by_tokens - -load_dotenv() _logger = logging.getLogger(__name__) -def _read_int_env(name: str, default: int) -> int: - raw = (os.getenv(name) or "").strip() - if not raw: - return default - try: - value = int(raw) - return value if value > 0 else default - except ValueError: - return default - - -MAX_RETRIEVE_TOKENS = _read_int_env("RETRIEVE_DATASET_TOKEN_LIMIT", 50_000) -RETRIEVE_REQUEST_TIMEOUT_SECONDS = _read_int_env("RETRIEVE_DATASET_REQUEST_TIMEOUT_SECONDS", 10) -RETRIEVE_TOTAL_TIMEOUT_SECONDS = _read_int_env("RETRIEVE_DATASET_TOTAL_TIMEOUT_SECONDS", 60) -RETRIEVE_MAX_CONCURRENCY = _read_int_env("RETRIEVE_DATASET_MAX_CONCURRENCY", 3) -LOCAL_RETRIEVE_TOTAL_TIMEOUT_SECONDS = _read_int_env("RETRIEVE_LOCAL_KB_TIMEOUT_SECONDS", 30) -LOCAL_RETRIEVE_STAGE_TIMEOUT_SECONDS = _read_int_env("RETRIEVE_LOCAL_KB_STAGE_TIMEOUT_SECONDS", 10) - -_public_retrieve_limiter: asyncio.Semaphore | None = None -_public_retrieve_limiter_loop: asyncio.AbstractEventLoop | None = None - - -class DatasetRetrievalTimeoutError(TimeoutError): - """Raised when the public knowledge-base fan-out exceeds its total deadline.""" - - -class DatasetRetrievalUnavailableError(RuntimeError): - """Raised when every targeted Dify dataset request fails upstream.""" - - -class LocalKnowledgeBaseTimeoutError(TimeoutError): - """Raised when private knowledge-base work exhausts its internal deadline.""" - - -def _get_public_retrieve_limiter() -> asyncio.Semaphore: - """Return one process-wide limiter bound to the active MCP event loop. - - The MCP server uses a single loop in production. Recreating the semaphore - when the loop changes keeps isolated pytest event loops safe as well. - """ - global _public_retrieve_limiter, _public_retrieve_limiter_loop - - loop = asyncio.get_running_loop() - if _public_retrieve_limiter is None or _public_retrieve_limiter_loop is not loop: - _public_retrieve_limiter = asyncio.Semaphore(RETRIEVE_MAX_CONCURRENCY) - _public_retrieve_limiter_loop = loop - return _public_retrieve_limiter - - -def _write_retrieve_log(message: str) -> None: - """Route retrieval diagnostics through logging, never process-global stdout.""" - _logger.info("%s", message.rstrip()) - - -def _resolve_dify_config() -> tuple[str, str]: - """DB (admin panel) → env fallback for Dify base URL / API key.""" - base_url = ( - (get_runtime_value("DIFY_URL") or os.getenv("DIFY_BASE_URL") or "").strip().rstrip("/") - ) - auth_token = (get_runtime_value("DIFY_API_KEY") or os.getenv("DIFY_AUTH_TOKEN") or "").strip() - return base_url, auth_token - - -def _normalize_token_field(records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Normalize record token key so token truncation is effective.""" - normalized: List[Dict[str, Any]] = [] - for record in records: - item = dict(record) - token_val = item.get("tokens", item.get("token", 0)) - try: - item["tokens"] = int(token_val or 0) - except (TypeError, ValueError): - item["tokens"] = 0 - normalized.append(item) - return normalized - - -def _retrieve_single_dataset( - dataset_id: str, - query: str, - top_k: int, - score_threshold: float, - search_method: str, - reranking_enable: bool, - weights: float, - base_url: str, - headers: dict, -) -> List[Dict[str, Any]]: - """Retrieve from a single Dify dataset. Returns cleaned records list.""" - url = f"{base_url}/datasets/{dataset_id}/retrieve" - payload: Dict[str, Any] = { - "query": query, - "retrieval_model": { - "search_method": search_method, - "reranking_enable": reranking_enable, - "top_k": top_k, - "score_threshold_enabled": True, - "score_threshold": score_threshold, - "weights": weights, - }, - } - - try: - resp = requests.post( - url, - headers=headers, - json=payload, - timeout=RETRIEVE_REQUEST_TIMEOUT_SECONDS, - ) - resp.raise_for_status() - data = resp.json() - records = data.get("records", []) - cleaned = clean_retrieve_document(records) - # If strict threshold yields no results, retry once with threshold disabled. - if not cleaned and score_threshold > 0: - retry_payload = dict(payload) - retrieval_model = dict(retry_payload.get("retrieval_model", {})) - retrieval_model["score_threshold_enabled"] = False - retry_payload["retrieval_model"] = retrieval_model - retry_resp = requests.post( - url, - headers=headers, - json=retry_payload, - timeout=RETRIEVE_REQUEST_TIMEOUT_SECONDS, - ) - retry_resp.raise_for_status() - retry_data = retry_resp.json() - retry_records = retry_data.get("records", []) - cleaned = clean_retrieve_document(retry_records) - # 附上 dataset_id，供前端调用详情接口 - for item in cleaned: - item["dataset_id"] = dataset_id - return cleaned - except Exception as exc: - _logger.warning("数据集 %s 查询失败: %s", dataset_id, exc) - return [] - - -def _finalize_retrieve_records( - records: List[Dict[str, Any]], - *, - top_k: int, -) -> List[Dict[str, Any]]: - """Sort, limit, normalize and token-truncate retrieved records.""" - if not records: - _logger.info("知识库未找到相关内容") - return [] - - for item in records: - if "score" not in item: - item["score"] = 0.0 - records.sort(key=lambda item: item.get("score", 0), reverse=True) - records = records[:top_k] - records = _normalize_token_field(records) - records = truncate_records_by_tokens( - records, - token_threshold=MAX_RETRIEVE_TOKENS, - writer=_write_retrieve_log, - ) - _logger.info("从知识库找到 %d 条相关记录", len(records)) - return records - - -def retrieve_dataset_content( - query: str, - dataset_id: str = "", - top_k: int = 10, - score_threshold: float = 0.4, - search_method: str = "hybrid_search", - reranking_enable: bool = False, - weights: float = 0.6, - *, - allowed_dataset_ids: str | None = None, - current_user_id: str | None = None, -) -> List[Dict[str, Any]]: - _logger.info("正在通过知识库搜索 %s 的结果", query) - - base_url, auth_token = _resolve_dify_config() - if not base_url or not auth_token: - _logger.error("知识库工具配置缺失：请设置 DIFY_URL 与 DIFY_API_KEY") - return [ - { - "error": "retrieve_dataset_content 配置缺失", - "hint": "请设置 DIFY_URL 与 DIFY_API_KEY", - } - ] - - headers = {"Authorization": f"Bearer {auth_token}", "Content-Type": "application/json"} - - # Resolve allowed set from header (HTTP mode) or env var - if allowed_dataset_ids is not None: - _allowed_set = {x.strip() for x in allowed_dataset_ids.split(",") if x.strip()} - else: - _allowed_set = get_allowed_dataset_ids() - - # If no allowed set configured, fetch all from Dify - if not _allowed_set: - try: - from core.kb.dify_kb import list_datasets - - _allowed_set = { - str(ds.get("id", "")).strip() for ds in list_datasets(timeout=5) if ds.get("id") - } - except Exception: - pass - - if not _allowed_set: - _logger.warning("没有可用的知识库数据集") - return [] - - # 权限强制（公有库权限分配）：按当前用户可见的 Dify 数据集收口 _allowed_set。 - # X-Allowed-Dataset-Ids 头已在 agent_factory 过滤过，这里是第二道防线——即便头为空 - # 触发了「拉全部数据集」的兜底，也只保留用户有权访问的（public + 已授权 scoped）。 - user_id = (current_user_id or "").strip() or os.getenv("CURRENT_USER_ID", "").strip() - if user_id: - try: - from core.db.engine import SessionLocal - - with SessionLocal() as _db: - accessible = set(get_dataset_levels(_db, user_id, sorted(_allowed_set)).keys()) - _allowed_set = {d for d in _allowed_set if d in accessible} - except Exception as exc: - _logger.warning("数据集权限解析失败，按空集处理：%s", exc) - _allowed_set = set() - if not _allowed_set: - _logger.warning("当前用户无可访问的知识库数据集") - return [] - - # If user specified a dataset_id, only search that one (with validation) - specified_id = (dataset_id or "").strip() - if specified_id: - if _allowed_set and specified_id not in _allowed_set: - _logger.warning("dataset_id %s 不在允许列表中", specified_id) - return [] - target_ids = [specified_id] - _logger.info("搜索指定数据集: %s", specified_id) - else: - # Default: search ALL allowed datasets - target_ids = sorted(_allowed_set) - _logger.info("正在搜索全部 %d 个数据集", len(target_ids)) - - all_cleaned: List[Dict[str, Any]] = [] - for ds_id in target_ids: - items = _retrieve_single_dataset( - dataset_id=ds_id, - query=query, - top_k=top_k, - score_threshold=score_threshold, - search_method=search_method, - reranking_enable=reranking_enable, - weights=weights, - base_url=base_url, - headers=headers, - ) - all_cleaned.extend(items) - - return _finalize_retrieve_records(all_cleaned, top_k=top_k) - - -async def _retrieve_single_dataset_async( - *, - client: httpx.AsyncClient, - limiter: asyncio.Semaphore, - dataset_id: str, - query: str, - top_k: int, - score_threshold: float, - search_method: str, - reranking_enable: bool, - weights: float, - base_url: str, - headers: dict[str, str], -) -> tuple[List[Dict[str, Any]], bool]: - """Retrieve one Dify dataset without blocking the MCP event loop.""" - url = f"{base_url}/datasets/{dataset_id}/retrieve" - payload: Dict[str, Any] = { - "query": query, - "retrieval_model": { - "search_method": search_method, - "reranking_enable": reranking_enable, - "top_k": top_k, - "score_threshold_enabled": True, - "score_threshold": score_threshold, - "weights": weights, - }, - } - - try: - async with limiter: - response = await client.post(url, headers=headers, json=payload) - response.raise_for_status() - cleaned = clean_retrieve_document(response.json().get("records", [])) - - if not cleaned and score_threshold > 0: - retry_payload = dict(payload) - retrieval_model = dict(retry_payload["retrieval_model"]) - retrieval_model["score_threshold_enabled"] = False - retry_payload["retrieval_model"] = retrieval_model - retry_response = await client.post( - url, - headers=headers, - json=retry_payload, - ) - retry_response.raise_for_status() - cleaned = clean_retrieve_document(retry_response.json().get("records", [])) - - for item in cleaned: - item["dataset_id"] = dataset_id - return cleaned, True - except asyncio.CancelledError: - raise - except Exception as exc: - _logger.warning( - "数据集 %s 异步查询失败 (%s): %s", - dataset_id, - type(exc).__name__, - exc, - ) - return [], False - - -def _resolve_public_retrieve_scope( - *, - allowed_dataset_ids: str | None, - current_user_id: str | None, -) -> tuple[str, str, set[str]]: - """Resolve Dify config and effective dataset permissions in a worker thread.""" - base_url, auth_token = _resolve_dify_config() - if not base_url or not auth_token: - return base_url, auth_token, set() - - if allowed_dataset_ids is not None: - allowed_set = {item.strip() for item in allowed_dataset_ids.split(",") if item.strip()} - else: - allowed_set = get_allowed_dataset_ids() - - if not allowed_set: - try: - from core.kb.dify_kb import list_datasets - - allowed_set = { - str(dataset.get("id", "")).strip() - for dataset in list_datasets(timeout=5) - if dataset.get("id") - } - except Exception as exc: - _logger.warning("拉取 Dify 数据集列表失败: %s", exc) - - user_id = (current_user_id or "").strip() or os.getenv("CURRENT_USER_ID", "").strip() - if allowed_set and user_id: - try: - from core.db.engine import SessionLocal - - with SessionLocal() as db: - accessible = set(get_dataset_levels(db, user_id, sorted(allowed_set)).keys()) - allowed_set = {dataset for dataset in allowed_set if dataset in accessible} - except Exception as exc: - _logger.warning("数据集权限解析失败，按空集处理: %s", exc) - allowed_set = set() - - return base_url, auth_token, allowed_set - - -async def retrieve_dataset_content_async( - query: str, - dataset_id: str = "", - top_k: int = 10, - score_threshold: float = 0.4, - search_method: str = "hybrid_search", - reranking_enable: bool = False, - weights: float = 0.6, - *, - allowed_dataset_ids: str | None = None, - current_user_id: str | None = None, -) -> List[Dict[str, Any]]: - """Async public KB retrieval with bounded fan-out and a total deadline.""" - loop = asyncio.get_running_loop() - deadline = loop.time() + RETRIEVE_TOTAL_TIMEOUT_SECONDS - try: - base_url, auth_token, allowed_set = await asyncio.wait_for( - asyncio.to_thread( - _resolve_public_retrieve_scope, - allowed_dataset_ids=allowed_dataset_ids, - current_user_id=current_user_id, - ), - timeout=RETRIEVE_TOTAL_TIMEOUT_SECONDS, - ) - except asyncio.TimeoutError as exc: - raise DatasetRetrievalTimeoutError( - f"公有知识库准备阶段超过 {RETRIEVE_TOTAL_TIMEOUT_SECONDS}s" - ) from exc - - if not base_url or not auth_token: - return [ - { - "error": "retrieve_dataset_content 配置缺失", - "hint": "请设置 DIFY_URL 与 DIFY_API_KEY", - } - ] - if not allowed_set: - return [] - - specified_id = (dataset_id or "").strip() - if specified_id: - if specified_id not in allowed_set: - _logger.warning("dataset_id %s 不在允许列表中", specified_id) - return [] - target_ids = [specified_id] - else: - target_ids = sorted(allowed_set) - - headers = {"Authorization": f"Bearer {auth_token}", "Content-Type": "application/json"} - limiter = _get_public_retrieve_limiter() - timeout = httpx.Timeout(RETRIEVE_REQUEST_TIMEOUT_SECONDS) - limits = httpx.Limits( - max_connections=RETRIEVE_MAX_CONCURRENCY, - max_keepalive_connections=RETRIEVE_MAX_CONCURRENCY, - ) - - async def _fan_out() -> tuple[List[Dict[str, Any]], int]: - async with httpx.AsyncClient(timeout=timeout, limits=limits) as client: - batches = await asyncio.gather( - *( - _retrieve_single_dataset_async( - client=client, - limiter=limiter, - dataset_id=target_id, - query=query, - top_k=top_k, - score_threshold=score_threshold, - search_method=search_method, - reranking_enable=reranking_enable, - weights=weights, - base_url=base_url, - headers=headers, - ) - for target_id in target_ids - ) - ) - failed_count = sum(1 for _, succeeded in batches if not succeeded) - records = [item for batch, _ in batches for item in batch] - return records, failed_count - - try: - remaining = deadline - loop.time() - if remaining <= 0: - raise asyncio.TimeoutError - records, failed_count = await asyncio.wait_for( - _fan_out(), - timeout=remaining, - ) - except asyncio.TimeoutError as exc: - raise DatasetRetrievalTimeoutError( - f"公有知识库检索超过 {RETRIEVE_TOTAL_TIMEOUT_SECONDS}s" - ) from exc - - if failed_count == len(target_ids): - raise DatasetRetrievalUnavailableError(f"全部 {len(target_ids)} 个公有知识库请求均失败") - if failed_count: - _logger.warning( - "公有知识库检索部分失败: %d/%d 个数据集不可用", - failed_count, - len(target_ids), - ) - - return _finalize_retrieve_records(records, top_k=top_k) - - -# ── List all datasets ───────────────────────────────────────────────────────── - -_list_logger = logging.getLogger(__name__ + ".list_datasets") - - -def list_all_datasets( - *, - allowed_dataset_ids: str | None = None, - allowed_kb_ids: str | None = None, - current_user_id: str | None = None, -) -> Dict[str, Any]: - """List all available public and private knowledge bases with document names.""" - public_datasets: List[Dict[str, Any]] = [] - private_datasets: List[Dict[str, Any]] = [] - - # ── Public datasets (Dify) ──────────────────────────────────────────────── - try: - from core.kb.dify_kb import is_dify_enabled - from core.kb.dify_kb import list_datasets as dify_list - from core.kb.dify_kb import list_documents as dify_list_docs - - if is_dify_enabled(): - if allowed_dataset_ids is not None: - _allowed_set = {x.strip() for x in allowed_dataset_ids.split(",") if x.strip()} - else: - _allowed_set = get_allowed_dataset_ids() - - datasets = dify_list(page=1, limit=100, timeout=(2, 5)) - - # 权限分配：按当前用户可见的数据集收口（public + 已授权 scoped）。 - _list_user_id = (current_user_id or "").strip() or os.getenv( - "CURRENT_USER_ID", "" - ).strip() - _accessible_ds: set[str] | None = None - if _list_user_id: - try: - from core.db.engine import SessionLocal - - _cand = [str(d.get("id", "")).strip() for d in datasets if d.get("id")] - with SessionLocal() as _db: - _accessible_ds = set(get_dataset_levels(_db, _list_user_id, _cand).keys()) - except Exception as exc: - _list_logger.warning("dataset access resolve failed: %s", exc) - _accessible_ds = set() - - for ds in datasets: - ds_id = str(ds.get("id", "")).strip() - if not ds_id: - continue - if _allowed_set and ds_id not in _allowed_set: - continue - if _accessible_ds is not None and ds_id not in _accessible_ds: - continue - - name = ds.get("name", ds_id) - desc = ds.get("description") or ds.get("desc") or "" - doc_count = ds.get("document_count", 0) - - # Fetch document titles (up to 20) - doc_titles: List[str] = [] - try: - docs_result = dify_list_docs(ds_id, page=1, limit=20) - for doc in docs_result.get("items", []): - title = doc.get("title", "").strip() - if title: - doc_titles.append(title) - except Exception as exc: - _list_logger.debug("Failed to list docs for dataset %s: %s", ds_id, exc) - - public_datasets.append( - { - "dataset_id": ds_id, - "name": name, - "description": desc, - "document_count": doc_count, - "document_titles": doc_titles, - "type": "public", - } - ) - except Exception as exc: - _list_logger.warning("Failed to list public datasets: %s", exc) - - # ── Private datasets (local KB) ─────────────────────────────────────────── +def _list_local_datasets( + *, allowed_kb_ids: str | None, current_user_id: str | None +) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: + shared_items: List[Dict[str, Any]] = [] + private_items: List[Dict[str, Any]] = [] try: from core.db.engine import SessionLocal from core.db.models import KBDocument, KBSpace - if allowed_kb_ids is not None: - allowed = {k.strip() for k in allowed_kb_ids.split(",") if k.strip()} - else: - allowed = set() - + allowed = {item.strip() for item in (allowed_kb_ids or "").split(",") if item.strip()} user_id = (current_user_id or "").strip() or os.getenv("CURRENT_USER_ID", "").strip() - with SessionLocal() as db: query = db.query(KBSpace).filter(KBSpace.deleted_at.is_(None)) if allowed: query = query.filter(KBSpace.kb_id.in_(allowed)) elif user_id: - # 权限分配：用户可见集 = 自己私有库 + 公有库 + 已授权的 scoped 库（单一真源） accessible = get_accessible_local_kb_ids(db, user_id) query = query.filter(KBSpace.kb_id.in_(accessible or {"__none__"})) - spaces = query.all() - - for space in spaces: - # Fetch document titles + for space in query.all(): docs = ( db.query(KBDocument) .filter( @@ -610,358 +58,56 @@ def list_all_datasets( .limit(20) .all() ) - doc_titles = [d.title for d in docs if d.title] - - # 按本地库实际可见性归类：非 private（public/scoped 共享库）→ 公有； - # private → 私有。先前把所有本地库一律塞进 private_datasets，导致公有库 - # 被智能体当私有库汇报——此处按 visibility 归正。两类都经 retrieve_local_kb - # 检索（kb_ 前缀），公有库走其 public 分支、私有库走 owner 隔离分支。 shared = is_shared_visibility(space.visibility) item = { "kb_id": space.kb_id, "name": space.name, "description": space.description or "", "document_count": space.document_count or len(docs), - "document_titles": doc_titles, - "type": "private" if not shared else "public", + "document_titles": [doc.title for doc in docs if doc.title], + "type": "public" if shared else "private", } - (public_datasets if shared else private_datasets).append(item) - except Exception as exc: - _list_logger.warning("Failed to list private KBs: %s", exc) - - return { - "public_datasets": public_datasets, - "private_datasets": private_datasets, - "total": len(public_datasets) + len(private_datasets), - } - - -# ── Private (local) KB retrieval ────────────────────────────────────────────── - -_local_kb_logger = logging.getLogger(__name__ + ".local_kb") - - -def _get_kb_detail_max_chars() -> int: - """Admin-panel managed via knowledge_base.detail_max_chars; resolved DB→env per call.""" - raw = (get_runtime_value("KB_DETAIL_CONTENT_MAX_CHARS") or "50000").strip() - try: - return int(raw) - except ValueError: - return 50000 - - -def _get_allowed_kb_ids() -> set[str]: - raw = os.getenv("LOCAL_KB_ALLOWED_IDS", "").strip() - if not raw: - return set() - return {k.strip() for k in raw.split(",") if k.strip()} - - -def _get_current_user_id() -> str: - return os.getenv("CURRENT_USER_ID", "").strip() - - -def _fetch_parent_contents(parent_ids: list[str]) -> dict[str, str]: - """Fetch parent chunk content from PostgreSQL by chunk_id list.""" - if not parent_ids: - return {} - try: - from core.db.engine import SessionLocal - from core.db.models import KBChunk - - db = SessionLocal() - try: - chunks = db.query(KBChunk).filter(KBChunk.chunk_id.in_(parent_ids)).all() - return {c.chunk_id: c.content for c in chunks} - finally: - db.close() - except Exception as exc: - _local_kb_logger.warning("Failed to fetch parent chunks from DB: %s", exc) - return {} - - -def _build_runtime_local_kb_section() -> str: - """Build runtime private KB list for tool description injection. - - NOTE: 详细的知识库简介和文档列表在系统提示词中动态注入（见 prompt_runtime.py）， - 此处仅提供 kb_id 与名称的快速参考。 - """ - allowed_raw = os.getenv("LOCAL_KB_ALLOWED_IDS", "").strip() - if not allowed_raw: - return "" - - allowed_ids = [k.strip() for k in allowed_raw.split(",") if k.strip()] - if not allowed_ids: - return "" - - # Try to fetch KB names from DB - kb_names: dict[str, str] = {} - try: - from core.db.engine import SessionLocal - from core.db.models import KBSpace - - db = SessionLocal() - try: - spaces = db.query(KBSpace).filter(KBSpace.kb_id.in_(allowed_ids)).all() - kb_names = {s.kb_id: s.name for s in spaces} - finally: - db.close() + (shared_items if shared else private_items).append(item) except Exception as exc: - _local_kb_logger.debug("Could not fetch KB names for tool description: %s", exc) - - lines = [] - for kid in allowed_ids: - name = kb_names.get(kid, kid) - lines.append(f"- {kid} | {name}") - - return "\n".join( - [ - "## 当前可用本地知识库（运行时注入，含公有库与私有库）", - "调用 `retrieve_local_kb` 时，`kb_id` 应从以下列表中选择（详细简介、可见性与文档列表见系统提示词）。", - "注意：本列表混含公有库与私有库，不要据此把其中的库一律当作私有库。", - "格式：`kb_id | 知识库名称`", - *lines, - "## 当前可用本地知识库（运行时注入）结束", - ] - ).strip() + _logger.warning("Failed to list local knowledge bases: %s", exc) + return shared_items, private_items -def retrieve_local_kb( - kb_id: str, - query: str, - top_k: int = 10, +def list_all_datasets( *, + allowed_dataset_ids: str | None = None, allowed_kb_ids: str | None = None, current_user_id: str | None = None, - reranker_enabled: str | None = None, -) -> Any: # 错误分支返回 list[dict]，成功/空命中分支返回 dict —— 异构返回，标注为 Any - """Search user's private KB and return ranked result chunks. - - Returns a list of dicts with keys: id, title, content, kb_id, score. - """ - deadline = time.monotonic() + LOCAL_RETRIEVE_TOTAL_TIMEOUT_SECONDS - - def _remaining_stage_timeout() -> float: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise LocalKnowledgeBaseTimeoutError( - f"私有知识库检索超过 {LOCAL_RETRIEVE_TOTAL_TIMEOUT_SECONDS}s" - ) - return min(float(LOCAL_RETRIEVE_STAGE_TIMEOUT_SECONDS), remaining) - - # ── Auth check ────────────────────────────────────────────────────────── - if allowed_kb_ids is not None: - allowed = {k.strip() for k in allowed_kb_ids.split(",") if k.strip()} - else: - allowed = _get_allowed_kb_ids() - - user_id = current_user_id if current_user_id is not None else _get_current_user_id() - - # Auto-resolve（权限分配单一真源）：未给 allowed 列表时，按当前用户的可见集解析—— - # 自己私有库 + 公有库 + 已授权的 scoped 库。无 user_id 时降级为全部库（仅 stdio/本地调试， - # HTTP 模式必有 X-Current-User-Id）。 - if not allowed: - try: - from core.db.engine import SessionLocal - from core.db.models import KBSpace - - with SessionLocal() as _db: - if user_id: - allowed = get_accessible_local_kb_ids(_db, user_id) - else: - spaces = ( - _db.query(KBSpace.kb_id) - .filter( - KBSpace.deleted_at.is_(None), - ) - .all() - ) - allowed = {s.kb_id for s in spaces if s.kb_id} - _local_kb_logger.info("Auto-resolved %d KB spaces", len(allowed)) - except Exception as exc: - _local_kb_logger.warning("Auto-resolve KB spaces failed: %s", exc) - - if not allowed: - _local_kb_logger.warning("retrieve_local_kb: no accessible private KBs") - return [{"error": "未找到可访问的私有知识库"}] - - kb_id = (kb_id or "").strip() - # Determine which KBs to search - if kb_id: - if kb_id not in allowed: - _local_kb_logger.warning("retrieve_local_kb: kb_id %s not in allowed list", kb_id) - return [{"error": f"无权访问知识库 {kb_id}"}] - search_kb_ids = [kb_id] - else: - # Search ALL allowed KBs - search_kb_ids = sorted(allowed) - _local_kb_logger.info("Searching all %d allowed KBs: %s", len(search_kb_ids), search_kb_ids) - - # Classify search targets into private (owner-isolated) vs public (global) KBs and, - # when headers didn't supply a user_id, resolve the owner of a private space — all in - # one query. Public KBs are admin-managed (visibility=="public") and searched by kb_id. - public_ids: list[str] = [] - private_ids: list[str] = list(search_kb_ids) - try: - from core.db.engine import SessionLocal - from core.db.models import KBSpace - - with SessionLocal() as _db: - rows = ( - _db.query(KBSpace.kb_id, KBSpace.visibility, KBSpace.user_id) - .filter( - KBSpace.kb_id.in_(search_kb_ids), - KBSpace.deleted_at.is_(None), - ) - .all() - ) - # public 与 scoped 都是「共享库」：按 kb_id 全局检索（向量行归属系统属主， - # 不能再叠加 user_id 过滤，否则被授权用户搜不到）。仅 private 才 owner 隔离。 - shared_set = {r.kb_id for r in rows if is_shared_visibility(r.visibility)} - public_ids = [k for k in search_kb_ids if k in shared_set] - private_ids = [k for k in search_kb_ids if k not in shared_set] - if private_ids and not user_id: - user_id = next((r.user_id for r in rows if r.kb_id in private_ids), user_id) - except Exception as exc: - _local_kb_logger.warning("retrieve_local_kb: visibility classification failed: %s", exc) - - if private_ids and not user_id: - return [{"error": "未能获取当前用户 ID"}] - - # ── Embed query ────────────────────────────────────────────────────────── - try: - from core.kb.kb_vector import embed_text, hybrid_search - - query_vec = embed_text(query, timeout=_remaining_stage_timeout()) - except LocalKnowledgeBaseTimeoutError: - raise - except Exception as exc: - _local_kb_logger.error("retrieve_local_kb: embed_text failed: %s", exc) - return [{"error": f"向量化失败：{exc}"}] - - # ── Hybrid search ──────────────────────────────────────────────────────── - try: - hits = hybrid_search( - user_id=user_id or "", - kb_ids=private_ids, - query=query, - query_vec=query_vec, - top_k=top_k * 3, # over-fetch before dedup - public_kb_ids=public_ids, - timeout=_remaining_stage_timeout(), - ) - except LocalKnowledgeBaseTimeoutError: - raise - except Exception as exc: - _local_kb_logger.error("retrieve_local_kb: hybrid_search failed: %s", exc) - return [{"error": f"检索失败：{exc}"}] - - # Build KB metadata for the response - kb_meta: list[dict[str, str]] = [] - try: - from core.db.engine import SessionLocal - from core.db.models import KBSpace - - with SessionLocal() as _db: - spaces = ( - _db.query(KBSpace) - .filter( - KBSpace.kb_id.in_(search_kb_ids), - ) - .all() - ) - kb_meta = [ - {"kb_id": s.kb_id, "name": s.name, "description": s.description or ""} - for s in spaces - ] - except Exception: - pass - - if not hits: - return {"available_kbs": kb_meta, "items": [], "message": "未找到相关内容"} - - # ── Dedup by parent_chunk_id (keep highest score) ──────────────────────── - seen: dict[str, dict] = {} - for hit in hits: - pid = hit.get("parent_chunk_id") or hit.get("chunk_id", "") - if pid not in seen or hit["score"] > seen[pid]["score"]: - seen[pid] = hit - - # Sort by score descending, take top_k - top_hits = sorted(seen.values(), key=lambda x: x["score"], reverse=True)[:top_k] - - # ── Optional reranker step ─────────────────────────────────────────────── - _reranker_flag = ( - reranker_enabled if reranker_enabled is not None else os.getenv("RERANKER_ENABLED", "") +) -> Dict[str, Any]: + public_items = list_external_datasets( + allowed_dataset_ids=allowed_dataset_ids, + allowed_kb_ids=allowed_kb_ids, + current_user_id=current_user_id, ) - if (_reranker_flag or "").lower() in ("true", "1"): - try: - from core.kb.kb_vector import is_reranker_configured, rerank - - if is_reranker_configured() and top_hits: - contents = [hit.get("content", "") for hit in top_hits] - reranked = rerank( - query, - contents, - top_n=top_k, - timeout=_remaining_stage_timeout(), - ) - reranked_hits = [] - for item in reranked: - idx = item.get("index", 0) - if 0 <= idx < len(top_hits): - hit = dict(top_hits[idx]) - hit["score"] = round(item.get("relevance_score", hit["score"]), 4) - reranked_hits.append(hit) - if reranked_hits: - top_hits = reranked_hits - _local_kb_logger.info("Reranker applied: %d results reranked", len(top_hits)) - except LocalKnowledgeBaseTimeoutError: - raise - except Exception as rerank_exc: - _local_kb_logger.warning( - "Reranker failed, falling back to original ranking: %s", rerank_exc - ) - - # ── Fetch parent content from PostgreSQL ───────────────────────────────── - _remaining_stage_timeout() - parent_ids = [h["parent_chunk_id"] for h in top_hits if h.get("parent_chunk_id")] - parent_map = _fetch_parent_contents(parent_ids) - - # ── Build results ──────────────────────────────────────────────────────── - results = [] - total_chars = 0 - kb_detail_max_chars = _get_kb_detail_max_chars() - for i, hit in enumerate(top_hits): - pid = hit.get("parent_chunk_id") or hit.get("chunk_id", "") - # Prefer parent content (full context); fall back to child snippet - content = parent_map.get(pid) or hit.get("content", "") - - if total_chars + len(content) > kb_detail_max_chars: - content = content[: max(0, kb_detail_max_chars - total_chars)] - if content: - results.append( - { - "id": pid, - "title": hit.get("title", ""), - "content": content, - "kb_id": hit.get("kb_id", kb_id), - "score": round(hit["score"], 4), - "chunk_index": hit.get("chunk_index", i), - } - ) - break + local_shared, private_items = _list_local_datasets( + allowed_kb_ids=allowed_kb_ids, current_user_id=current_user_id + ) + public_items.extend(local_shared) + return { + "public_datasets": public_items, + "private_datasets": private_items, + "total": len(public_items) + len(private_items), + } - total_chars += len(content) - results.append( - { - "id": pid, - "title": hit.get("title", ""), - "content": content, - "kb_id": hit.get("kb_id", kb_id), - "score": round(hit["score"], 4), - "chunk_index": hit.get("chunk_index", i), - } - ) - return {"available_kbs": kb_meta, "items": results} +__all__ = [ + "DatasetRetrievalTimeoutError", + "DatasetRetrievalUnavailableError", + "LOCAL_RETRIEVE_STAGE_TIMEOUT_SECONDS", + "LOCAL_RETRIEVE_TOTAL_TIMEOUT_SECONDS", + "LocalKnowledgeBaseTimeoutError", + "MAX_RETRIEVE_TOKENS", + "RETRIEVE_MAX_CONCURRENCY", + "RETRIEVE_REQUEST_TIMEOUT_SECONDS", + "RETRIEVE_TOTAL_TIMEOUT_SECONDS", + "_build_runtime_local_kb_section", + "list_all_datasets", + "retrieve_dataset_content", + "retrieve_dataset_content_async", + "retrieve_local_kb", +] diff --git a/src/backend/mcp_servers/retrieve_dataset_content_mcp/local_impl.py b/src/backend/mcp_servers/retrieve_dataset_content_mcp/local_impl.py new file mode 100644 index 00000000..8fc158cc --- /dev/null +++ b/src/backend/mcp_servers/retrieve_dataset_content_mcp/local_impl.py @@ -0,0 +1,359 @@ +"""Local knowledge-base retrieval shared by all editions.""" + +from __future__ import annotations + +import logging +import os +import time +from typing import Any + +from core.auth.kb_permissions import get_accessible_local_kb_ids, is_shared_visibility +from core.config.runtime_env import get_runtime_value + + +def _read_int_env(name: str, default: int) -> int: + raw = (os.getenv(name) or "").strip() + if not raw: + return default + try: + value = int(raw) + return value if value > 0 else default + except ValueError: + return default + + +LOCAL_RETRIEVE_TOTAL_TIMEOUT_SECONDS = _read_int_env("RETRIEVE_LOCAL_KB_TIMEOUT_SECONDS", 30) +LOCAL_RETRIEVE_STAGE_TIMEOUT_SECONDS = _read_int_env("RETRIEVE_LOCAL_KB_STAGE_TIMEOUT_SECONDS", 10) + + +class LocalKnowledgeBaseTimeoutError(TimeoutError): + """Raised when local knowledge-base work exhausts its internal deadline.""" + + +_local_kb_logger = logging.getLogger(__name__ + ".local_kb") + + +def _get_kb_detail_max_chars() -> int: + """Admin-panel managed via knowledge_base.detail_max_chars; resolved DB→env per call.""" + raw = (get_runtime_value("KB_DETAIL_CONTENT_MAX_CHARS") or "50000").strip() + try: + return int(raw) + except ValueError: + return 50000 + + +def _get_allowed_kb_ids() -> set[str]: + raw = os.getenv("LOCAL_KB_ALLOWED_IDS", "").strip() + if not raw: + return set() + return {k.strip() for k in raw.split(",") if k.strip()} + + +def _get_current_user_id() -> str: + return os.getenv("CURRENT_USER_ID", "").strip() + + +def _fetch_parent_contents(parent_ids: list[str]) -> dict[str, str]: + """Fetch parent chunk content from PostgreSQL by chunk_id list.""" + if not parent_ids: + return {} + try: + from core.db.engine import SessionLocal + from core.db.models import KBChunk + + db = SessionLocal() + try: + chunks = db.query(KBChunk).filter(KBChunk.chunk_id.in_(parent_ids)).all() + return {c.chunk_id: c.content for c in chunks} + finally: + db.close() + except Exception as exc: + _local_kb_logger.warning("Failed to fetch parent chunks from DB: %s", exc) + return {} + + +def _build_runtime_local_kb_section() -> str: + """Build runtime private KB list for tool description injection. + + NOTE: 详细的知识库简介和文档列表在系统提示词中动态注入（见 prompt_runtime.py）， + 此处仅提供 kb_id 与名称的快速参考。 + """ + allowed_raw = os.getenv("LOCAL_KB_ALLOWED_IDS", "").strip() + if not allowed_raw: + return "" + + allowed_ids = [k.strip() for k in allowed_raw.split(",") if k.strip()] + if not allowed_ids: + return "" + + # Try to fetch KB names from DB + kb_names: dict[str, str] = {} + try: + from core.db.engine import SessionLocal + from core.db.models import KBSpace + + db = SessionLocal() + try: + spaces = db.query(KBSpace).filter(KBSpace.kb_id.in_(allowed_ids)).all() + kb_names = {s.kb_id: s.name for s in spaces} + finally: + db.close() + except Exception as exc: + _local_kb_logger.debug("Could not fetch KB names for tool description: %s", exc) + + lines = [] + for kid in allowed_ids: + name = kb_names.get(kid, kid) + lines.append(f"- {kid} | {name}") + + return "\n".join( + [ + "## 当前可用本地知识库（运行时注入，含公有库与私有库）", + "调用 `retrieve_local_kb` 时，`kb_id` 应从以下列表中选择（详细简介、可见性与文档列表见系统提示词）。", + "注意：本列表混含公有库与私有库，不要据此把其中的库一律当作私有库。", + "格式：`kb_id | 知识库名称`", + *lines, + "## 当前可用本地知识库（运行时注入）结束", + ] + ).strip() + + +def retrieve_local_kb( + kb_id: str, + query: str, + top_k: int = 10, + *, + allowed_kb_ids: str | None = None, + current_user_id: str | None = None, + reranker_enabled: str | None = None, +) -> Any: # 错误分支返回 list[dict]，成功/空命中分支返回 dict —— 异构返回，标注为 Any + """Search user's private KB and return ranked result chunks. + + Returns a list of dicts with keys: id, title, content, kb_id, score. + """ + deadline = time.monotonic() + LOCAL_RETRIEVE_TOTAL_TIMEOUT_SECONDS + + def _remaining_stage_timeout() -> float: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise LocalKnowledgeBaseTimeoutError( + f"私有知识库检索超过 {LOCAL_RETRIEVE_TOTAL_TIMEOUT_SECONDS}s" + ) + return min(float(LOCAL_RETRIEVE_STAGE_TIMEOUT_SECONDS), remaining) + + # ── Auth check ────────────────────────────────────────────────────────── + if allowed_kb_ids is not None: + allowed = {k.strip() for k in allowed_kb_ids.split(",") if k.strip()} + else: + allowed = _get_allowed_kb_ids() + + user_id = current_user_id if current_user_id is not None else _get_current_user_id() + + # Auto-resolve（权限分配单一真源）：未给 allowed 列表时，按当前用户的可见集解析—— + # 自己私有库 + 公有库 + 已授权的 scoped 库。无 user_id 时降级为全部库（仅 stdio/本地调试， + # HTTP 模式必有 X-Current-User-Id）。 + if not allowed: + try: + from core.db.engine import SessionLocal + from core.db.models import KBSpace + + with SessionLocal() as _db: + if user_id: + allowed = get_accessible_local_kb_ids(_db, user_id) + else: + spaces = ( + _db.query(KBSpace.kb_id) + .filter( + KBSpace.deleted_at.is_(None), + ) + .all() + ) + allowed = {s.kb_id for s in spaces if s.kb_id} + _local_kb_logger.info("Auto-resolved %d KB spaces", len(allowed)) + except Exception as exc: + _local_kb_logger.warning("Auto-resolve KB spaces failed: %s", exc) + + if not allowed: + _local_kb_logger.warning("retrieve_local_kb: no accessible private KBs") + return [{"error": "未找到可访问的私有知识库"}] + + kb_id = (kb_id or "").strip() + # Determine which KBs to search + if kb_id: + if kb_id not in allowed: + _local_kb_logger.warning("retrieve_local_kb: kb_id %s not in allowed list", kb_id) + return [{"error": f"无权访问知识库 {kb_id}"}] + search_kb_ids = [kb_id] + else: + # Search ALL allowed KBs + search_kb_ids = sorted(allowed) + _local_kb_logger.info("Searching all %d allowed KBs: %s", len(search_kb_ids), search_kb_ids) + + # Classify search targets into private (owner-isolated) vs public (global) KBs and, + # when headers didn't supply a user_id, resolve the owner of a private space — all in + # one query. Public KBs are admin-managed (visibility=="public") and searched by kb_id. + public_ids: list[str] = [] + private_ids: list[str] = list(search_kb_ids) + try: + from core.db.engine import SessionLocal + from core.db.models import KBSpace + + with SessionLocal() as _db: + rows = ( + _db.query(KBSpace.kb_id, KBSpace.visibility, KBSpace.user_id) + .filter( + KBSpace.kb_id.in_(search_kb_ids), + KBSpace.deleted_at.is_(None), + ) + .all() + ) + # public 与 scoped 都是「共享库」：按 kb_id 全局检索（向量行归属系统属主， + # 不能再叠加 user_id 过滤，否则被授权用户搜不到）。仅 private 才 owner 隔离。 + shared_set = {r.kb_id for r in rows if is_shared_visibility(r.visibility)} + public_ids = [k for k in search_kb_ids if k in shared_set] + private_ids = [k for k in search_kb_ids if k not in shared_set] + if private_ids and not user_id: + user_id = next((r.user_id for r in rows if r.kb_id in private_ids), user_id) + except Exception as exc: + _local_kb_logger.warning("retrieve_local_kb: visibility classification failed: %s", exc) + + if private_ids and not user_id: + return [{"error": "未能获取当前用户 ID"}] + + # ── Embed query ────────────────────────────────────────────────────────── + try: + from core.kb.kb_vector import embed_text, hybrid_search + + query_vec = embed_text(query, timeout=_remaining_stage_timeout()) + except LocalKnowledgeBaseTimeoutError: + raise + except Exception as exc: + _local_kb_logger.error("retrieve_local_kb: embed_text failed: %s", exc) + return [{"error": f"向量化失败：{exc}"}] + + # ── Hybrid search ──────────────────────────────────────────────────────── + try: + hits = hybrid_search( + user_id=user_id or "", + kb_ids=private_ids, + query=query, + query_vec=query_vec, + top_k=top_k * 3, # over-fetch before dedup + public_kb_ids=public_ids, + timeout=_remaining_stage_timeout(), + ) + except LocalKnowledgeBaseTimeoutError: + raise + except Exception as exc: + _local_kb_logger.error("retrieve_local_kb: hybrid_search failed: %s", exc) + return [{"error": f"检索失败：{exc}"}] + + # Build KB metadata for the response + kb_meta: list[dict[str, str]] = [] + try: + from core.db.engine import SessionLocal + from core.db.models import KBSpace + + with SessionLocal() as _db: + spaces = ( + _db.query(KBSpace) + .filter( + KBSpace.kb_id.in_(search_kb_ids), + ) + .all() + ) + kb_meta = [ + {"kb_id": s.kb_id, "name": s.name, "description": s.description or ""} + for s in spaces + ] + except Exception: + pass + + if not hits: + return {"available_kbs": kb_meta, "items": [], "message": "未找到相关内容"} + + # ── Dedup by parent_chunk_id (keep highest score) ──────────────────────── + seen: dict[str, dict] = {} + for hit in hits: + pid = hit.get("parent_chunk_id") or hit.get("chunk_id", "") + if pid not in seen or hit["score"] > seen[pid]["score"]: + seen[pid] = hit + + # Sort by score descending, take top_k + top_hits = sorted(seen.values(), key=lambda x: x["score"], reverse=True)[:top_k] + + # ── Optional reranker step ─────────────────────────────────────────────── + _reranker_flag = ( + reranker_enabled if reranker_enabled is not None else os.getenv("RERANKER_ENABLED", "") + ) + if (_reranker_flag or "").lower() in ("true", "1"): + try: + from core.kb.kb_vector import is_reranker_configured, rerank + + if is_reranker_configured() and top_hits: + contents = [hit.get("content", "") for hit in top_hits] + reranked = rerank( + query, + contents, + top_n=top_k, + timeout=_remaining_stage_timeout(), + ) + reranked_hits = [] + for item in reranked: + idx = item.get("index", 0) + if 0 <= idx < len(top_hits): + hit = dict(top_hits[idx]) + hit["score"] = round(item.get("relevance_score", hit["score"]), 4) + reranked_hits.append(hit) + if reranked_hits: + top_hits = reranked_hits + _local_kb_logger.info("Reranker applied: %d results reranked", len(top_hits)) + except LocalKnowledgeBaseTimeoutError: + raise + except Exception as rerank_exc: + _local_kb_logger.warning( + "Reranker failed, falling back to original ranking: %s", rerank_exc + ) + + # ── Fetch parent content from PostgreSQL ───────────────────────────────── + _remaining_stage_timeout() + parent_ids = [h["parent_chunk_id"] for h in top_hits if h.get("parent_chunk_id")] + parent_map = _fetch_parent_contents(parent_ids) + + # ── Build results ──────────────────────────────────────────────────────── + results = [] + total_chars = 0 + kb_detail_max_chars = _get_kb_detail_max_chars() + for i, hit in enumerate(top_hits): + pid = hit.get("parent_chunk_id") or hit.get("chunk_id", "") + # Prefer parent content (full context); fall back to child snippet + content = parent_map.get(pid) or hit.get("content", "") + + if total_chars + len(content) > kb_detail_max_chars: + content = content[: max(0, kb_detail_max_chars - total_chars)] + if content: + results.append( + { + "id": pid, + "title": hit.get("title", ""), + "content": content, + "kb_id": hit.get("kb_id", kb_id), + "score": round(hit["score"], 4), + "chunk_index": hit.get("chunk_index", i), + } + ) + break + + total_chars += len(content) + results.append( + { + "id": pid, + "title": hit.get("title", ""), + "content": content, + "kb_id": hit.get("kb_id", kb_id), + "score": round(hit["score"], 4), + "chunk_index": hit.get("chunk_index", i), + } + ) + + return {"available_kbs": kb_meta, "items": results} diff --git a/src/backend/mcp_servers/retrieve_dataset_content_mcp/server.py b/src/backend/mcp_servers/retrieve_dataset_content_mcp/server.py index 80dbedb5..cb903af6 100755 --- a/src/backend/mcp_servers/retrieve_dataset_content_mcp/server.py +++ b/src/backend/mcp_servers/retrieve_dataset_content_mcp/server.py @@ -273,7 +273,7 @@ async def retrieve_dataset_content( Returns: dict: {"public_datasets": [...], "private_datasets": [...], "total": N} - - public_datasets：公有/共享知识库（含 Dify 外接数据集与本地公有库）。带 dataset_id 的用 + - public_datasets：公有/共享知识库（含外接数据集与本地公有库）。带 dataset_id 的用 retrieve_dataset_content 检索；带 kb_id 的（本地公有库）用 retrieve_local_kb 检索。 - private_datasets：仅当前用户自己的私有库（kb_id），用 retrieve_local_kb 检索。 用户问"有几个公有知识库 / 公有库列表"时以 public_datasets 为准，不要把本地公有库当私有库。 diff --git a/src/backend/mcp_servers/site_publish_mcp/impl.py b/src/backend/mcp_servers/site_publish_mcp/impl.py index fa29f4d4..0e7d756f 100644 --- a/src/backend/mcp_servers/site_publish_mcp/impl.py +++ b/src/backend/mcp_servers/site_publish_mcp/impl.py @@ -1,9 +1,4 @@ -"""site_publish MCP —— 业务实现（转发到 backend 内部发布接口）。 - -mcp 容器没有沙箱访问权，所以本工具不直接碰沙箱：把发布请求连同用户/会话身份 -一起 POST 到 backend 的 ``/v1/internal/sites/publish``（backend 有沙箱），由它完成 -"打包沙箱目录 → 取回 → 解包 → 落库托管"。鉴权走共享密钥 ``X-Internal-Token``。 -""" +"""Community-edition site publishing MCP implementation.""" from __future__ import annotations @@ -36,13 +31,9 @@ async def publish_site( site_id: str = "", visibility: str = "public", description: str = "", - team_id: str = "", ) -> Dict[str, Any]: if not user_id: return {"error": "当前会话缺少用户身份，无法发布站点"} - # src_dir 可留空：项目模式下 backend 会自动定位到会话绑定的项目文件夹。 - # 非项目会话且缺少 chat_id 又没传 src_dir 时，backend 会返回可读错误。 - # source_dir（构建型站点的源码工程目录）原样透传，语义见 backend 接口。 payload = { "src_dir": src_dir, @@ -52,7 +43,6 @@ async def publish_site( "site_id": site_id, "visibility": visibility, "description": description, - "team_id": team_id, "user_id": user_id, "chat_id": chat_id, } @@ -61,27 +51,22 @@ async def publish_site( if token: headers["X-Internal-Token"] = token - url = f"{_backend_url()}/v1/internal/sites/publish" try: async with httpx.AsyncClient(timeout=120.0) as client: - resp = await client.post(url, json=payload, headers=headers) + response = await client.post( + f"{_backend_url()}/v1/internal/sites/publish", + json=payload, + headers=headers, + ) except httpx.HTTPError as exc: return {"error": f"发布请求失败（无法连到后端）: {exc}"} - if resp.status_code != 200: - body = "" - try: - body = resp.text[:300] - except Exception: # noqa: BLE001 - pass - return {"error": f"发布接口返回 {resp.status_code}: {body}"} - + if response.status_code != 200: + return {"error": f"发布接口返回 {response.status_code}: {response.text[:300]}"} try: - envelope = resp.json() - except Exception as exc: # noqa: BLE001 + envelope = response.json() + except Exception as exc: return {"error": f"发布接口返回非 JSON: {exc}"} - - # 后端用统一信封 {code,message,data,...}，真实结果在 data data = envelope.get("data") if isinstance(envelope, dict) else None if isinstance(data, dict): return data diff --git a/src/backend/mcp_servers/site_publish_mcp/server.py b/src/backend/mcp_servers/site_publish_mcp/server.py index 071b371a..610368dc 100644 --- a/src/backend/mcp_servers/site_publish_mcp/server.py +++ b/src/backend/mcp_servers/site_publish_mcp/server.py @@ -1,22 +1,11 @@ #!/usr/bin/env python3 -"""streamable-http MCP server：对话建站发布（把沙箱静态站目录发布为平台托管站点）。 - -身份 / 会话经 HTTP 头注入（由后端 agent_factory 对所有 MCP 统一设置）： - X-Current-User-Id 当前用户（站点归属，缺失则拒绝） - X-Chat-Id 当前会话 id（web 主对话的沙箱会话键，取回站点目录） - X-Conversation-Id 外部渠道会话 id（钉钉等才有值；作 X-Chat-Id 的兜底） - -本工具是「站点」插件的核心能力，替代早期的内置 publish_site 原生工具。发布动作需要 -沙箱访问权（在 backend 侧），所以本 server 只做转发：把参数 + 身份 POST 到 backend 的 -``/v1/internal/sites/publish`` 完成。见 api/routes/v1/internal_sites.py。 -""" +"""Community-edition site publishing MCP server.""" from __future__ import annotations from typing import Any, Dict, Optional from mcp.server.fastmcp import Context, FastMCP - from mcp_servers.site_publish_mcp import impl mcp = FastMCP("hugagent-site-publish") @@ -30,8 +19,8 @@ def _hdr(ctx: Optional[Context], name: str) -> Optional[str]: if ctx is None: return None try: - v = ctx.request_context.request.headers.get(name) - return v or None + value = ctx.request_context.request.headers.get(name) + return value or None except Exception: return None @@ -45,64 +34,13 @@ async def publish_site( site_id: str = "", visibility: str = "public", description: str = "", - team_id: str = "", ctx: Context | None = None, ) -> Dict[str, Any]: - """把当前站点项目（或指定沙箱目录）发布为平台托管站点，返回可直接访问的 URL。 - - 适用场景：用户要求「做一个网站/页面/门户/展示站/H5 并能访问」。 - 发布分两条路径，按站点形态选择： - - ① 静态站（手写 HTML/CSS/JS）： - 1) 用 write/bash 在**当前项目文件夹**里生成完整静态站 - （必须有 index.html 入口；可以有子目录、css/js/图片）； - 2) publish_site(title='站点名') → 返回 url（src_dir 留空，后端自动取项目文件夹）； - 3) 把 url 以 markdown 链接交付给用户（形如 /site//）。 - - ② 构建型站点（React/Vite 工程，沙箱内 npm run build 后发布产物）： - 1) 按 site-builder 技能的 React 流程建工程、构建出静态产物； - 2) publish_site(title='站点名', - src_dir='/workspace/.site-dist/<工程名>', ← 构建产物目录 - source_dir='<源码工程目录>') ← 两个参数都必传 - 产物进站点托管，**源码工程**镜像进项目文件夹（保证「编辑」进来的是 - 可改的源码而不是编译产物）；漏传 source_dir 会导致项目里的源码被产物覆盖。 - - 编辑已发布站点：从站点卡片「编辑」进入的会话里，项目文件夹已带回源码文件。 - 静态站直接改完调 publish_site(title='站点名')；构建型工程（项目里有 package.json） - 改源码 → 重新构建 → 仍按 ② 带双参数发布。均无需 site_id，后端按项目自动定位。 - 非项目会话（罕见）：显式传 src_dir 指定沙箱站点根目录；更新已有站点带 site_id。 - - 限制：静态文件 + 平台内置轻后端 API；≤300 文件、总量 ≤30MB； - 外部 CDN 资源在内网环境可能不可达，样式/脚本尽量内联或本地化。 - 公开站点在浏览器里以沙箱模式运行（无 cookie/localStorage）， - 不要生成依赖登录态或本地存储的逻辑；需要持久化用下面的站点 API。 - - 站点内置轻后端 API（站内 JS 用相对路径 fetch，已配好 CORS）： - - KV 存储（计数器/游戏分数/配置）： - 读 GET __api/kv/ → {value, exists} - 写 PUT __api/kv/ body: {"value": "..."}（≤4KB，≤200 键） - - 表单收集（留言/报名/反馈，站主可在站点管理里导出 CSV）： - POST __api/forms/ body: 扁平 JSON 对象（≤8KB） - 注意：__api/ 是保留前缀，站点文件不能用这个目录名； - fetch 相对路径必须不带前导 /（如 fetch('__api/kv/score')）。 - - Args: - title (`str`): 站点标题（显示在用户的站点管理列表）。 - src_dir (`str`, 可选): 沙盒里要发布的站点内容根目录（在 /workspace/ 下）。 - 静态站的项目会话留空即可（后端自动定位项目文件夹）；构建型站点必须 - 指向构建产物目录（如 /workspace/.site-dist/<工程名>）。 - source_dir (`str`, 可选): 构建型站点的**源码工程目录**。传入后发布产物进 - 托管、源码镜像进项目文件夹。静态站不传。 - slug (`str`, 可选): 自定义访问路径（3-50 位小写字母/数字/连字符）。不传则自动生成。 - site_id (`str`, 可选): 更新已有站点时传（首次发布返回值里有）。 - visibility (`str`, 可选): public=任何人凭链接访问（默认）； - private=仅用户本人登录后可见；team=指定团队成员可见。 - description (`str`, 可选): 一句话站点描述。 - team_id (`str`, 可选): visibility=team 时的授权团队；不传则用用户所在的第一个团队。 + """Publish a site from the current sandbox and return its hosted URL. - Returns: - JSON: {ok, site_id, slug, url, title, visibility, version, file_count, - total_size_bytes, mirrored_from} 或 {error: '...'}。 + ``visibility`` accepts ``public`` or ``private``. Static sites may omit + ``src_dir`` in a project chat. Build-based sites pass the build output as + ``src_dir`` and the editable source folder as ``source_dir``. """ return await impl.publish_site( user_id=_hdr(ctx, _HDR_USER) or "", @@ -114,7 +52,6 @@ async def publish_site( site_id=site_id, visibility=visibility, description=description, - team_id=team_id, ) diff --git a/src/backend/orchestration/chat_run_executor.py b/src/backend/orchestration/chat_run_executor.py index c34561aa..82003ea4 100644 --- a/src/backend/orchestration/chat_run_executor.py +++ b/src/backend/orchestration/chat_run_executor.py @@ -613,10 +613,7 @@ async def _emit(event: Dict[str, Any]) -> None: extra_data=_persist_extra, ) # Build a ProjectScope from the workflow context and pass it - # explicitly: in a team project, without this line → - # scope=None → pinned files are treated as a non-project chat - # and written as orphan rows with user_folder_id=NULL/ - # team_id=NULL, leaking into the personal MySpace root. + # explicitly so pinned files keep their project ownership. from core.services.project_scope import project_scope_from_context _persist_artifacts( diff --git a/src/backend/orchestration/citations.py b/src/backend/orchestration/citations.py index d330d9c1..2e3d82ca 100644 --- a/src/backend/orchestration/citations.py +++ b/src/backend/orchestration/citations.py @@ -15,13 +15,15 @@ @dataclass class CitationItem: - id: str # e.g. "internet_search-1" + id: str # e.g. "internet_search-1" tool_name: str tool_id: Optional[str] title: str url: str snippet: str - source_type: str # internet | knowledge_base | database | industry_news | ai_news | chain_info | unknown + source_type: ( + str # internet | knowledge_base | database | industry_news | ai_news | chain_info | unknown + ) def to_dict(self) -> Dict[str, Any]: return asdict(self) @@ -80,9 +82,12 @@ def extract_citations( if tool_name == "get_chain_information": return _chain_info(tool_id, source_type, result) if tool_name in { - "search_company", "get_company_base_info", - "get_company_business_analysis", "get_company_tech_insight", - "get_company_funding", "get_company_risk_warning", + "search_company", + "get_company_base_info", + "get_company_business_analysis", + "get_company_tech_insight", + "get_company_funding", + "get_company_risk_warning", }: return _company_profile(tool_name, tool_id, source_type, result) except Exception: @@ -143,15 +148,17 @@ def _internet_search(tool_id: Optional[str], source_type: str, data: dict) -> Li title = str(item.get("title") or item.get("url") or "互联网搜索结果")[:120] url = str(item.get("url", "")) snippet = str(item.get("content") or item.get("snippet") or "")[:300] - out.append(CitationItem( - id=f"internet_search-{i}", - tool_name="internet_search", - tool_id=tool_id, - title=title, - url=url, - snippet=snippet, - source_type=source_type, - )) + out.append( + CitationItem( + id=f"internet_search-{i}", + tool_name="internet_search", + tool_id=tool_id, + title=title, + url=url, + snippet=snippet, + source_type=source_type, + ) + ) return out @@ -161,28 +168,26 @@ def _dataset_content(tool_id: Optional[str], source_type: str, data: dict) -> Li for i, item in enumerate(items, 1): if not isinstance(item, dict): continue - # Support both the cleaned format (文件名称/文件内容 keys) and the raw Dify format (document/segment) + # Support both the normalized format and generic external-provider records. doc = item.get("document") or {} seg = item.get("segment") or {} - title = str( - item.get("文件名称") - or doc.get("name") or doc.get("title") - or "知识库文档" - )[:120] - snippet = str( - item.get("文件内容") - or seg.get("content") or item.get("content") - or "" - )[:3000] - out.append(CitationItem( - id=f"retrieve_dataset_content-{i}", - tool_name="retrieve_dataset_content", - tool_id=tool_id, - title=title, - url="", - snippet=snippet, - source_type=source_type, - )) + title = str(item.get("文件名称") or doc.get("name") or doc.get("title") or "知识库文档")[ + :120 + ] + snippet = str(item.get("文件内容") or seg.get("content") or item.get("content") or "")[ + :3000 + ] + out.append( + CitationItem( + id=f"retrieve_dataset_content-{i}", + tool_name="retrieve_dataset_content", + tool_id=tool_id, + title=title, + url="", + snippet=snippet, + source_type=source_type, + ) + ) return out @@ -196,30 +201,34 @@ def _local_kb(tool_id: Optional[str], source_type: str, data: dict) -> List[Cita continue title = str(item.get("title") or "私有知识库文档")[:120] snippet = str(item.get("content") or "")[:3000] - out.append(CitationItem( - id=f"retrieve_local_kb-{i}", - tool_name="retrieve_local_kb", - tool_id=tool_id, - title=title, - url="", - snippet=snippet, - source_type=source_type, - )) + out.append( + CitationItem( + id=f"retrieve_local_kb-{i}", + tool_name="retrieve_local_kb", + tool_id=tool_id, + title=title, + url="", + snippet=snippet, + source_type=source_type, + ) + ) return out def _database(tool_id: Optional[str], source_type: str, data: dict) -> List[CitationItem]: res = data.get("result", data) snippet = str(res) if not isinstance(res, str) else res - return [CitationItem( - id="query_database-1", - tool_name="query_database", - tool_id=tool_id, - title="数据库查询结果", - url="", - snippet=snippet[:3000], - source_type=source_type, - )] + return [ + CitationItem( + id="query_database-1", + tool_name="query_database", + tool_id=tool_id, + title="数据库查询结果", + url="", + snippet=snippet[:3000], + source_type=source_type, + ) + ] def _news( @@ -238,28 +247,32 @@ def _news( summary = str(item.get("摘要") or item.get("summary") or "") time_str = str(item.get("时间") or "") snippet = (f"[{time_str}] {summary}" if time_str else summary)[:3000] - out.append(CitationItem( - id=f"{tool_name}-{i}", - tool_name=tool_name, - tool_id=tool_id, - title=title, - url=url, - snippet=snippet, - source_type=source_type, - )) + out.append( + CitationItem( + id=f"{tool_name}-{i}", + tool_name=tool_name, + tool_id=tool_id, + title=title, + url=url, + snippet=snippet, + source_type=source_type, + ) + ) return out def _chain_info(tool_id: Optional[str], source_type: str, data: dict) -> List[CitationItem]: - return [CitationItem( - id="get_chain_information-1", - tool_name="get_chain_information", - tool_id=tool_id, - title="产业链分析报告", - url="", - snippet="产业链深度全景分析数据", - source_type=source_type, - )] + return [ + CitationItem( + id="get_chain_information-1", + tool_name="get_chain_information", + tool_id=tool_id, + title="产业链分析报告", + url="", + snippet="产业链深度全景分析数据", + source_type=source_type, + ) + ] _COMPANY_TOOL_TITLES: Dict[str, str] = { @@ -291,26 +304,30 @@ def _company_profile( item.get("企业状态", ""), ] snippet = " · ".join(str(p) for p in snippet_parts if p)[:300] - out.append(CitationItem( - id=f"search_company-{i}", - tool_name=tool_name, - tool_id=tool_id, - title=title, - url="", - snippet=snippet, - source_type=source_type, - )) + out.append( + CitationItem( + id=f"search_company-{i}", + tool_name=tool_name, + tool_id=tool_id, + title=title, + url="", + snippet=snippet, + source_type=source_type, + ) + ) return out # Other 5 tools: single citation title = _COMPANY_TOOL_TITLES.get(tool_name, "企业画像") snippet = json.dumps(data, ensure_ascii=False)[:500] if data else "" - return [CitationItem( - id=f"{tool_name}-1", - tool_name=tool_name, - tool_id=tool_id, - title=title, - url="", - snippet=snippet, - source_type=source_type, - )] + return [ + CitationItem( + id=f"{tool_name}-1", + tool_name=tool_name, + tool_id=tool_id, + title=title, + url="", + snippet=snippet, + source_type=source_type, + ) + ] diff --git a/src/backend/orchestration/local_subprocess.py b/src/backend/orchestration/local_subprocess.py index ecb6e248..cbd30b6c 100644 --- a/src/backend/orchestration/local_subprocess.py +++ b/src/backend/orchestration/local_subprocess.py @@ -14,17 +14,20 @@ ``127.0.0.1:8900`` (``SANDBOX_RUNNER_URL`` points here). Pure host subprocess executor — no container needed to run Python/bash. -These are best-effort: a spawn failure is logged and the backend still serves -(tools that need the missing sidecar degrade to an error to the model, main loop -unaffected). +These sidecars are part of the local product's readiness contract. Startup waits +for every registered MCP port and verifies the three default-plugin tool lists; +on failure the API lifespan aborts instead of reporting a misleading healthy +desktop service with missing tools. """ from __future__ import annotations import asyncio +import contextlib import os import signal import sys +import time from typing import List, Optional, Tuple from core.config.settings import settings @@ -35,13 +38,34 @@ # (label, argv) for each managed child. argv[0] is the current interpreter. _PROCS: List[Tuple[str, "asyncio.subprocess.Process"]] = [] +# These are not optional conveniences in the local/desktop product: they back +# the three plugins installed on the first zero-state boot. Keeping this +# contract independent from ``_ports.PORTS`` is deliberate — if a CE packaging +# overlay accidentally drops one registration (the historical site_publish +# failure), startup must fail visibly instead of declaring the backend ready +# while silently omitting the tool. +_REQUIRED_PLUGIN_MCP_TOOLS = { + "automation_task": "list_scheduled_tasks", + "skill_manager": "list_my_skills", + "site_publish": "publish_site", +} + + +def _ready_timeout_seconds() -> float: + raw = os.getenv("LOCAL_SIDECAR_READY_TIMEOUT_SECONDS", "30") + try: + return max(1.0, float(raw)) + except (TypeError, ValueError): + return 30.0 + def _child_env() -> dict: - """Env for children: inherit ours, pin loopback host defaults.""" + """Env for children: inherit ours and force local-only MCP networking.""" env = dict(os.environ) - # MCP servers still bind 0.0.0.0 inside their own process; the backend reaches - # them via MCP_HOST. Pin it to loopback for the single-machine profile. - env.setdefault("MCP_HOST", "127.0.0.1") + # Both the advertised host and the actual listener stay on loopback for the + # single-machine profile. + env["MCP_HOST"] = "127.0.0.1" + env["MCP_BIND_HOST"] = "127.0.0.1" return env @@ -56,33 +80,168 @@ async def _spawn(label: str, argv: List[str]) -> Optional["asyncio.subprocess.Pr _PROCS.append((label, proc)) logger.info("local_sidecar_spawned", sidecar=label, pid=proc.pid) return proc - except Exception as exc: # noqa: BLE001 — best-effort, never block startup + except Exception as exc: # noqa: BLE001 — normalized into readiness failure by caller logger.warning("local_sidecar_spawn_failed", sidecar=label, error=str(exc)) return None +async def _tcp_port_ready(host: str, port: int) -> bool: + try: + reader, writer = await asyncio.wait_for( + asyncio.open_connection(host, port), + timeout=0.5, + ) + del reader + writer.close() + with contextlib.suppress(Exception): + await writer.wait_closed() + return True + except (OSError, asyncio.TimeoutError): + return False + + +async def _wait_for_mcp_ports( + launcher: "asyncio.subprocess.Process", + ports: dict[str, int], + *, + timeout: float, +) -> None: + """Wait until every launchable local MCP server is accepting connections.""" + deadline = time.monotonic() + timeout + pending = dict(ports) + while pending and time.monotonic() < deadline: + if launcher.returncode is not None: + raise RuntimeError(f"MCP 启动器提前退出（exit={launcher.returncode}）") + checks = await asyncio.gather( + *(_tcp_port_ready("127.0.0.1", port) for port in pending.values()) + ) + pending = { + server_id: port + for (server_id, port), ready in zip(pending.items(), checks) + if not ready + } + if pending: + await asyncio.sleep(0.2) + if pending: + details = ", ".join(f"{server_id}:{port}" for server_id, port in pending.items()) + raise RuntimeError(f"MCP 服务未在 {timeout:.0f} 秒内就绪：{details}") + + +async def _list_mcp_tool_names(server_id: str, port: int) -> set[str]: + """Use the production MCP client to verify a server's actual tool list.""" + from core.llm.mcp_pool import make_client + + client = make_client( + server_id, + { + "transport": "streamable_http", + "url": f"http://127.0.0.1:{port}/mcp/", + "transport_timeout": 5, + }, + is_stateful=False, + ) + try: + tools = await client.list_tools() + return {str(name) for tool in tools if (name := getattr(tool, "name", None))} + finally: + try: + await client.close() + except asyncio.CancelledError: + # Some MCP transports use cancellation internally while closing. + # Suppress that implementation detail, but preserve cancellation of + # the actual application startup task. + task = asyncio.current_task() + if task is not None and task.cancelling(): + raise + except Exception: # noqa: BLE001 — readiness result is already known + pass + + +async def _verify_required_plugin_tools(ports: dict[str, int]) -> None: + """Assert that every default plugin MCP exposes its contract tool.""" + missing_registrations = set(_REQUIRED_PLUGIN_MCP_TOOLS) - set(ports) + if missing_registrations: + raise RuntimeError("默认插件缺少 MCP 端口注册：" + ", ".join(sorted(missing_registrations))) + + async def _verify(server_id: str, expected_tool: str) -> None: + names = await _list_mcp_tool_names(server_id, ports[server_id]) + if expected_tool not in names: + raise RuntimeError(f"MCP {server_id} 已监听但缺少必需工具 {expected_tool}") + + await asyncio.gather( + *( + _verify(server_id, expected_tool) + for server_id, expected_tool in _REQUIRED_PLUGIN_MCP_TOOLS.items() + ) + ) + + async def start_local_sidecars() -> None: """Spawn the MCP launcher + script_runner sidecar (local profile only).""" if not settings.deploy.is_local: return py = sys.executable or "python" + from mcp_servers._launcher import PORTS as launcher_ports + from mcp_servers._ports import PORTS as server_ports + from mcp_servers._ports import package_name + + launchable_ports = { + server_id: port + for server_id, port in server_ports.items() + if launcher_ports.get(package_name(server_id)) == port + } + missing_required = set(_REQUIRED_PLUGIN_MCP_TOOLS) - set(launchable_ports) + if missing_required: + raise RuntimeError( + "默认插件 MCP 未进入本地启动清单：" + ", ".join(sorted(missing_required)) + ) + # 1) MCP launcher — one streamable-http server per port, self-supervised. - await _spawn("mcp_launcher", [py, "-m", "mcp_servers._launcher"]) + launcher = await _spawn("mcp_launcher", [py, "-m", "mcp_servers._launcher"]) + if launcher is None: + raise RuntimeError("无法启动 MCP 服务管理进程") # 2) Code-execution sidecar — host subprocess executor on 127.0.0.1:8900. # Only start it when script_runner is the selected provider (default). if settings.sandbox.provider == "script_runner": - await _spawn( + runner = await _spawn( "script_runner", [ - py, "-m", "uvicorn", + py, + "-m", + "uvicorn", "services.script_runner_service.server:app", - "--host", "127.0.0.1", - "--port", "8900", - "--log-level", "warning", + "--host", + "127.0.0.1", + "--port", + "8900", + "--log-level", + "warning", ], ) + if runner is None: + await stop_local_sidecars() + raise RuntimeError("无法启动本机代码执行服务") + + # Do not let uvicorn finish its lifespan (and therefore let the desktop + # shell report /health as ready) until the sidecars behind the advertised + # default plugins are genuinely usable. + try: + await _wait_for_mcp_ports( + launcher, + launchable_ports, + timeout=_ready_timeout_seconds(), + ) + await _verify_required_plugin_tools(launchable_ports) + logger.info( + "local_mcp_sidecars_ready", + servers=len(launchable_ports), + required_plugins=sorted(_REQUIRED_PLUGIN_MCP_TOOLS), + ) + except BaseException: + await stop_local_sidecars() + raise async def stop_local_sidecars() -> None: diff --git a/src/backend/orchestration/memory_integration.py b/src/backend/orchestration/memory_integration.py index 68ab55d0..e474b843 100644 --- a/src/backend/orchestration/memory_integration.py +++ b/src/backend/orchestration/memory_integration.py @@ -37,8 +37,9 @@ async def launch_memory_retrieval( if not memory_enabled or not user_id: return None - effective_budget = (budget_ms if budget_ms is not None - else settings.memory.retrieval_budget_ms) / 1000.0 + effective_budget = ( + budget_ms if budget_ms is not None else settings.memory.retrieval_budget_ms + ) / 1000.0 async def _fetch() -> Optional[str]: try: @@ -82,8 +83,9 @@ async def build_frozen_memory_block( try: profile_md = await profile.get(user_id, workspace_id) except Exception as exc: - logger.warning("[memory] profile fetch failed user=%s ws=%s: %s", - user_id, workspace_id, exc) + logger.warning( + "[memory] profile fetch failed user=%s ws=%s: %s", user_id, workspace_id, exc + ) # Fact layer (L2) fact_text = "" @@ -96,7 +98,9 @@ async def build_frozen_memory_block( wait_budget_s = max(0.1, settings.memory.retrieval_budget_ms / 1000.0) fact_text = await asyncio.wait_for(memory_task, timeout=wait_budget_s) or "" except asyncio.TimeoutError: - logger.info("[memory] fact retrieval still running past wait window, skipping injection") + logger.info( + "[memory] fact retrieval still running past wait window, skipping injection" + ) # The task finishes in the background and is released; not cancelled (the result can be used for the next round of log statistics) except asyncio.CancelledError: raise @@ -128,7 +132,11 @@ async def build_frozen_memory_block( block = "\n".join(parts).strip() logger.info( "[memory] frozen block built user=%s ws=%s chars=%d profile=%d facts=%d", - user_id, workspace_id, len(block), len(profile_md or ""), len(fact_text or ""), + user_id, + workspace_id, + len(block), + len(profile_md or ""), + len(fact_text or ""), ) return block @@ -162,17 +170,10 @@ async def build_user_identity_block(user_id: str) -> str: def _query() -> tuple: from core.db.engine import SessionLocal from core.db.models import LocalUser, UserShadow + with SessionLocal() as db: - row = ( - db.query(UserShadow.username) - .filter(UserShadow.user_id == user_id) - .first() - ) - nick = ( - db.query(LocalUser.nickname) - .filter(LocalUser.user_id == user_id) - .first() - ) + row = db.query(UserShadow.username).filter(UserShadow.user_id == user_id).first() + nick = db.query(LocalUser.nickname).filter(LocalUser.user_id == user_id).first() return ( (row[0] or "").strip() if row else "", (nick[0] or "").strip() if nick else "", @@ -192,7 +193,8 @@ def _query() -> tuple: block = "" if lines: block = ( - "## 当前用户\n" + "\n".join(lines) + "## 当前用户\n" + + "\n".join(lines) + "\n需要称呼用户时，用上述昵称（无昵称则用用户名）自然称呼。" ) _identity_cache[user_id] = (now + _IDENTITY_CACHE_TTL_S, block) @@ -215,23 +217,14 @@ async def inject_frozen_memory( return session_messages parts: list[str] = [] if identity_block: - parts.append( - "\n" - f"{identity_block}\n" - "" - ) + parts.append("\n" f"{identity_block}\n" "") if frozen_block: - parts.append( - "\n" - f"{frozen_block}\n" - "" - ) + parts.append("\n" f"{frozen_block}\n" "") return [ { "role": "user", "content": ( - "\n\n".join(parts) - + "\n（以上为会话启动时系统注入的背景快照，本会话内不变，" + "\n\n".join(parts) + "\n（以上为会话启动时系统注入的背景快照，本会话内不变，" "用作回答参考，请勿直接复述。）" ), }, @@ -264,9 +257,8 @@ def save_memories_background( - each extractor has its own 30s timeout - sanitize → write L1/L2/Session → audit - `scope_user_id` under team projects = ``f"team:{team_id}"``, so all team members' writes - go into the same mem0 user_id bucket, and reads pull from that bucket to achieve sharing. - ``user_id`` remains the real user, used for audit and metadata.author_user_id. + ``scope_user_id`` optionally selects an edition-owned shared memory bucket. + ``user_id`` remains the real user for audit metadata. """ if not (write_enabled and full_response and user_id): return diff --git a/src/backend/orchestration/workflow.py b/src/backend/orchestration/workflow.py index a01b7154..99db1164 100644 --- a/src/backend/orchestration/workflow.py +++ b/src/backend/orchestration/workflow.py @@ -31,6 +31,7 @@ requires_output_review, ) from core.services.ontology_service import resolve_runtime_asset_tags +from core.services.project_scope import edition_project_context_keys from orchestration.citations import extract_citations_with_offset from orchestration.streaming import StreamingAgent @@ -42,9 +43,8 @@ "project_folder_name", "project_folder_kind", "project_folder_id", - "project_team_id", # only set for team kind; passed by agent_factory to the MySpace tools for the TeamFolder path "project_files", -) +) + edition_project_context_keys() def _extract_project_ctx(context: Dict[str, Any]) -> Optional[Dict[str, Any]]: diff --git a/src/backend/plugin_bundles/marketplace/sites/plugin.json b/src/backend/plugin_bundles/marketplace/sites/plugin.json index 73a26aa9..0b1ad761 100644 --- a/src/backend/plugin_bundles/marketplace/sites/plugin.json +++ b/src/backend/plugin_bundles/marketplace/sites/plugin.json @@ -2,7 +2,7 @@ "name": "sites", "version": "1.1.0", "display_name": "站点·对话建站", - "description": "用对话把想法做成真实网站并一键发布上线：简单内容直接生成静态站，复杂/精美需求用预装的 React 工程模板（antd + echarts + tailwind）在沙箱内构建，建站前还会生成 3 个设计方案预览图供你挑选。调 publish_site 发布成平台托管站点，拿到形如 /site// 的链接即可访问。支持自定义访问地址、公开/私密/团队可见、版本回滚、访问统计，以及站点内置轻后端（KV 存储 + 表单收集）。装上后在对话里描述需求即可建站，也可在「实验室 → 站点」里管理。", + "description": "用对话把想法做成真实网站并一键发布上线：简单内容直接生成静态站，复杂/精美需求用预装的 React 工程模板（antd + echarts + tailwind）在沙箱内构建，建站前还会生成 3 个设计方案预览图供你挑选。调 publish_site 发布成平台托管站点，拿到形如 /site// 的链接即可访问。支持自定义访问地址、公开/私密可见、版本回滚、访问统计，以及站点内置轻后端（KV 存储 + 表单收集）。装上后在对话里描述需求即可建站，也可在「实验室 → 站点」里管理。", "category": "信息处理", "author": "HugAgentOS", "components": { @@ -20,7 +20,7 @@ "display_name": "对话建站发布", "description": "把沙箱里的静态网站目录发布为平台托管站点，返回可访问 URL。身份/会话走 X-Current-User-Id / X-Conversation-Id 头；发布动作转发到后端内部接口完成（后端有沙箱访问权）。", "tools": [ - {"name": "publish_site", "description": "把沙箱里的一个网站目录（/workspace 下，必须含 index.html）发布为平台托管站点，返回形如 /site// 的可访问 URL。用户要'做个网站/页面/门户/展示站/看板/H5 并能访问'时：静态站先用 write/bash 生成后直接发布；React 构建型站点（site-builder 技能路径 B）构建出产物后发布，src_dir 指产物目录、source_dir 指源码工程目录（两参必传，源码镜像进项目、产物进托管）。更新已有站点：带 site_id 重新发布，URL 不变、版本 +1。可选 slug（自定义地址）/visibility（public/private/team）/description/team_id。站点支持内置轻后端 API：相对路径 fetch __api/kv/（KV 存储）与 POST __api/forms/（表单收集）。限制 ≤300 文件、≤30MB。"} + {"name": "publish_site", "description": "把沙箱里的一个网站目录（/workspace 下，必须含 index.html）发布为平台托管站点，返回形如 /site// 的可访问 URL。用户要'做个网站/页面/门户/展示站/看板/H5 并能访问'时：静态站先用 write/bash 生成后直接发布；React 构建型站点（site-builder 技能路径 B）构建出产物后发布，src_dir 指产物目录、source_dir 指源码工程目录（两参必传，源码镜像进项目、产物进托管）。更新已有站点：带 site_id 重新发布，URL 不变、版本 +1。可选 slug（自定义地址）/visibility（public/private）/description。站点支持内置轻后端 API：相对路径 fetch __api/kv/（KV 存储）与 POST __api/forms/（表单收集）。限制 ≤300 文件、≤30MB。"} ] } } diff --git a/src/backend/plugin_bundles/marketplace/sites/skills/site-builder/SKILL.md b/src/backend/plugin_bundles/marketplace/sites/skills/site-builder/SKILL.md index 173342f3..cfa6201b 100644 --- a/src/backend/plugin_bundles/marketplace/sites/skills/site-builder/SKILL.md +++ b/src/backend/plugin_bundles/marketplace/sites/skills/site-builder/SKILL.md @@ -150,7 +150,7 @@ bash "${SITE_TEMPLATE_HOME:-/opt/site-template}/init-react-site.sh" /workspace/s - 公开站点在浏览器里以**沙箱模式**运行（无 cookie / localStorage）——不要写依赖 登录态或浏览器本地存储的逻辑，持久化用下面的轻后端 API。 - 可见性 `visibility`：`public`（默认，凭链接访问）/ `private`（仅本人登录可见）/ - `team`（指定团队成员可见，配合 `team_id`）。 + ## 站点内置轻后端 API（可选，需要动态能力时用） diff --git a/src/backend/prompts/kb_lite_section.py b/src/backend/prompts/kb_lite_section.py index ef723374..fe201f6c 100644 --- a/src/backend/prompts/kb_lite_section.py +++ b/src/backend/prompts/kb_lite_section.py @@ -10,7 +10,6 @@ from time import monotonic from typing import Dict, List, Optional, Tuple - # --------------------------------------------------------------------------- # Lightweight KB catalog — name + description only (no document lists). # Injected into system prompt per user's enabled_kbs, from cached data. @@ -30,7 +29,7 @@ def invalidate_kb_lite_cache() -> None: def _build_kb_lite_section(enabled_kb_ids: Optional[List[str]]) -> str: """Build a minimal KB catalog (name + description) for system prompt injection. - Only uses cached Dify dataset list (no extra API calls) and fast DB queries. + Uses a cached external collection list plus fast local DB queries. Typical output: 3-10 lines, 300-800 chars. """ if not enabled_kb_ids: @@ -47,40 +46,47 @@ def _build_kb_lite_section(enabled_kb_ids: Optional[List[str]]) -> str: return text import logging + _log = logging.getLogger(__name__) - dify_ids = [kid for kid in enabled_kb_ids if not kid.startswith("kb_")] + external_ids = [kid for kid in enabled_kb_ids if not kid.startswith("kb_")] local_ids = [kid for kid in enabled_kb_ids if kid.startswith("kb_")] lines: List[str] = [] - # ── Public datasets (Dify) — from cached list, no extra HTTP calls ──── - if dify_ids: + # ── Externally managed shared collections — cached, no extra HTTP calls ── + if external_ids: try: - from core.kb.dify_kb import is_dify_enabled, list_datasets - if is_dify_enabled(): - dify_set = set(dify_ids) - datasets = list_datasets(page=1, limit=100, timeout=(1, 2)) + from core.kb.external_provider import is_enabled, list_collections + + if is_enabled(): + external_set = set(external_ids) + datasets = list_collections(page=1, limit=100, timeout=(1, 2)) for ds in datasets: ds_id = str(ds.get("id", "")).strip() - if ds_id and ds_id in dify_set: + if ds_id and ds_id in external_set: name = ds.get("name", ds_id) desc = ds.get("description") or ds.get("desc") or "" desc_part = f"：{desc[:120]}" if desc else "" lines.append(f"- {name}（公有，dataset_id: `{ds_id}`）{desc_part}") except Exception as exc: - _log.debug("[kb_lite] Dify list failed: %s", exc) + _log.debug("[kb_lite] external collection list failed: %s", exc) # ── Private KBs — fast DB query ─────────────────────────────────────── if local_ids: try: from core.db.engine import SessionLocal from core.db.models import KBSpace + with SessionLocal() as db: - spaces = db.query(KBSpace).filter( - KBSpace.kb_id.in_(local_ids), - KBSpace.deleted_at.is_(None), - ).all() + spaces = ( + db.query(KBSpace) + .filter( + KBSpace.kb_id.in_(local_ids), + KBSpace.deleted_at.is_(None), + ) + .all() + ) for s in spaces: desc_part = f"：{s.description[:120]}" if s.description else "" lines.append(f"- {s.name}（私有，kb_id: `{s.kb_id}`）{desc_part}") @@ -93,8 +99,7 @@ def _build_kb_lite_section(enabled_kb_ids: Optional[List[str]]) -> str: result = ( "## 当前启用的知识库\n" "当用户提问涉及以下知识库名称或简介中的关键词时，应**主动**调用对应检索工具，无需等待用户显式要求。\n" - "调用 `list_datasets` 可获取更详细的文档列表。\n\n" - + "\n".join(lines) + "调用 `list_datasets` 可获取更详细的文档列表。\n\n" + "\n".join(lines) ) with _kb_lite_cache_lock: diff --git a/src/backend/tests/api/test_projects.py b/src/backend/tests/api/test_projects.py deleted file mode 100644 index 37a1aa04..00000000 --- a/src/backend/tests/api/test_projects.py +++ /dev/null @@ -1,411 +0,0 @@ -"""Project module unit tests (project ↔ MySpace folder hard-binding version). - -Covers: -- Creating a personal project auto-creates a user_folder of the same name -- Duplicate project name rejected -- Specifying an existing folder at creation -- Rejected when the folder is already bound to another project -- Team member rejected from creating a team project; owner passes + team subfolder auto-created -- Uploaded files land directly in the bound folder (also appear in the MySpace list) -- Uploading a file with a path auto-mkdirs subfolders -- The in-project file list is the bound folder's subtree -- Soft-deleting a project does not delete the bound folder itself -""" - -from __future__ import annotations - -import pytest -from fastapi import HTTPException - -from core.db.models import ( - Artifact, - Project, - Team, - TeamMember, - UserFolder, - UserShadow, -) -from core.db.repository import ArtifactRepository -from core.services.project_file_service import ProjectFileService -from core.services.project_service import ProjectService - - -# ── Fixtures ────────────────────────────────────────────────────────────── - - -@pytest.fixture -def alice(db_session): - u = UserShadow(user_id="user_alice", username="Alice", email="alice@example.com") - db_session.add(u) - db_session.commit() - return u - - -@pytest.fixture -def bob(db_session): - u = UserShadow(user_id="user_bob", username="Bob", email="bob@example.com") - db_session.add(u) - db_session.commit() - return u - - -@pytest.fixture -def alice_team(db_session, alice, bob): - team = Team(team_id="team_acme", name="Acme", owner_user_id=alice.user_id, source="manual") - db_session.add(team) - db_session.add(TeamMember(team_id=team.team_id, user_id=alice.user_id, role="owner", file_permission="editor")) - db_session.add(TeamMember(team_id=team.team_id, user_id=bob.user_id, role="member", file_permission="viewer")) - db_session.commit() - return team - - -@pytest.fixture(autouse=True) -def fake_storage(monkeypatch): - store: dict[str, bytes] = {} - - class _Fake: - def upload_bytes(self, data: bytes, key: str) -> str: - store[key] = data - return f"local://{key}" - - def download_bytes(self, key: str) -> bytes: - return store[key] - - def delete(self, key: str) -> None: - store.pop(key, None) - - from core.services import project_file_service as pfs - - monkeypatch.setattr(pfs, "get_storage", lambda: _Fake()) - - -# ── Create / linked folder ─────────────────────────────────────────────── - - -def test_create_personal_auto_creates_folder(db_session, alice): - svc = ProjectService(db_session) - p = svc.create_personal(alice.user_id, name="P1", description="hi") - assert p.linked_folder_id is not None - folder = db_session.query(UserFolder).filter(UserFolder.folder_id == p.linked_folder_id).first() - assert folder is not None - assert folder.name == "P1" - assert folder.parent_folder_id is None - - -def test_create_personal_duplicate_folder_name_appends_suffix(db_session, alice): - """When creating a second project with the same name, the auto-created folder appends a suffix to avoid conflict.""" - svc = ProjectService(db_session) - p1 = svc.create_personal(alice.user_id, name="Demo") - # Delete p1 (so a same-named project can be created), but the folder is kept - svc.soft_delete(p1.project_id, alice.user_id) - p2 = svc.create_personal(alice.user_id, name="Demo") - folder = db_session.query(UserFolder).filter(UserFolder.folder_id == p2.linked_folder_id).first() - assert folder.name == "Demo (2)" # does not conflict with the existing "Demo" - - -def test_create_personal_with_existing_folder(db_session, alice): - folder = UserFolder(folder_id="ufld_x", user_id=alice.user_id, parent_folder_id=None, name="Already") - db_session.add(folder) - db_session.commit() - svc = ProjectService(db_session) - p = svc.create_personal(alice.user_id, name="UseExisting", linked_folder_id=folder.folder_id) - assert p.linked_folder_id == folder.folder_id - - -def test_existing_folder_used_by_another_project_rejected(db_session, alice): - folder = UserFolder(folder_id="ufld_y", user_id=alice.user_id, parent_folder_id=None, name="Shared") - db_session.add(folder) - db_session.commit() - svc = ProjectService(db_session) - svc.create_personal(alice.user_id, name="First", linked_folder_id=folder.folder_id) - with pytest.raises(HTTPException) as exc: - svc.create_personal(alice.user_id, name="Second", linked_folder_id=folder.folder_id) - assert exc.value.status_code == 400 - - -def test_duplicate_personal_project_name_rejected(db_session, alice): - svc = ProjectService(db_session) - svc.create_personal(alice.user_id, name="Dup") - with pytest.raises(HTTPException): - svc.create_personal(alice.user_id, name="Dup") - - -# ── Team project ───────────────────────────────────────────────────────── - - -def test_team_member_cannot_create_team_project(db_session, alice_team, bob): - svc = ProjectService(db_session) - with pytest.raises(HTTPException) as exc: - svc.create_team(bob.user_id, alice_team.team_id, name="Bob") - assert exc.value.status_code == 403 - - -def test_team_owner_creates_team_project_auto_team_folder(db_session, alice_team, alice, bob): - svc = ProjectService(db_session) - p = svc.create_team(alice.user_id, alice_team.team_id, name="TeamP") - assert p.linked_team_folder_id is not None - from core.db.models import TeamFolder - f = db_session.query(TeamFolder).filter(TeamFolder.folder_id == p.linked_team_folder_id).first() - assert f.name == "TeamP" - assert f.team_id == alice_team.team_id - - # Bob is a member and can see the team project - items, total = svc.list_visible(bob.user_id) - assert total == 1 - assert items[0]["folder_name"] == "TeamP" - - -# ── File operations ────────────────────────────────────────────────────── - - -def test_upload_file_lands_in_linked_folder(db_session, alice): - svc = ProjectService(db_session) - project = svc.create_personal(alice.user_id, name="UPL") - pfs = ProjectFileService(db_session) - item = pfs.upload(project, alice.user_id, b"hello", "a.txt", "text/plain") - assert item["artifact_id"].startswith("pj_") - - art = db_session.query(Artifact).filter(Artifact.artifact_id == item["artifact_id"]).first() - assert art.user_folder_id == project.linked_folder_id - - -def test_upload_with_subpath_creates_nested_folder(db_session, alice): - svc = ProjectService(db_session) - project = svc.create_personal(alice.user_id, name="NestedUp") - pfs = ProjectFileService(db_session) - item = pfs.upload(project, alice.user_id, b"x", "sub/deep/x.txt", "text/plain") - # Both subfolders 'sub' and 'sub/deep' should be created - sub = ( - db_session.query(UserFolder) - .filter( - UserFolder.user_id == alice.user_id, - UserFolder.parent_folder_id == project.linked_folder_id, - UserFolder.name == "sub", - UserFolder.deleted_at.is_(None), - ) - .first() - ) - assert sub is not None - deep = ( - db_session.query(UserFolder) - .filter( - UserFolder.user_id == alice.user_id, - UserFolder.parent_folder_id == sub.folder_id, - UserFolder.name == "deep", - UserFolder.deleted_at.is_(None), - ) - .first() - ) - assert deep is not None - art = db_session.query(Artifact).filter(Artifact.artifact_id == item["artifact_id"]).first() - assert art.user_folder_id == deep.folder_id - - -def test_list_files_returns_subtree(db_session, alice): - svc = ProjectService(db_session) - project = svc.create_personal(alice.user_id, name="LST") - pfs = ProjectFileService(db_session) - pfs.upload(project, alice.user_id, b"a", "root.txt", "text/plain") - pfs.upload(project, alice.user_id, b"b", "child/inner.txt", "text/plain") - items = pfs.list_files(project) - names = {it["name"] for it in items} - assert "root.txt" in names - assert "child/inner.txt" in names - - -def test_project_uploaded_files_visible_in_myspace(db_session, alice): - """New design: project upload = MySpace upload, so it should also be visible in the main MySpace list.""" - svc = ProjectService(db_session) - project = svc.create_personal(alice.user_id, name="MS") - pfs = ProjectFileService(db_session) - pfs.upload(project, alice.user_id, b"x", "z.txt", "text/plain") - - repo = ArtifactRepository(db_session) - rows, total = repo.list_by_user_with_chat(user_id=alice.user_id, personal_only=True) - assert total == 1 - assert rows[0]["artifact"].filename == "z.txt" - - -# ── Soft delete ────────────────────────────────────────────────────────── - - -def test_myspace_rel_under_project_scope_redirects(): - """With a personal project scope passed in, myspace_rel's output is prefixed with the - bound folder name (without duplicating the prefix). scope is now an explicit parameter, - no longer via ContextVar.""" - from core.llm.tools.myspace_vfs import myspace_rel - from core.services.project_scope import ProjectScope - - scope = ProjectScope( - project_id="prj_x", kind="personal", - root_folder_id="ufld_x", folder_name="P1", - ) - # Root directory - assert myspace_rel("/myspace", "u1", scope) == "P1" - # Subpath - assert myspace_rel("/myspace/foo.txt", "u1", scope) == "P1/foo.txt" - # Already under the project: no duplication - assert myspace_rel("/myspace/P1/foo.txt", "u1", scope) == "P1/foo.txt" - # Non-myspace paths still return None - assert myspace_rel("/workspace/skills/x", "u1", scope) is None - - # Without scope, back to normal behavior - assert myspace_rel("/myspace/foo.txt", "u1") == "foo.txt" - - -def test_project_scope_team_kind_also_prefixes(): - """A team-project scope likewise prefixes the bound folder name (so relative paths written by the LLM land under the project).""" - from core.llm.tools.myspace_vfs import myspace_rel - from core.services.project_scope import ProjectScope - - scope = ProjectScope( - project_id="prj_t", kind="team", - root_folder_id="fld_t", folder_name="TP", - team_id="team_42", - ) - assert myspace_rel("/myspace/x.txt", "u1", scope) == "TP/x.txt" - - -def test_soft_delete_project_keeps_folder(db_session, alice): - svc = ProjectService(db_session) - project = svc.create_personal(alice.user_id, name="Del") - folder_id = project.linked_folder_id - assert svc.soft_delete(project.project_id, alice.user_id) is True - # The folder still exists (still accessible from the MySpace side) - folder = db_session.query(UserFolder).filter(UserFolder.folder_id == folder_id).first() - assert folder is not None - assert folder.deleted_at is None - - -# ── _persist_artifacts auto-routing under project scope ────────────────── - - -def test_persist_artifacts_under_personal_project_scope_routes_to_folder( - db_session, alice, monkeypatch, -): - """With a personal project scope active, _persist_artifacts automatically routes the artifact to the bound folder. - - Covers the bash→sandbox_get_artifact→pin_to_workspace chain: previously the artifact - row had no user_folder_id and the file showed up in the "My Space" root directory - instead of inside the project. - """ - from api.routes.v1 import chats as chats_module - from core.db.models import ChatSession - from core.services.project_scope import ProjectScope - - svc = ProjectService(db_session) - project = svc.create_personal(alice.user_id, name="Proj42") - chat = ChatSession(chat_id="chat_pa1", user_id=alice.user_id, title="t") - db_session.add(chat) - db_session.commit() - - # _persist_artifacts internally queries db.query(ArtifactModel.artifact_id); the same - # db_session must be used — just pass it in directly (the fixture already provides the - # same engine bound to the session) - collected = [{ - "file_id": "ai_chart_001", - "name": "chart.png", - "mime_type": "image/png", - "size": 1234, - "storage_key": "artifacts/ai_chart_001", - "url": "/files/ai_chart_001", - "tool_name": "pin_to_workspace", - }] - - scope = ProjectScope( - project_id=project.project_id, - kind="personal", - root_folder_id=project.linked_folder_id, - folder_name="Proj42", - ) - chats_module._persist_artifacts( - db_session, alice.user_id, chat.chat_id, collected, scope=scope, - ) - - row = db_session.query(Artifact).filter( - Artifact.artifact_id == "ai_chart_001", - ).first() - assert row is not None - assert row.user_folder_id == project.linked_folder_id - - -def test_persist_artifacts_without_scope_stays_at_root(db_session, alice): - """Non-project chats are unaffected: user_folder_id stays empty (lands in the MySpace root).""" - from api.routes.v1 import chats as chats_module - from core.db.models import ChatSession - - chat = ChatSession(chat_id="chat_nr1", user_id=alice.user_id, title="t") - db_session.add(chat) - db_session.commit() - - collected = [{ - "file_id": "ai_chart_002", - "name": "chart.png", - "mime_type": "image/png", - "size": 1234, - "storage_key": "artifacts/ai_chart_002", - "url": "/files/ai_chart_002", - "tool_name": "pin_to_workspace", - }] - chats_module._persist_artifacts( - db_session, alice.user_id, chat.chat_id, collected, - ) - row = db_session.query(Artifact).filter( - Artifact.artifact_id == "ai_chart_002", - ).first() - assert row is not None - assert row.user_folder_id is None - - -def test_persist_artifacts_under_team_scope_routes_to_team_folder( - db_session, alice, -): - """team scope: artifacts are written with team_id + team_folder_id, no longer leaking into the personal MySpace root. - - Regression guard: a past implementation early-returned on team scope (PR 1 read-only), - compounded by chat_run_executor / automation_scheduler failing to pass scope, so team - project AI output was written as orphan rows with user_folder_id=NULL/team_id=NULL - landing in the personal root. - """ - from api.routes.v1 import chats as chats_module - from core.db.models import ChatSession, Team, TeamFolder - from core.services.project_scope import ProjectScope - - # PostgreSQL enforces foreign keys: artifacts.team_id → teams.team_id and - # artifacts.team_folder_id → team_folders.folder_id, so the corresponding rows must be - # created first (previously optional under SQLite which doesn't enforce FKs; required - # after switching to a real database). - db_session.add(Team(team_id="team_xxx", name="TeamProj", - owner_user_id=alice.user_id, source="manual")) - db_session.add(TeamFolder(folder_id="tfld_xxx", team_id="team_xxx", - name="TeamProj", created_by=alice.user_id)) - chat = ChatSession(chat_id="chat_tm1", user_id=alice.user_id, title="t") - db_session.add(chat) - db_session.commit() - - collected = [{ - "file_id": "ai_chart_003", - "name": "chart.png", - "mime_type": "image/png", - "size": 1234, - "storage_key": "artifacts/ai_chart_003", - "url": "/files/ai_chart_003", - "tool_name": "pin_to_workspace", - }] - scope = ProjectScope( - project_id="prj_t", - kind="team", - root_folder_id="tfld_xxx", - folder_name="TeamProj", - team_id="team_xxx", - ) - chats_module._persist_artifacts( - db_session, alice.user_id, chat.chat_id, collected, scope=scope, - ) - row = db_session.query(Artifact).filter( - Artifact.artifact_id == "ai_chart_003", - ).first() - assert row is not None - assert row.team_id == "team_xxx" - assert row.team_folder_id == "tfld_xxx" - assert row.user_folder_id is None diff --git a/src/backend/tests/mcp/test_blocking_lane.py b/src/backend/tests/mcp/test_blocking_lane.py new file mode 100644 index 00000000..a94e9575 --- /dev/null +++ b/src/backend/tests/mcp/test_blocking_lane.py @@ -0,0 +1,22 @@ +"""Edition-neutral bounded blocking-lane contract.""" + +import asyncio +import time + +import pytest + + +@pytest.mark.asyncio +async def test_blocking_lane_keeps_slot_until_timed_out_thread_finishes(): + from mcp_servers.retrieve_dataset_content_mcp.server import _BlockingLane + + lane = _BlockingLane(name="test", max_workers=1) + + with pytest.raises(TimeoutError): + await lane.run(lambda: time.sleep(0.08), timeout=0.02) + + with pytest.raises(TimeoutError): + await lane.run(lambda: "too-early", timeout=0.02) + + await asyncio.sleep(0.07) + assert await lane.run(lambda: "done", timeout=0.05) == "done" diff --git a/src/backend/tests/mcp/test_knowledge_retrieval_concurrency.py b/src/backend/tests/mcp/test_knowledge_retrieval_concurrency.py deleted file mode 100644 index 0a8a5c3d..00000000 --- a/src/backend/tests/mcp/test_knowledge_retrieval_concurrency.py +++ /dev/null @@ -1,187 +0,0 @@ -from __future__ import annotations - -import asyncio -import time - -import pytest - - -class _FakeResponse: - def __init__(self, dataset_id: str) -> None: - self._dataset_id = dataset_id - - def raise_for_status(self) -> None: - return None - - def json(self) -> dict: - return { - "records": [ - { - "segment": { - "content": f"content-{self._dataset_id}", - "tokens": 1, - "document": { - "id": f"doc-{self._dataset_id}", - "name": f"document-{self._dataset_id}", - }, - } - } - ] - } - - -class _FakeAsyncClient: - active = 0 - max_active = 0 - delay = 0.02 - - def __init__(self, **_kwargs) -> None: - pass - - async def __aenter__(self): - return self - - async def __aexit__(self, *_args) -> None: - return None - - async def post(self, url: str, **_kwargs) -> _FakeResponse: - type(self).active += 1 - type(self).max_active = max(type(self).max_active, type(self).active) - try: - await asyncio.sleep(type(self).delay) - finally: - type(self).active -= 1 - return _FakeResponse(url.rsplit("/", 2)[-2]) - - -@pytest.mark.asyncio -async def test_public_retrieval_is_nonblocking_and_globally_bounded(monkeypatch): - from mcp_servers.retrieve_dataset_content_mcp import impl - - monkeypatch.setattr(impl, "RETRIEVE_MAX_CONCURRENCY", 2) - monkeypatch.setattr(impl, "RETRIEVE_TOTAL_TIMEOUT_SECONDS", 1) - monkeypatch.setattr(impl.httpx, "AsyncClient", _FakeAsyncClient) - monkeypatch.setattr( - impl, - "_resolve_public_retrieve_scope", - lambda **_kwargs: ("http://dify.test", "token", {f"dataset-{i}" for i in range(6)}), - ) - impl._public_retrieve_limiter = None - impl._public_retrieve_limiter_loop = None - _FakeAsyncClient.active = 0 - _FakeAsyncClient.max_active = 0 - _FakeAsyncClient.delay = 0.02 - - retrieval = asyncio.gather( - impl.retrieve_dataset_content_async(query="first"), - impl.retrieve_dataset_content_async(query="second"), - ) - ticks = 0 - while not retrieval.done(): - ticks += 1 - await asyncio.sleep(0.005) - - first_items, second_items = await retrieval - assert len(first_items) == 6 - assert len(second_items) == 6 - assert _FakeAsyncClient.max_active == 2 - assert ticks >= 3, "event loop should remain responsive while Dify requests are in flight" - - -@pytest.mark.asyncio -async def test_public_retrieval_has_one_end_to_end_deadline(monkeypatch): - from mcp_servers.retrieve_dataset_content_mcp import impl - - monkeypatch.setattr(impl, "RETRIEVE_MAX_CONCURRENCY", 1) - monkeypatch.setattr(impl, "RETRIEVE_TOTAL_TIMEOUT_SECONDS", 0.05) - monkeypatch.setattr(impl.httpx, "AsyncClient", _FakeAsyncClient) - monkeypatch.setattr( - impl, - "_resolve_public_retrieve_scope", - lambda **_kwargs: ("http://dify.test", "token", {"dataset-1"}), - ) - impl._public_retrieve_limiter = None - impl._public_retrieve_limiter_loop = None - _FakeAsyncClient.active = 0 - _FakeAsyncClient.max_active = 0 - _FakeAsyncClient.delay = 1.0 - - started = time.monotonic() - with pytest.raises(impl.DatasetRetrievalTimeoutError): - await impl.retrieve_dataset_content_async(query="test") - assert time.monotonic() - started < 0.25 - - -@pytest.mark.asyncio -async def test_all_public_upstream_failures_are_not_reported_as_empty_results(monkeypatch): - from mcp_servers.retrieve_dataset_content_mcp import impl - - class _FailingClient(_FakeAsyncClient): - async def post(self, _url: str, **_kwargs): - raise OSError("upstream unavailable") - - monkeypatch.setattr(impl, "RETRIEVE_MAX_CONCURRENCY", 2) - monkeypatch.setattr(impl, "RETRIEVE_TOTAL_TIMEOUT_SECONDS", 1) - monkeypatch.setattr(impl.httpx, "AsyncClient", _FailingClient) - monkeypatch.setattr( - impl, - "_resolve_public_retrieve_scope", - lambda **_kwargs: ("http://dify.test", "token", {"dataset-1", "dataset-2"}), - ) - impl._public_retrieve_limiter = None - impl._public_retrieve_limiter_loop = None - - with pytest.raises(impl.DatasetRetrievalUnavailableError, match="全部 2 个"): - await impl.retrieve_dataset_content_async(query="test") - - -@pytest.mark.asyncio -async def test_blocking_lane_keeps_slot_until_timed_out_thread_finishes(): - from mcp_servers.retrieve_dataset_content_mcp.server import _BlockingLane - - lane = _BlockingLane(name="test", max_workers=1) - - with pytest.raises(TimeoutError): - await lane.run(lambda: time.sleep(0.08), timeout=0.02) - - # The first caller timed out, but its thread is still alive. A new caller - # must not be admitted into another unbounded executor queue. - with pytest.raises(TimeoutError): - await lane.run(lambda: "too-early", timeout=0.02) - - await asyncio.sleep(0.07) - assert await lane.run(lambda: "done", timeout=0.05) == "done" - - -@pytest.mark.asyncio -async def test_cancelling_blocking_caller_does_not_release_a_live_thread_slot(): - from mcp_servers.retrieve_dataset_content_mcp.server import _BlockingLane - - lane = _BlockingLane(name="cancel-test", max_workers=1) - task = asyncio.create_task(lane.run(lambda: time.sleep(0.08), timeout=1.0)) - await asyncio.sleep(0.01) - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - - with pytest.raises(TimeoutError): - await lane.run(lambda: "too-early", timeout=0.02) - - await asyncio.sleep(0.07) - assert await lane.run(lambda: "done", timeout=0.05) == "done" - - -@pytest.mark.asyncio -async def test_public_timeout_is_returned_as_a_tool_result(monkeypatch): - from mcp_servers.retrieve_dataset_content_mcp import impl, server - - async def _timeout(**_kwargs): - raise impl.DatasetRetrievalTimeoutError("public retrieval deadline") - - monkeypatch.setattr(impl, "retrieve_dataset_content_async", _timeout) - result = await server.retrieve_dataset_content(query="test") - - assert result["items"] == [] - assert result["error"]["code"] == "tool_timeout" - assert result["error"]["tool"] == "retrieve_dataset_content" - assert result["error"]["retryable"] is True diff --git a/src/backend/tests/ontology/test_ontology_harness.py b/src/backend/tests/ontology/test_ontology_harness.py index 709addac..1243fef5 100644 --- a/src/backend/tests/ontology/test_ontology_harness.py +++ b/src/backend/tests/ontology/test_ontology_harness.py @@ -1010,6 +1010,50 @@ def test_user_runtime_resolver_preserves_opt_out(db_session): assert opted_in is True assert runtime["review_level"] == "committee" + service.set_pack_flags("enterprise_risk", is_enabled=False) + opted_in, runtime = build_user_ontology_runtime( + user_id=user.user_id, + task="分析企业风险", + db=db_session, + ) + assert opted_in is False + assert runtime == {"enabled": False, "packs": [], "review_level": "none"} + + +def test_chat_context_treats_admin_disabled_pack_as_effective_opt_out(db_session, monkeypatch): + from api.routes.v1 import chats as chat_routes + from api.schemas import ChatRequest + from sqlalchemy.orm import sessionmaker + + service = OntologyService(db_session) + service.create_version(_sample_payload(), actor_id="tester", activate=True) + service.set_pack_flags("enterprise_risk", is_enabled=False, is_default=True) + + session_factory = sessionmaker(bind=db_session.get_bind()) + monkeypatch.setattr(chat_routes, "SessionLocal", session_factory) + monkeypatch.setattr(chat_routes, "_backfill_artifact_cache", lambda *_args: None) + monkeypatch.setattr( + chat_routes, + "_collect_historical_attachments", + lambda **_kwargs: [], + ) + + context = chat_routes._build_ctx( + ChatRequest(chat_id="ontology_disabled_chat", message="分析企业风险"), + "ontology_user", + [], + [], + [], + ontology_enabled=True, + ) + + assert context["ontology_enabled"] is False + assert context["ontology_runtime"] == { + "enabled": False, + "packs": [], + "review_level": "none", + } + def test_explicit_user_correction_enters_human_review_queue(db_session): from core.db.models import UserShadow diff --git a/src/backend/tests/services/test_site_service.py b/src/backend/tests/services/test_site_service.py deleted file mode 100644 index 82d0bebd..00000000 --- a/src/backend/tests/services/test_site_service.py +++ /dev/null @@ -1,234 +0,0 @@ -"""Site hosting SiteService unit tests (publish / fetch file / slug / quota / delete).""" - -import pytest - -from core.db.models import UserShadow -from core.infra.exceptions import BadRequestError, ResourceNotFoundError -from core.services import site_service as ss -from core.services.site_service import SiteService, normalize_rel_path, guess_site_mime - - -@pytest.fixture() -def user(db_session): - u = UserShadow(user_id="site_tester", username="site_tester") - db_session.add(u) - db_session.commit() - return u - - -@pytest.fixture() -def svc(db_session, tmp_path, monkeypatch): - # Store into tmp_path to avoid polluting the real STORAGE_PATH - monkeypatch.setenv("STORAGE_TYPE", "local") - monkeypatch.setenv("STORAGE_PATH", str(tmp_path)) - # LocalStorageBackend needs resetting if it is a module-level singleton cache; if get_storage builds a fresh one each time, no effect - return SiteService(db_session) - - -BASIC_FILES = [ - ("index.html", b"

hi

"), - ("css/style.css", b"h1{}"), -] - - -def test_publish_and_resolve(svc, user): - site = svc.publish(user_id=user.user_id, files=BASIC_FILES, title="T1") - assert site.slug.startswith("s-") - assert site.current_version == 1 - assert site.file_count == 2 - - got = svc.resolve_site_file(site, "") - assert got is not None - content, mime = got - assert content == b"

hi

" - assert mime.startswith("text/html") - - got = svc.resolve_site_file(site, "css/style.css") - assert got[0] == b"h1{}" - assert got[1].startswith("text/css") - - # SPA fallback: an extensionless path falls back to the entry file - got = svc.resolve_site_file(site, "some/route") - assert got[0] == b"

hi

" - - # A non-existent path with an extension → None - assert svc.resolve_site_file(site, "nope.png") is None - - -def test_publish_new_version_keeps_url(svc, user): - site = svc.publish(user_id=user.user_id, files=BASIC_FILES, title="T2", slug="my-site") - s2 = svc.publish( - user_id=user.user_id, - files=[("index.html", b"v2")], - title="T2v2", - site_id=site.site_id, - ) - assert s2.slug == "my-site" - assert s2.current_version == 2 - assert svc.resolve_site_file(s2, "")[0] == b"v2" - versions = (s2.extra_data or {}).get("versions") - assert [v["version"] for v in versions] == [1, 2] - - -def test_slug_rules(svc, user): - with pytest.raises(BadRequestError): - svc.publish(user_id=user.user_id, files=BASIC_FILES, title="x", slug="AB") - with pytest.raises(BadRequestError): - svc.publish(user_id=user.user_id, files=BASIC_FILES, title="x", slug="api") - svc.publish(user_id=user.user_id, files=BASIC_FILES, title="x", slug="taken-slug") - with pytest.raises(BadRequestError): - svc.publish(user_id=user.user_id, files=BASIC_FILES, title="x", slug="taken-slug") - - -def test_entry_file_required(svc, user): - with pytest.raises(BadRequestError): - svc.publish(user_id=user.user_id, files=[("a.css", b"x")], title="x") - # A sole html at the root can serve as the entry - site = svc.publish(user_id=user.user_id, files=[("main.html", b"

m

")], title="x") - assert site.entry_file == "main.html" - assert svc.resolve_site_file(site, "")[0] == b"

m

" - - -def test_path_safety(): - assert normalize_rel_path("../etc/passwd") is None - # After stripping the leading slash it is treated as an in-site path (request paths naturally have no leading /) - assert normalize_rel_path("/abs") == "abs" - assert normalize_rel_path("a/../../b") is None - assert normalize_rel_path("a\\b") is None - assert normalize_rel_path("a/./b") == "a/b" - - -def test_limits(svc, user, monkeypatch): - monkeypatch.setattr(ss, "MAX_SITE_FILES", 2) - with pytest.raises(BadRequestError): - svc.publish( - user_id=user.user_id, - files=[("index.html", b"x"), ("a.js", b"x"), ("b.js", b"x")], - title="x", - ) - - -def test_delete_frees_slug_and_permission(svc, user, db_session): - site = svc.publish(user_id=user.user_id, files=BASIC_FILES, title="del", slug="del-me") - with pytest.raises(ResourceNotFoundError): - svc.delete_site(site.site_id, "another_user") - svc.delete_site(site.site_id, user.user_id) - assert svc.repo.get_by_slug("del-me") is None - # slug is freed and can be claimed again - again = svc.publish(user_id=user.user_id, files=BASIC_FILES, title="del2", slug="del-me") - assert again.slug == "del-me" - - -def test_mime_guess(): - assert guess_site_mime("a.js").startswith("text/javascript") - assert guess_site_mime("a.svg").startswith("image/svg+xml") - assert guess_site_mime("a.woff2") == "font/woff2" - assert guess_site_mime("a.bin") == "application/octet-stream" - - -# ── Team visibility / rollback / KV / forms (roadmap 1/2/4/5) ──────────────── - -@pytest.fixture() -def team(db_session, user): - from core.db.models import Team, TeamMember, UserShadow - - mate = UserShadow(user_id="site_mate", username="site_mate") - outsider = UserShadow(user_id="site_outsider", username="site_outsider") - t = Team(team_id="team_site_test", name="站点测试团队") - db_session.add_all([mate, outsider, t]) - db_session.flush() - db_session.add_all([ - TeamMember(team_id=t.team_id, user_id=user.user_id, role="owner"), - TeamMember(team_id=t.team_id, user_id="site_mate", role="member"), - ]) - db_session.commit() - return t - - -def test_team_visibility_authorize(svc, user, team): - site = svc.publish( - user_id=user.user_id, files=BASIC_FILES, title="team site", visibility="team", - ) - assert site.team_id == team.team_id # when no team_id is passed, take the first team - assert svc.authorize_view(site, user.user_id) is True # site owner - assert svc.authorize_view(site, "site_mate") is True # team member - assert svc.authorize_view(site, "site_outsider") is False # non-member - assert svc.authorize_view(site, None) is False # anonymous - - -def test_team_visibility_requires_membership(svc, user, team): - with pytest.raises(BadRequestError): - svc.publish( - user_id="site_outsider", files=BASIC_FILES, title="x", - visibility="team", team_id=team.team_id, - ) - - -def test_rollback(svc, user): - site = svc.publish(user_id=user.user_id, files=[("index.html", b"v1")], title="rb") - svc.publish(user_id=user.user_id, files=[("index.html", b"v2")], title="rb", site_id=site.site_id) - s3 = svc.publish(user_id=user.user_id, files=[("index.html", b"v3")], title="rb", site_id=site.site_id) - assert s3.current_version == 3 - - rolled = svc.rollback(site.site_id, user.user_id, 2) - assert rolled.current_version == 2 - assert svc.resolve_site_file(rolled, "")[0] == b"v2" - # Publishing again after rollback → version number continues from the historical max (4, not 3) - s4 = svc.publish(user_id=user.user_id, files=[("index.html", b"v4")], title="rb", site_id=site.site_id) - assert s4.current_version == 4 - with pytest.raises(BadRequestError): - svc.rollback(site.site_id, user.user_id, 99) - - -def test_kv(svc, user): - site = svc.publish(user_id=user.user_id, files=BASIC_FILES, title="kv") - assert svc.kv_get(site, "score") is None - svc.kv_set(site, "score", "42") - assert svc.kv_get(site, "score") == "42" - svc.kv_set(site, "score", "43") - assert svc.kv_get(site, "score") == "43" - assert svc.kv_delete(site, "score") is True - assert svc.kv_get(site, "score") is None - with pytest.raises(BadRequestError): - svc.kv_set(site, "bad key!", "x") - with pytest.raises(BadRequestError): - svc.kv_set(site, "big", "x" * 5000) - - -def test_form_submissions_and_export(svc, user, db_session): - site = svc.publish(user_id=user.user_id, files=BASIC_FILES, title="表单站") - svc.submit_form(site, "contact", {"name": "张三", "msg": "你好"}, client_ip="1.2.3.4") - svc.submit_form(site, "contact", {"name": "李四", "phone": "138"}) - items, total = svc.repo.submission_list(site.site_id) - assert total == 2 - with pytest.raises(BadRequestError): - svc.submit_form(site, "bad key!", {"a": 1}) - with pytest.raises(BadRequestError): - svc.submit_form(site, "contact", {}) - - result = svc.export_submissions_to_artifact(site.site_id, user.user_id) - assert result["rows"] == 2 - assert result["filename"].endswith(".csv") - from core.db.models import Artifact - - row = db_session.query(Artifact).filter( - Artifact.artifact_id == result["artifact_id"] - ).first() - assert row is not None and row.user_id == user.user_id - - -def test_reserved_api_prefix(svc, user): - with pytest.raises(BadRequestError): - svc.publish( - user_id=user.user_id, - files=[("index.html", b"x"), ("__api/kv.js", b"x")], - title="x", - ) - - -def test_view_count_increment(svc, user, db_session): - site = svc.publish(user_id=user.user_id, files=BASIC_FILES, title="pv") - svc.repo.increment_view(site.site_id) - svc.repo.increment_view(site.site_id) - db_session.expire_all() - assert svc.repo.get_by_id(site.site_id).view_count == 2 diff --git a/src/backend/tests/services/user_folder_service_selftest.py b/src/backend/tests/services/user_folder_service_selftest.py index 8b4d8ea5..5cbd9259 100644 --- a/src/backend/tests/services/user_folder_service_selftest.py +++ b/src/backend/tests/services/user_folder_service_selftest.py @@ -1,4 +1,4 @@ -"""Selftest: UserFolderService — key-path verification mirroring TeamFolderService. +"""Selftest: UserFolderService key-path verification. How to run: PYTHONPATH=src/backend python -m tests.user_folder_service_selftest or run automatically as part of make selftest. @@ -13,9 +13,6 @@ def main() -> int: try: - from sqlalchemy import create_engine - from sqlalchemy.orm import sessionmaker - from core.db.engine import Base from core.db.models import Artifact, UserFolder, UserShadow from core.services.user_folder_service import ( @@ -23,6 +20,8 @@ def main() -> int: UserFolderService, _sanitize_name, ) + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker except ModuleNotFoundError as e: print(f"user_folder_service_selftest: SKIP (missing dependency: {e})") return 0 @@ -117,29 +116,33 @@ def main() -> int: # ── cascade soft delete ── # attach an artifact under f1 - db.add(Artifact( - artifact_id="a1", - user_id=uid, - user_folder_id=f1, - type="document", - title="t", - filename="t.txt", - size_bytes=1, - mime_type="text/plain", - storage_key="k", - )) + db.add( + Artifact( + artifact_id="a1", + user_id=uid, + user_folder_id=f1, + type="document", + title="t", + filename="t.txt", + size_bytes=1, + mime_type="text/plain", + storage_key="k", + ) + ) # also attach one under f1's child (already moved to f3) - db.add(Artifact( - artifact_id="a2", - user_id=uid, - user_folder_id=f2, # f2 already moved under f3 - type="document", - title="t2", - filename="t2.txt", - size_bytes=1, - mime_type="text/plain", - storage_key="k2", - )) + db.add( + Artifact( + artifact_id="a2", + user_id=uid, + user_folder_id=f2, # f2 already moved under f3 + type="document", + title="t2", + filename="t2.txt", + size_bytes=1, + mime_type="text/plain", + storage_key="k2", + ) + ) db.commit() cnt_f1 = svc.count_affected_artifacts(f1, uid) diff --git a/src/backend/tests/skills/test_registry.py b/src/backend/tests/skills/test_registry.py new file mode 100644 index 00000000..d8c5e42f --- /dev/null +++ b/src/backend/tests/skills/test_registry.py @@ -0,0 +1,25 @@ +"""Focused tests for the agent-skill SKILL.md parser.""" + +from core.agent_skills.registry import _load_skill_metadata_from_str, _split_frontmatter + + +def test_frontmatter_accepts_windows_crlf_line_endings(): + raw = ( + "---\r\n" + "name: windows-skill\r\n" + "description: A skill packaged on Windows.\r\n" + "version: 1.2.3\r\n" + "---\r\n" + "\r\n" + "# Windows skill\r\n" + "\r\n" + "Follow the instructions.\r\n" + ) + + frontmatter, body = _split_frontmatter(raw) + metadata = _load_skill_metadata_from_str(raw, "windows-skill") + + assert frontmatter["name"] == "windows-skill" + assert body.startswith("\n# Windows skill\n") + assert metadata.description == "A skill packaged on Windows." + assert metadata.version == "1.2.3" diff --git a/src/backend/tests/test_ce_agent_visibility.py b/src/backend/tests/test_ce_agent_visibility.py index 9878dbe2..bcde6090 100644 --- a/src/backend/tests/test_ce_agent_visibility.py +++ b/src/backend/tests/test_ce_agent_visibility.py @@ -1,7 +1,5 @@ """CE sub-agent visibility regression tests.""" -from types import SimpleNamespace - from core.db.edition_tables import ce_create_all from core.db.models import UserAgent, UserShadow from core.services.user_agent_service import UserAgentService @@ -9,13 +7,15 @@ from sqlalchemy.orm import sessionmaker -def test_ce_list_for_user_does_not_query_team_members(monkeypatch): - """A fresh CE database omits team tables, but chat still resolves visible agents.""" - from core.db.repository import agent as agent_repository +def test_ce_list_for_user_has_no_organization_repository_contracts(): + """A fresh CE database and repository expose only personal/admin agents.""" + from core.db.repository.agent import UserAgentRepository engine = create_engine("sqlite:///:memory:") ce_create_all(engine) assert inspect(engine).has_table("team_members") is False + assert not hasattr(UserAgentRepository, "count_team_agents") + assert not hasattr(UserAgentRepository, "list_for_team") session = sessionmaker(bind=engine)() try: @@ -54,12 +54,6 @@ def test_ce_list_for_user_does_not_query_team_members(monkeypatch): ] ) session.commit() - monkeypatch.setattr( - agent_repository, - "settings", - SimpleNamespace(edition=SimpleNamespace(is_ee=False)), - ) - agents = UserAgentService(session).list_for_user("ce-user") assert {item["agent_id"] for item in agents} == {"admin-enabled", "ce-personal"} diff --git a/src/backend/tests/test_ce_runtime_contracts.py b/src/backend/tests/test_ce_runtime_contracts.py index f58fbe76..4c395358 100644 --- a/src/backend/tests/test_ce_runtime_contracts.py +++ b/src/backend/tests/test_ce_runtime_contracts.py @@ -1,6 +1,8 @@ """Regression tests for CE-only runtime seams.""" +import inspect import json +import re from pathlib import Path from types import SimpleNamespace @@ -8,12 +10,94 @@ from api.routes.v1.users import router as users_router from cli import DEFAULT_LOCAL_CONTEXT_LENGTH, build_parser, configure_model from core.db import model_repository -from core.licensing import license_manager -def test_ce_license_never_blocks_business_routes(): - assert license_manager.mode() == "ce" - assert license_manager.is_active() is True +def test_ce_has_no_commercial_license_package(): + import importlib.util + + assert importlib.util.find_spec("core.licensing") is None + + +def test_ce_has_no_enterprise_modules_or_model_exports(): + import importlib.util + + import core.db.models as models + + for module_name in ( + "edition_ee", + "core.kb.dify_kb", + "core.auth.team_permissions", + "core.services.team_service", + "core.db.repository.team", + ): + assert importlib.util.find_spec(module_name) is None + for model_name in ( + "Team", + "TeamMember", + "TeamFolder", + "Role", + "RoleAssignment", + "InviteCode", + "MarketplaceVisibilityGrant", + "ChatSessionUserState", + ): + assert not hasattr(models, model_name) + + +def test_ce_runtime_sources_have_no_commercial_symbols(): + backend = Path(__file__).resolve().parents[1] + pattern = re.compile( + r"\b(?:TeamMember|TeamFolder|team_id|team_folder_id|" + r"linked_team_folder_id|share_scope|grant_team_ids|team_read|team_edit|" + r"list_team_files|stage_team_file|resolve_team_file_permission|" + r"require_team_file_permission|team_cache_dir|team_folders)\b|" + r"/v1/(?:me/teams|my-teams|teams)(?:/|\b)|\bkind=team\b" + ) + hits = [] + for relative_root in ("api", "core", "mcp_servers", "orchestration"): + for path in (backend / relative_root).rglob("*.py"): + for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if pattern.search(line): + hits.append(f"{path.relative_to(backend)}:{line_no}") + assert hits == [] + + +def test_ce_openapi_and_tables_have_no_organization_contracts(): + from api.app import app + from core.db.engine import Base + + openapi = app.openapi() + paths = set(openapi["paths"]) + assert "/v1/chats/{chat_id}/share" not in paths + assert not any( + path == prefix or path.startswith(prefix + "/") + for path in paths + for prefix in ("/v1/me/teams", "/v1/my-teams", "/v1/teams") + ) + openapi_text = json.dumps(openapi, ensure_ascii=False).lower() + for term in ( + "team_id", + "team_folder_id", + "share_scope", + "team_read", + "team_edit", + "团队", + ): + assert term not in openapi_text + agent_properties = openapi["components"]["schemas"]["AgentCreateRequest"]["properties"] + assert "team_id" not in agent_properties + kb_properties = openapi["components"]["schemas"]["CreateKBSpaceRequest"]["properties"] + assert "grant_team_ids" not in kb_properties + assert "chat_session_user_states" not in Base.metadata.tables + forbidden_columns = { + "artifacts": {"team_id", "team_folder_id"}, + "chat_sessions": {"share_scope"}, + "marketplace_listing_states": {"visibility"}, + "projects": {"team_id", "linked_team_folder_id"}, + "user_agents": {"team_id"}, + } + for table_name, names in forbidden_columns.items(): + assert not (set(Base.metadata.tables[table_name].columns.keys()) & names) def test_ce_auth_router_keeps_local_session_contract(): @@ -49,6 +133,7 @@ def test_dingtalk_status_env_does_not_import_ee_sandbox_module(monkeypatch, tmp_ def test_ce_ignores_stale_database_query_builtin(db_session, monkeypatch): + from core.config.catalog_runtime import get_runtime_catalog, invalidate_runtime_catalog_cache from core.db.models import AdminMcpServer from core.services import mcp_service from core.services.mcp_service import McpServerConfigService @@ -81,6 +166,100 @@ def test_ce_ignores_stale_database_query_builtin(db_session, monkeypatch): assert "query_database" not in servers assert "custom_remote_mcp" in servers + invalidate_runtime_catalog_cache() + catalog = get_runtime_catalog(db_session) + catalog_mcp_ids = {item["id"] for item in catalog["mcp"]} + assert "database_query" not in catalog_mcp_ids + assert "query_database" not in catalog_mcp_ids + assert "custom_remote_mcp" in catalog_mcp_ids + + assert mcp_service.prune_removed_builtin_mcp_servers(db_session) == ["query_database"] + assert db_session.get(AdminMcpServer, "query_database") is None + assert db_session.get(AdminMcpServer, "custom_remote_mcp") is not None + + +def test_ce_local_launcher_covers_default_plugin_mcp_servers(): + from cli import _DEFAULT_PLUGINS + from mcp_servers._launcher import PORTS as LAUNCHER_PORTS + from mcp_servers._ports import PORTS, package_name + + assert _DEFAULT_PLUGINS == ["automation", "skill-manager", "sites"] + assert { + "automation_task": 9108, + "skill_manager": 9112, + "site_publish": 9113, + }.items() <= PORTS.items() + assert package_name("site_publish") == "site_publish_mcp" + assert LAUNCHER_PORTS["site_publish_mcp"] == 9113 + + +def test_ce_fresh_database_bootstraps_default_plugins(db_session): + from core.db.models import AdminMcpServer, ContentBlock, InstalledPlugin + from core.services import plugin_service + + assert plugin_service.ensure_default_plugins_bootstrapped(db_session) is True + installs = db_session.query(InstalledPlugin).filter(InstalledPlugin.owner_user_id.is_(None)) + assert {row.slug for row in installs.all()} == { + "automation", + "skill-manager", + "sites", + } + assert { + row.source_plugin + for row in db_session.query(AdminMcpServer) + .filter(AdminMcpServer.source_plugin.is_not(None)) + .all() + } == {"automation", "skill-manager", "sites"} + marker = db_session.get(ContentBlock, plugin_service.DEFAULT_BOOTSTRAP_MARKER_ID) + assert marker.payload["plugins"] == ["automation", "skill-manager", "sites"] + assert plugin_service.ensure_default_plugins_bootstrapped(db_session) is False + + +def test_ce_launcher_and_catalog_cover_core_mcp_servers(): + from core.services.mcp_service import BUILTIN_MCP_SERVERS + from mcp_servers._launcher import PORTS as LAUNCHER_PORTS + from mcp_servers._ports import PORTS, package_name + + expected = { + "retrieve_dataset_content": 9100, + "internet_search": 9102, + "generate_chart_tool": 9104, + "web_fetch": 9106, + "batch_runner": 9107, + } + assert expected.items() <= PORTS.items() + assert { + package_name(server_id): port for server_id, port in expected.items() + }.items() <= LAUNCHER_PORTS.items() + + specs = {str(item["server_id"]): item for item in BUILTIN_MCP_SERVERS} + assert expected.keys() <= specs.keys() + assert all(specs[server_id]["is_enabled"] is True for server_id in expected) + + +def test_ce_fresh_database_seeds_core_mcp_servers_enabled(db_session): + from core.db.models import AdminMcpServer + from core.services.mcp_service import seed_builtin_mcp_servers_if_empty + + expected = { + "retrieve_dataset_content", + "internet_search", + "generate_chart_tool", + "web_fetch", + "batch_runner", + } + seeded = set(seed_builtin_mcp_servers_if_empty(db_session)) + rows = { + row.server_id: row + for row in db_session.query(AdminMcpServer) + .filter(AdminMcpServer.server_id.in_(expected)) + .all() + } + + assert expected <= seeded + assert expected == rows.keys() + assert all(row.is_enabled is True for row in rows.values()) + def test_onboard_cli_has_safe_context_window_default(): args = build_parser().parse_args(["onboard"]) @@ -156,7 +335,6 @@ def test_ce_bundles_compilable_default_ontology_pack(): def test_ce_schema_keeps_ontology_control_plane_tables(): import core.db.models # noqa: F401 register all model metadata - from core.db.edition_tables import EE_ONLY_TABLES from core.db.engine import Base ontology_tables = { @@ -168,4 +346,24 @@ def test_ce_schema_keeps_ontology_control_plane_tables(): } assert ontology_tables <= set(Base.metadata.tables) - assert ontology_tables.isdisjoint(EE_ONLY_TABLES) + + +def test_ce_schema_can_create_without_enterprise_foreign_keys(db_session): + """The fixture's create_all() is the assertion; keep one explicit DB touch.""" + assert db_session.execute(__import__("sqlalchemy").text("SELECT 1")).scalar_one() == 1 + + +def test_ce_site_contract_has_no_organization_scope(): + from api.app import app + from core.db.engine import Base + + schemas = app.openapi()["components"]["schemas"] + assert "team_id" not in schemas["UpdateSiteRequest"]["properties"] + assert "team_id" not in schemas["PublishBody"]["properties"] + assert "team_id" not in Base.metadata.tables["sites"].columns + + +def test_ce_site_publish_tool_has_no_organization_parameter(): + from mcp_servers.site_publish_mcp.server import publish_site + + assert "team_id" not in inspect.signature(publish_site).parameters diff --git a/src/backend/tests/test_default_plugin_bootstrap.py b/src/backend/tests/test_default_plugin_bootstrap.py new file mode 100644 index 00000000..89169bda --- /dev/null +++ b/src/backend/tests/test_default_plugin_bootstrap.py @@ -0,0 +1,93 @@ +"""CE first-boot defaults shared by browser/Compose deployments.""" + +import importlib +from types import SimpleNamespace + +import pytest +from core.db.models import AdminMcpServer, ContentBlock, InstalledPlugin +from core.services import plugin_service + + +def test_ce_default_plugins_bootstrap_once_and_are_globally_available(db_session): + assert plugin_service.ensure_default_plugins_bootstrapped(db_session) is True + + installs = ( + db_session.query(InstalledPlugin) + .filter(InstalledPlugin.owner_user_id.is_(None)) + .order_by(InstalledPlugin.slug) + .all() + ) + assert [row.slug for row in installs] == ["automation", "sites", "skill-manager"] + assert all(row.created_by == "system_bootstrap" for row in installs) + + plugin_mcps = { + row.source_plugin: row + for row in db_session.query(AdminMcpServer) + .filter(AdminMcpServer.owner_user_id.is_(None)) + .filter(AdminMcpServer.source_plugin.is_not(None)) + .all() + } + assert set(plugin_mcps) == {"automation", "skill-manager", "sites"} + assert all(row.is_enabled is True for row in plugin_mcps.values()) + + visible_to_user = plugin_service.list_installed( + db_session, + owner_user_id="fresh_ce_user", + include_global=True, + ) + assert {item["slug"] for item in visible_to_user} == { + "automation", + "skill-manager", + "sites", + } + assert all(item["enabled"] is True for item in visible_to_user) + + marker = db_session.get(ContentBlock, plugin_service.DEFAULT_BOOTSTRAP_MARKER_ID) + assert marker.payload == { + "version": 1, + "plugins": ["automation", "skill-manager", "sites"], + } + assert plugin_service.ensure_default_plugins_bootstrapped(db_session) is False + + +def test_default_plugin_marker_preserves_later_user_uninstall(db_session): + plugin_service.ensure_default_plugins_bootstrapped(db_session) + plugin_service.uninstall_plugin( + db_session, + "sites@global", + owner_user_id=None, + ) + + assert plugin_service.ensure_default_plugins_bootstrapped(db_session) is False + assert db_session.get(InstalledPlugin, "sites@global") is None + + +@pytest.mark.asyncio +async def test_ce_compose_startup_runs_default_plugin_bootstrap(monkeypatch): + app_module = importlib.import_module("api.app") + engine_module = importlib.import_module("core.db.engine") + calls = [] + + class FakeSession: + def close(self): + calls.append("close") + + session = FakeSession() + monkeypatch.setattr( + app_module, + "settings", + SimpleNamespace( + edition=SimpleNamespace(edition="ce"), + deploy=SimpleNamespace(is_local=False), + ), + ) + monkeypatch.setattr(engine_module, "SessionLocal", lambda: session) + monkeypatch.setattr( + plugin_service, + "ensure_default_plugins_bootstrapped", + lambda db: calls.append(db) or True, + ) + + await app_module._startup_seed_default_plugins() + + assert calls == [session, "close"] diff --git a/src/backend/tests/test_kb_permissions.py b/src/backend/tests/test_kb_permissions.py deleted file mode 100644 index da853e7c..00000000 --- a/src/backend/tests/test_kb_permissions.py +++ /dev/null @@ -1,201 +0,0 @@ -"""Knowledge base permission assignment system tests: resolver (hidden-by-default / allowlist) + grant repository + permission service. - -Hidden-by-default model (KB management has no visibility UI): - - Shared KBs are hidden from everyone by default; only granted users/teams can see them. - - private KBs belong to the owner only; owner/super-admin are always admin. - - Grant precedence: personal grants override team grants. -Create permissions split into two: can_create_private_kb / can_create_public_kb. -""" - -from types import SimpleNamespace - -import pytest -from core.auth import kb_permissions as kp -from core.db.models import KBSpace, Team, TeamMember, UserShadow -from core.db.repository import KBGrantRepository -from core.services.kb_permission_service import KBPermissionService - - -@pytest.fixture -def seeded(db_session): - db = db_session - db.add_all( - [ - UserShadow(user_id="u_owner", username="owner"), - UserShadow(user_id="u_alice", username="alice"), - UserShadow(user_id="u_bob", username="bob"), - UserShadow(user_id="u_admin", username="admin", extra_data={"role": "super_admin"}), - UserShadow(user_id="system_public_kb", username="系统公共"), - Team(team_id="t1", name="T1"), - TeamMember(team_id="t1", user_id="u_alice", role="member"), - KBSpace(kb_id="kb_pub", user_id="system_public_kb", name="Pub", visibility="public"), - KBSpace( - kb_id="kb_shared", user_id="system_public_kb", name="Shared", visibility="public" - ), - KBSpace(kb_id="kb_priv", user_id="u_owner", name="Priv", visibility="private"), - ] - ) - db.commit() - r = KBGrantRepository(db) - r.upsert("kb_shared", "local", "team", "t1", "edit", "u_admin") # team t1 → edit - r.upsert("kb_shared", "local", "user", "u_bob", "view", "u_admin") # bob → view - return db - - -# ── Hidden-by-default / allowlist ─────────────────────────────────────────────── - - -def test_unshared_kb_hidden_from_everyone(seeded): - # kb_pub has no grants → ordinary users cannot see it - assert "kb_pub" not in kp.get_accessible_local_kb_levels(seeded, "u_alice") - assert "kb_pub" not in kp.get_accessible_local_kb_levels(seeded, "u_bob") - assert kp.resolve_local_kb_level(seeded, "u_alice", "kb_pub") == "none" - - -def test_only_granted_can_see(seeded): - assert ( - kp.get_accessible_local_kb_levels(seeded, "u_alice")["kb_shared"] == "edit" - ) # inherited from team - assert kp.get_accessible_local_kb_levels(seeded, "u_bob")["kb_shared"] == "view" # direct grant - assert "kb_shared" not in kp.get_accessible_local_kb_levels( - seeded, "u_owner" - ) # not granted, not visible - - -def test_grant_then_revoke(seeded): - KBGrantRepository(seeded).upsert("kb_pub", "local", "user", "u_alice", "view", "u_admin") - assert kp.get_accessible_local_kb_levels(seeded, "u_alice")["kb_pub"] == "view" - # Revoke (replace all with empty) → invisible again - KBGrantRepository(seeded).replace_for_principal("user", "u_alice", []) - assert "kb_pub" not in kp.get_accessible_local_kb_levels(seeded, "u_alice") - - -def test_personal_overrides_team(seeded): - KBGrantRepository(seeded).upsert("kb_shared", "local", "user", "u_alice", "view", "u_admin") - assert kp.get_accessible_local_kb_levels(seeded, "u_alice")["kb_shared"] == "view" - assert kp.resolve_local_kb_level(seeded, "u_alice", "kb_shared") == "view" - - -def test_private_owner_only(seeded): - assert kp.get_accessible_local_kb_levels(seeded, "u_owner")["kb_priv"] == "admin" - assert "kb_priv" not in kp.get_accessible_local_kb_levels(seeded, "u_alice") - - -def test_super_admin_sees_all(seeded): - lv = kp.get_accessible_local_kb_levels(seeded, "u_admin") - assert lv == {"kb_pub": "admin", "kb_shared": "admin", "kb_priv": "admin"} - - -def test_filter_blocks_unauthorized(seeded): - assert kp.filter_accessible_kb_ids(seeded, "u_bob", ["kb_priv", "kb_shared", "kb_pub"]) == [ - "kb_shared" - ] - assert kp.filter_accessible_kb_ids(seeded, "u_owner", ["kb_shared"]) == [] - - -def test_has_kb_permission_ordering(): - assert kp.has_kb_permission("admin", "edit") - assert not kp.has_kb_permission("view", "edit") - assert not kp.has_kb_permission("none", "view") - - -# ── Service ───────────────────────────────────────────────────────────────────── - - -def test_service_list_grantable(seeded): - by_id = {r["resource_id"]: r for r in KBPermissionService(seeded).list_grantable_resources()} - assert "kb_pub" in by_id and "kb_shared" in by_id - assert "kb_priv" not in by_id - assert "visibility" not in by_id["kb_pub"] - - -def test_service_replace_and_get_principal_grants(seeded): - svc = KBPermissionService(seeded) - svc.replace_principal_grants( - "team", - "t1", - [ - {"resource_id": "kb_pub", "resource_type": "local", "level": "view"}, - ], - granted_by="config_admin", - ) - assert {g["resource_id"]: g["level"] for g in svc.get_principal_grants("team", "t1")} == { - "kb_pub": "view" - } - - -def test_capability_defaults_split_create_perms(): - from core.auth.capabilities import BOOL_CAPABILITY_DEFAULTS - - assert BOOL_CAPABILITY_DEFAULTS.get("can_create_private_kb") is False - assert BOOL_CAPABILITY_DEFAULTS.get("can_create_public_kb") is False - assert "can_create_kb" not in BOOL_CAPABILITY_DEFAULTS - - -def test_ce_forces_public_kb_capability_off_for_existing_super_admin(seeded, monkeypatch): - import core.auth.capabilities as capabilities - - monkeypatch.setattr( - capabilities, - "settings", - SimpleNamespace(edition=SimpleNamespace(edition="ce")), - ) - caps = capabilities.resolve_user_capabilities(seeded, "u_admin") - assert caps["can_create_public_kb"] is False - assert capabilities.user_has_capability(seeded, "u_admin", "can_create_public_kb") is False - - -@pytest.mark.asyncio -async def test_ce_rejects_public_knowledge_base_creation(db_session, monkeypatch): - import api.routes.v1.kb as kb_routes - from api.routes.v1.kb_models import CreateKBSpaceRequest - from core.auth.backend import UserContext - from core.infra.exceptions import BadRequestError - - monkeypatch.setattr( - kb_routes, - "settings", - SimpleNamespace(edition=SimpleNamespace(edition="ce")), - ) - request = CreateKBSpaceRequest(name="Public", visibility="public") - user = UserContext(user_id="u_owner", user_center_id="u_owner", username="owner") - - with pytest.raises(BadRequestError, match="CE 仅支持私有知识库"): - await kb_routes.create_kb_space(request=request, user=user, db=db_session) - - -@pytest.mark.asyncio -async def test_ce_catalog_returns_only_owned_private_knowledge_bases(seeded, monkeypatch): - import api.routes.v1.catalog as catalog_routes - from core.auth.backend import UserContext - - seeded.add_all( - [ - KBSpace(kb_id="kb_admin_private", user_id="u_admin", name="Mine", visibility="private"), - KBSpace(kb_id="kb_admin_public", user_id="u_admin", name="Shared", visibility="public"), - ] - ) - seeded.commit() - monkeypatch.setattr( - catalog_routes, - "settings", - SimpleNamespace(edition=SimpleNamespace(edition="ce")), - ) - monkeypatch.setattr( - catalog_routes, - "get_runtime_catalog", - lambda _db: {"skills": [], "agents": [], "mcp": []}, - ) - monkeypatch.setattr(catalog_routes, "_load_owned_capability_items", lambda *_args: ([], [])) - monkeypatch.setattr(catalog_routes, "_plugin_component_ids", lambda *_args: (set(), set())) - monkeypatch.setattr(catalog_routes, "is_dify_enabled", lambda: True) - monkeypatch.setattr( - catalog_routes, - "_list_datasets_cached", - lambda: [{"id": "dify_public", "name": "Dify Public"}], - ) - - user = UserContext(user_id="u_admin", user_center_id="u_admin", username="admin") - response = await catalog_routes.get_catalog_items(user=user, db=seeded) - - assert [item["id"] for item in response["data"]["kb"]] == ["kb_admin_private"] diff --git a/src/backend/tests/test_local_memory_defaults.py b/src/backend/tests/test_local_memory_defaults.py index fe628468..4c49f0ff 100644 --- a/src/backend/tests/test_local_memory_defaults.py +++ b/src/backend/tests/test_local_memory_defaults.py @@ -1,6 +1,8 @@ """Local one-command profile memory defaults.""" +import os from pathlib import Path +from types import SimpleNamespace import cli import pytest @@ -9,10 +11,74 @@ def test_local_profile_enables_memory_runtime_by_default(tmp_path, monkeypatch): monkeypatch.setenv("HUGAGENT_HOME", str(tmp_path / "hugagent-home")) monkeypatch.delenv("MEM0_ENABLED", raising=False) + monkeypatch.delenv("HUGAGENT_BOOTSTRAP_DEFAULT_PLUGINS", raising=False) defaults = cli.apply_local_env(port=18000) assert defaults["MEM0_ENABLED"] == "true" + assert defaults["HUGAGENT_BOOTSTRAP_DEFAULT_PLUGINS"] == "1" + assert os.environ["HUGAGENT_BOOTSTRAP_DEFAULT_PLUGINS"] == "1" + + +def test_local_bootstrap_installs_recommended_plugins_only_once(tmp_path, monkeypatch): + home = tmp_path / "hugagent-home" + monkeypatch.setenv("HUGAGENT_HOME", str(home)) + cli.apply_local_env(port=18000) + installs = [] + provisions = [] + + def fake_install(slugs): + installs.append(list(slugs)) + return list(slugs) + + monkeypatch.setattr(cli, "install_plugins", fake_install) + monkeypatch.setattr( + cli, + "provision_site_template", + lambda verbose=False: provisions.append(verbose) or True, + ) + + assert cli.ensure_default_plugins_once() is True + assert cli.ensure_default_plugins_once() is False + assert installs == [["automation", "skill-manager", "sites"]] + assert provisions == [True, False] + assert (home / ".default-plugins-v1").read_text(encoding="utf-8").splitlines() == [ + "automation", + "skill-manager", + "sites", + ] + + +def test_local_bootstrap_retries_after_partial_plugin_failure(tmp_path, monkeypatch): + home = tmp_path / "hugagent-home" + monkeypatch.setenv("HUGAGENT_HOME", str(home)) + cli.apply_local_env(port=18000) + monkeypatch.setattr(cli, "install_plugins", lambda _slugs: ["automation", "skill-manager"]) + + with pytest.raises(RuntimeError, match="sites"): + cli.ensure_default_plugins_once() + + assert not (home / ".default-plugins-v1").exists() + + +def test_local_serve_fails_readiness_when_default_plugin_bootstrap_fails(monkeypatch): + monkeypatch.setenv("HUGAGENT_BOOTSTRAP_DEFAULT_PLUGINS", "1") + monkeypatch.setattr(cli, "apply_local_env", lambda _port: {}) + monkeypatch.setattr(cli, "_ensure_schema_and_seed", lambda: None) + + def fail_bootstrap(): + raise RuntimeError("sites missing") + + monkeypatch.setattr(cli, "ensure_default_plugins_once", fail_bootstrap) + + with pytest.raises(RuntimeError, match="sites missing"): + cli.cmd_serve( + SimpleNamespace( + port=18000, + host="127.0.0.1", + no_browser=True, + ) + ) def test_ce_installer_pins_compatible_milvus_lite_stack(): @@ -32,6 +98,26 @@ def test_ce_installer_pins_compatible_milvus_lite_stack(): assert "pymilvus[milvus-lite]>=2.5.0" not in installer +def test_ce_one_command_installer_bootstraps_default_plugins(): + repo_root = Path(__file__).resolve().parents[3] + installer_path = repo_root / "ce" / "overlay" / "install.sh" + if not installer_path.is_file(): + installer_path = repo_root / "install.sh" + + installer = installer_path.read_text(encoding="utf-8") + + assert "export HUGAGENT_BOOTSTRAP_DEFAULT_PLUGINS=1" in installer + + +def test_desktop_local_server_bootstraps_default_plugins(): + repo_root = Path(__file__).resolve().parents[3] + launcher = (repo_root / "desktop" / "src-tauri" / "src" / "local_server.rs").read_text( + encoding="utf-8" + ) + + assert '.env("HUGAGENT_BOOTSTRAP_DEFAULT_PLUGINS", "1")' in launcher + + @pytest.mark.parametrize( "relative_path", [ diff --git a/src/backend/tests/test_local_profile.py b/src/backend/tests/test_local_profile.py index 5f97775d..060814ae 100644 --- a/src/backend/tests/test_local_profile.py +++ b/src/backend/tests/test_local_profile.py @@ -11,7 +11,6 @@ """ import asyncio -from contextlib import nullcontext from pathlib import Path from types import SimpleNamespace @@ -54,14 +53,14 @@ async def writer(): def test_bigint_pk_autoincrements_on_sqlite(db_session): - """AuditLog.log_id (BigIntPK) must autoincrement under SQLite, not stay NULL.""" - from core.db.models import AuditLog + """A CE-safe BigIntPK must autoincrement under SQLite, not stay NULL.""" + from core.db.models import MemorySanitizerRule - row = AuditLog(action="unit.test") + row = MemorySanitizerRule(rule_type="classified", pattern="unit-test") db_session.add(row) db_session.commit() db_session.refresh(row) - assert row.log_id is not None and row.log_id >= 1 + assert row.id is not None and row.id >= 1 # ── Built-in MCP catalog seed ──────────────────────────────────────────────── @@ -226,25 +225,14 @@ def test_ce_web_onboarding_requires_main_model_and_clears_checkpoint(db_session, assert shadow.extra_data["onboarding_completed_version"] == 1 -def test_ce_optional_permission_layers_use_savepoints(): - from core.auth.capabilities import team_default_permissions_for_user - from core.auth.role_permissions import role_permissions_for_user +def test_ce_has_no_optional_organization_permission_layers(): + import importlib.util - class MissingOptionalTablesSession: - def __init__(self): - self.savepoints = 0 + from core.auth.edition_capabilities import default_capability_layers_for_user - def begin_nested(self): - self.savepoints += 1 - return nullcontext() - - def query(self, *_args, **_kwargs): - raise RuntimeError("optional CE table is absent") - - db = MissingOptionalTablesSession() - assert role_permissions_for_user(db, "user_ce_admin") == {} - assert team_default_permissions_for_user(db, "user_ce_admin") == {} - assert db.savepoints == 2 + assert importlib.util.find_spec("core.auth.role_permissions") is None + assert importlib.util.find_spec("core.auth.team_permissions") is None + assert default_capability_layers_for_user(None, "user_ce_admin") == () def test_ce_branding_repairs_persistent_page_config(db_session, monkeypatch): diff --git a/src/backend/tests/test_local_subprocess.py b/src/backend/tests/test_local_subprocess.py new file mode 100644 index 00000000..e9f39332 --- /dev/null +++ b/src/backend/tests/test_local_subprocess.py @@ -0,0 +1,87 @@ +"""Local/desktop sidecar startup contracts.""" + +from types import SimpleNamespace + +import pytest +from mcp_servers import _serve +from orchestration import local_subprocess + + +def test_child_env_binds_local_mcp_to_loopback(monkeypatch): + monkeypatch.setenv("MCP_HOST", "mcp") + monkeypatch.delenv("MCP_BIND_HOST", raising=False) + + env = local_subprocess._child_env() + + assert env["MCP_HOST"] == "127.0.0.1" + assert env["MCP_BIND_HOST"] == "127.0.0.1" + + +def test_streamable_http_bind_host_defaults_to_compose_and_supports_local(monkeypatch): + monkeypatch.delenv("MCP_BIND_HOST", raising=False) + assert _serve._streamable_http_bind_host() == "0.0.0.0" + + monkeypatch.setenv("MCP_BIND_HOST", "127.0.0.1") + assert _serve._streamable_http_bind_host() == "127.0.0.1" + + +def test_required_default_plugin_servers_are_launchable(): + from mcp_servers._launcher import PORTS as launcher_ports + from mcp_servers._ports import PORTS, package_name + + for server_id, expected_tool in local_subprocess._REQUIRED_PLUGIN_MCP_TOOLS.items(): + assert expected_tool + assert server_id in PORTS + assert launcher_ports[package_name(server_id)] == PORTS[server_id] + + +@pytest.mark.asyncio +async def test_local_start_waits_for_ports_and_verifies_plugin_tools(monkeypatch): + calls = [] + + class DummyProcess: + returncode = None + pid = 42 + + async def fake_spawn(label, argv): + calls.append(("spawn", label, tuple(argv))) + return DummyProcess() + + async def fake_wait(launcher, ports, *, timeout): + calls.append(("wait", launcher.pid, dict(ports), timeout)) + + async def fake_verify(ports): + calls.append(("verify", dict(ports))) + + monkeypatch.setattr( + local_subprocess, + "settings", + SimpleNamespace( + deploy=SimpleNamespace(is_local=True), + sandbox=SimpleNamespace(provider="script_runner"), + ), + ) + monkeypatch.setattr(local_subprocess, "_spawn", fake_spawn) + monkeypatch.setattr(local_subprocess, "_wait_for_mcp_ports", fake_wait) + monkeypatch.setattr(local_subprocess, "_verify_required_plugin_tools", fake_verify) + + await local_subprocess.start_local_sidecars() + + assert [call[1] for call in calls if call[0] == "spawn"] == [ + "mcp_launcher", + "script_runner", + ] + waited_ports = next(call[2] for call in calls if call[0] == "wait") + assert set(local_subprocess._REQUIRED_PLUGIN_MCP_TOOLS) <= set(waited_ports) + assert any(call[0] == "verify" for call in calls) + + +@pytest.mark.asyncio +async def test_required_plugin_tool_contract_rejects_missing_registration(): + with pytest.raises(RuntimeError, match="site_publish"): + await local_subprocess._verify_required_plugin_tools( + { + "automation_task": 9108, + "skill_manager": 9112, + } + ) diff --git a/src/backend/tests/test_marketplace_visibility.py b/src/backend/tests/test_marketplace_visibility.py deleted file mode 100644 index cb68fc26..00000000 --- a/src/backend/tests/test_marketplace_visibility.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Marketplace item visibility scope: public/scoped settings → user/team/role three-principal resolution → list/single-item filtering.""" - -from __future__ import annotations - -import pytest - -from core.auth.marketplace_visibility import get_hidden_item_ids, is_item_visible -from core.db.models import ( - MarketplaceVisibilityGrant, - Role, - RoleAssignment, - Team, - TeamMember, - UserShadow, -) -from core.infra.exceptions import BadRequestError, ResourceNotFoundError -from core.services import marketplace_listing as ml - - -def _seed(db): - """u_admin=super admin; u_team ∈ t1; u_role directly assigned r1; u_trole ∈ t2 (t2 has department default role r2); u_none has no affiliation.""" - db.add(UserShadow(user_id="u_admin", username="admin", extra_data={"role": "super_admin"})) - for uid in ("u_team", "u_role", "u_trole", "u_none"): - db.add(UserShadow(user_id=uid, username=uid, extra_data={})) - db.add(Team(team_id="t1", name="团队一", owner_user_id="u_team")) - db.add(Team(team_id="t2", name="团队二", owner_user_id="u_trole")) - db.add(TeamMember(team_id="t1", user_id="u_team", role="member")) - db.add(TeamMember(team_id="t2", user_id="u_trole", role="member")) - db.add(Role(role_id="r1", name="分析师", permissions={})) - db.add(Role(role_id="r2", name="研发", permissions={})) - db.add(RoleAssignment(role_id="r1", principal_type="user", principal_id="u_role")) - db.add(RoleAssignment(role_id="r2", principal_type="team", principal_id="t2")) - db.commit() - - -def test_default_public_everyone_visible(db_session): - db = db_session - _seed(db) - # Missing row = public: with no scoped item there are no hidden items for anyone - assert get_hidden_item_ids(db, ml.KIND_SKILL, "u_none") == set() - assert is_item_visible(db, ml.KIND_SKILL, "any-skill", "u_none") - vis = ml.get_listing_visibility(db, ml.KIND_SKILL, "any-skill") - assert vis["visibility"] == "public" and vis["grants"] == [] - - -def test_scoped_user_team_role_grants(db_session): - db = db_session - _seed(db) - ml.set_listing_visibility( - db, ml.KIND_SKILL, "s1", - visibility="scoped", - grants=[ - {"principal_type": "user", "principal_id": "u_role"}, - {"principal_type": "team", "principal_id": "t1"}, - {"principal_type": "role", "principal_id": "r2"}, - ], - updated_by="admin", - ) - # Personal grant / team member / role acquired via team (t2→r2) → visible; unaffiliated → not visible - assert is_item_visible(db, ml.KIND_SKILL, "s1", "u_role") - assert is_item_visible(db, ml.KIND_SKILL, "s1", "u_team") - assert is_item_visible(db, ml.KIND_SKILL, "s1", "u_trole") - assert not is_item_visible(db, ml.KIND_SKILL, "s1", "u_none") - # Super admin always visible; anonymous (empty user_id) not visible - assert is_item_visible(db, ml.KIND_SKILL, "s1", "u_admin") - assert not is_item_visible(db, ml.KIND_SKILL, "s1", None) - # kind isolation: the same item_id in a different marketplace is unaffected - assert is_item_visible(db, ml.KIND_PLUGIN, "s1", "u_none") - - -def test_role_direct_assignment(db_session): - db = db_session - _seed(db) - ml.set_listing_visibility( - db, ml.KIND_AGENT, "a1", - visibility="scoped", - grants=[{"principal_type": "role", "principal_id": "r1"}], - ) - # r1 is directly assigned to the individual u_role - assert is_item_visible(db, ml.KIND_AGENT, "a1", "u_role") - assert not is_item_visible(db, ml.KIND_AGENT, "a1", "u_team") - - -def test_annotate_and_filter_visibility(db_session): - db = db_session - _seed(db) - ml.set_listing_visibility( - db, ml.KIND_PLUGIN, "p1", - visibility="scoped", - grants=[{"principal_type": "user", "principal_id": "u_team"}], - ) - items = [{"slug": "p1"}, {"slug": "p2"}] - - # User side: an ungranted user cannot see p1 - out = ml.annotate_and_filter( - db, ml.KIND_PLUGIN, [dict(i) for i in items], - id_key="slug", include_disabled=False, viewer_user_id="u_none", - ) - assert [i["slug"] for i in out] == ["p2"] - assert out[0]["visibility"] == "public" - - # User side: a granted user can see it, annotated as scoped - out = ml.annotate_and_filter( - db, ml.KIND_PLUGIN, [dict(i) for i in items], - id_key="slug", include_disabled=False, viewer_user_id="u_team", - ) - assert {i["slug"]: i["visibility"] for i in out} == {"p1": "scoped", "p2": "public"} - - # Admin side: no filtering, all visible + visibility annotation - out = ml.annotate_and_filter( - db, ml.KIND_PLUGIN, [dict(i) for i in items], - id_key="slug", include_disabled=True, - ) - assert {i["slug"]: i["visibility"] for i in out} == {"p1": "scoped", "p2": "public"} - - -def test_set_back_to_public_clears_grants(db_session): - db = db_session - _seed(db) - ml.set_listing_visibility( - db, ml.KIND_SKILL, "s2", - visibility="scoped", - grants=[{"principal_type": "user", "principal_id": "u_role"}], - ) - assert not is_item_visible(db, ml.KIND_SKILL, "s2", "u_none") - ml.set_listing_visibility(db, ml.KIND_SKILL, "s2", visibility="public") - assert is_item_visible(db, ml.KIND_SKILL, "s2", "u_none") - assert db.query(MarketplaceVisibilityGrant).filter_by(kind=ml.KIND_SKILL, item_id="s2").count() == 0 - # The enable/disable switch is not broken by visibility-scope settings (missing row defaults to enabled → still enabled after upsert) - assert ml.get_disabled_ids(db, ml.KIND_SKILL) == set() - - -def test_set_visibility_validation(db_session): - db = db_session - _seed(db) - with pytest.raises(BadRequestError): - ml.set_listing_visibility(db, ml.KIND_SKILL, "s3", visibility="secret") - with pytest.raises(BadRequestError): - ml.set_listing_visibility(db, ml.KIND_SKILL, "s3", visibility="scoped", grants=[]) - with pytest.raises(BadRequestError): - ml.set_listing_visibility( - db, ml.KIND_SKILL, "s3", - visibility="scoped", grants=[{"principal_type": "dept", "principal_id": "x"}], - ) - # Duplicate grants are deduplicated - res = ml.set_listing_visibility( - db, ml.KIND_SKILL, "s3", - visibility="scoped", - grants=[ - {"principal_type": "user", "principal_id": "u_role"}, - {"principal_type": "user", "principal_id": "u_role"}, - ], - ) - assert len(res["grants"]) == 1 - - -def test_ensure_item_visible_guard(db_session): - db = db_session - _seed(db) - ml.set_listing_visibility( - db, ml.KIND_AGENT, "a2", - visibility="scoped", - grants=[{"principal_type": "team", "principal_id": "t1"}], - ) - ml.ensure_item_visible(db, ml.KIND_AGENT, "a2", "u_team", resource="marketplace_agent") - with pytest.raises(ResourceNotFoundError): - ml.ensure_item_visible(db, ml.KIND_AGENT, "a2", "u_none", resource="marketplace_agent") diff --git a/src/backend/tests/test_oa_sso_service.py b/src/backend/tests/test_oa_sso_service.py deleted file mode 100644 index f3e41987..00000000 --- a/src/backend/tests/test_oa_sso_service.py +++ /dev/null @@ -1,210 +0,0 @@ -"""OA single sign-on service-layer tests: HMAC signature verification + auto account creation (random strong password) + dept_id team binding.""" - -import dataclasses -import hashlib -import hmac -import time - -import pytest - -from core.config import settings as settings_mod -from core.db.models import LocalUser, Team, TeamMember, UserShadow -from core.services.oa_sso_service import OASsoError, OASsoService, verify_signature - - -@pytest.fixture -def oa_cfg(): - """Temporarily replace settings.oa_sso (a frozen dataclass, bypassed via replace + __setattr__).""" - original = settings_mod.settings.oa_sso - - def apply(**kw): - new = dataclasses.replace(original, **kw) - object.__setattr__(settings_mod.settings, "oa_sso", new) - return new - - yield apply - object.__setattr__(settings_mod.settings, "oa_sso", original) - - -# ── HMAC signature verification ───────────────────────────────────────────── - -def _sign(secret: str, user_id: str, dept_id: str, ts: str, nonce: str) -> str: - base = "\n".join([user_id, dept_id, ts, nonce]) - return hmac.new(secret.encode(), base.encode(), hashlib.sha256).hexdigest() - - -def test_signature_skipped_when_no_secret(oa_cfg): - oa_cfg(sign_secret="") - # No exception means it is allowed through (intranet debugging only) - verify_signature(user_id="u1", dept_id="d1", timestamp="", nonce="", signature="") - - -def test_signature_valid_and_invalid(oa_cfg): - secret = "topsecret" - oa_cfg(sign_secret=secret, sign_ttl_seconds=300) - - ts = str(int(time.time())) - good = _sign(secret, "2031613182211670018", "D100", ts, "n1") - verify_signature(user_id="2031613182211670018", dept_id="D100", timestamp=ts, nonce="n1", signature=good) - - with pytest.raises(OASsoError): - verify_signature(user_id="2031613182211670018", dept_id="D100", timestamp=ts, nonce="n1", signature="deadbeef") - - -def test_signature_expired(oa_cfg): - secret = "topsecret" - oa_cfg(sign_secret=secret, sign_ttl_seconds=60) - - old_ts = str(int(time.time()) - 600) - sig = _sign(secret, "u1", "", old_ts, "") - with pytest.raises(OASsoError): - verify_signature(user_id="u1", dept_id="", timestamp=old_ts, nonce="", signature=sig) - - -# ── Auto account creation + team binding ──────────────────────────────────── - -def test_provision_creates_local_account_with_strong_password(db_session, oa_cfg): - oa_cfg(default_role="member") - oa_uid = "2031613182211670018" - - service = OASsoService(db_session) - user, team_id, created = service.provision(oa_user_id=oa_uid, dept_id="D100") - db_session.commit() - - assert created is True - # Account name = OA user_id; user_center_id also uses OA user_id as the idempotency key - assert user.username == oa_uid - assert user.user_center_id == oa_uid - assert (user.extra_data or {}).get("auth_source") == "oa_sso" - - # A real local account was created, and the password is a non-empty strong hash (never the plaintext user_id) - local = db_session.query(LocalUser).filter(LocalUser.user_id == user.user_id).first() - assert local is not None - assert local.status == "active" - assert local.password_hash and oa_uid not in local.password_hash - - # Auto-create a team by dept_id + default member - assert team_id is not None - team = db_session.query(Team).filter(Team.team_id == team_id).first() - assert team.sso_department == "D100" - assert team.source == "sso_auto" - member = ( - db_session.query(TeamMember) - .filter(TeamMember.team_id == team_id, TeamMember.user_id == user.user_id) - .first() - ) - assert member.role == "member" - - -def test_provision_applies_team_default_roles(db_session, oa_cfg): - """When OA direct-push auto-creates a team, the is_team_default roles should be attached to that team (the fix).""" - from core.db.models import Role, RoleAssignment - - oa_cfg(default_role="member") - # One default role + one non-default role, confirm only the former is attached - db_session.add(Role(role_id="role_def", name="默认部门角色", permissions={}, is_team_default=True)) - db_session.add(Role(role_id="role_plain", name="普通角色", permissions={}, is_team_default=False)) - db_session.commit() - - service = OASsoService(db_session) - user, team_id, created = service.provision(oa_user_id="oa_roleuser", dept_id="D200") - db_session.commit() - - assert created is True and team_id is not None - team_role_ids = { - rid - for (rid,) in db_session.query(RoleAssignment.role_id) - .filter( - RoleAssignment.principal_type == "team", - RoleAssignment.principal_id == team_id, - ) - .all() - } - assert team_role_ids == {"role_def"} - - -def test_provision_is_idempotent(db_session, oa_cfg): - oa_cfg(default_role="member") - oa_uid = "oa_999" - - service = OASsoService(db_session) - user1, team1, created1 = service.provision(oa_user_id=oa_uid, dept_id="D100") - db_session.commit() - user2, team2, created2 = service.provision(oa_user_id=oa_uid, dept_id="D100") - db_session.commit() - - assert created1 is True and created2 is False - assert user1.user_id == user2.user_id # No duplicate account creation - assert team1 == team2 - - # No duplicate account creation / no duplicate team join - assert db_session.query(UserShadow).filter(UserShadow.user_center_id == oa_uid).count() == 1 - assert ( - db_session.query(TeamMember) - .filter(TeamMember.team_id == team1, TeamMember.user_id == user1.user_id) - .count() - == 1 - ) - - -def test_provision_without_dept_skips_team(db_session, oa_cfg): - oa_cfg(default_role="member") - service = OASsoService(db_session) - user, team_id, created = service.provision(oa_user_id="oa_nodept", dept_id=None) - db_session.commit() - - assert created is True - assert team_id is None - assert db_session.query(TeamMember).filter(TeamMember.user_id == user.user_id).count() == 0 - - -def test_provision_username_collision_gets_suffixed(db_session, oa_cfg): - oa_cfg(default_role="member") - # Pre-occupy username = "dup" (different user_center_id) - db_session.add(UserShadow(user_id="user_pre", user_center_id="pre_center", username="dup")) - db_session.commit() - - service = OASsoService(db_session) - user, _, created = service.provision(oa_user_id="dup", dept_id=None) - db_session.commit() - - assert created is True - assert user.user_center_id == "dup" - assert user.username != "dup" # Suffixed after a collision - assert user.username.startswith("dup") - - -def test_provision_missing_user_id_rejected(db_session, oa_cfg): - service = OASsoService(db_session) - with pytest.raises(OASsoError) as ei: - service.provision(oa_user_id=" ", dept_id="D100") - assert ei.value.status_code == 400 - - -# ── One-time ticket (redirect login token exchange) ───────────────────────── - -async def test_ticket_is_single_use(monkeypatch): - from core.auth import oa_ticket_store - from core.auth.oa_ticket_store import consume_ticket, issue_ticket - - monkeypatch.setattr(oa_ticket_store, "_use_memory_store", lambda: True) - - ticket = await issue_ticket({"user_id": "user_1", "dept_id": "D100"}) - first = await consume_ticket(ticket) - assert first is not None - assert first["user_id"] == "user_1" - assert first["dept_id"] == "D100" - - # Single use: the second attempt must come up empty (replay protection) - second = await consume_ticket(ticket) - assert second is None - - -async def test_ticket_invalid_returns_none(monkeypatch): - from core.auth import oa_ticket_store - from core.auth.oa_ticket_store import consume_ticket - - monkeypatch.setattr(oa_ticket_store, "_use_memory_store", lambda: True) - - assert await consume_ticket("") is None - assert await consume_ticket("not-a-real-ticket") is None diff --git a/src/backend/tests/test_role_capabilities.py b/src/backend/tests/test_role_capabilities.py deleted file mode 100644 index fc6320cf..00000000 --- a/src/backend/tests/test_role_capabilities.py +++ /dev/null @@ -1,202 +0,0 @@ -"""Tests for role capability packs + four-layer resolution (personal explicit → role union → team default → system default).""" - -from __future__ import annotations - -from core.auth.capabilities import resolve_capabilities, resolve_user_capabilities -from core.auth.role_permissions import ( - merge_role_permissions, - normalize_role_permissions, - role_permissions_for_user, -) -from core.db.models import Role, RoleAssignment, Team, TeamMember, UserShadow -from core.services.role_service import RoleService - - -# ── Pure functions: role normalization / merge ──────────────────────────────────── -def test_normalize_role_only_keeps_granted_true(): - n = normalize_role_permissions( - {"can_add_skill": True, "can_add_mcp": False, "junk": "x", "allowed_apps": ["a", "a", "b"]} - ) - assert n == {"can_add_skill": True, "allowed_apps": ["a", "b"]} - assert normalize_role_permissions(None) == {} - - -def test_merge_role_union(): - m = merge_role_permissions([ - {"can_add_skill": True, "allowed_apps": ["a"]}, - {"can_use_api_key": True, "allowed_apps": ["b"]}, - ]) - assert m["can_add_skill"] is True and m["can_use_api_key"] is True - assert m["allowed_apps"] == ["a", "b"] - - -# ── Pure functions: four-layer fall-through ──────────────────────────────────── -def test_resolve_role_beats_team(): - # Role grants True, team default False → role wins (the role layer sits above the team layer) - r = resolve_capabilities({}, {"can_use_api_key": True}, {"can_use_api_key": False}) - assert r["can_use_api_key"] is True - - -def test_resolve_personal_off_beats_role_on(): - r = resolve_capabilities({"can_add_skill": False}, {"can_add_skill": True}, {}) - assert r["can_add_skill"] is False - - -def test_resolve_team_fills_when_role_silent(): - # Role is silent, team default on → team wins - r = resolve_capabilities({}, {}, {"can_add_skill": True}) - assert r["can_add_skill"] is True - - -def test_resolve_allowed_apps_role_layer_priority(): - r = resolve_capabilities({}, {"allowed_apps": ["a", "b"]}, {"allowed_apps": ["c"]}) - assert r["allowed_apps"] == ["a", "b"] - - -def test_personal_all_apps_overrides_team_restriction(): - from core.auth.capabilities import ALL_APPS - # Team restricts to 3 apps, personal "force all" sentinel → final is all (None), overriding the team restriction - r = resolve_capabilities({"allowed_apps": ALL_APPS}, {}, {"allowed_apps": ["a", "b"]}) - assert r["allowed_apps"] is None - # Personal does not set allowed_apps → follows the team restriction - r2 = resolve_capabilities({}, {}, {"allowed_apps": ["a", "b"]}) - assert r2["allowed_apps"] == ["a", "b"] - - -def test_backward_compat_single_layer(): - # Old signature resolve_capabilities(meta, team_defaults) semantics unchanged - r = resolve_capabilities({}, {"can_add_skill": True}) - assert r["can_add_skill"] is True and r["can_use_api_key"] is False - - -# ── DB integration ──────────────────────────────────────────────────── -def _mk_user(db, uid: str, meta: dict | None = None) -> None: - db.add(UserShadow(user_id=uid, username=uid, extra_data=meta or {})) - - -def _mk_role(db, rid: str, perms: dict) -> None: - db.add(Role(role_id=rid, name=rid, permissions=perms)) - - -def test_ce_degrades_to_empty_without_rows(db_session): - # No role assignments at all → role_permissions_for_user returns {} - _mk_user(db_session, "u_norole") - db_session.commit() - assert role_permissions_for_user(db_session, "u_norole") == {} - - -def test_direct_user_role_grants_capability(db_session): - _mk_user(db_session, "ur1") - _mk_role(db_session, "r_skill", {"can_add_skill": True}) - db_session.add(RoleAssignment(role_id="r_skill", principal_type="user", principal_id="ur1")) - db_session.commit() - caps = resolve_user_capabilities(db_session, "ur1") - assert caps["can_add_skill"] is True - assert caps["can_use_api_key"] is False # not granted → system default - - -def test_team_role_inherited_by_member(db_session): - # Department default role: assigned to a team → members inherit in real time - _mk_user(db_session, "ur2") - _mk_role(db_session, "r_dept", {"can_system_config": True}) - db_session.add(Team(team_id="t_dept", name="t_dept")) - db_session.add(TeamMember(team_id="t_dept", user_id="ur2", role="member")) - db_session.add(RoleAssignment(role_id="r_dept", principal_type="team", principal_id="t_dept")) - db_session.commit() - caps = resolve_user_capabilities(db_session, "ur2") - assert caps["can_system_config"] is True - - -def test_multi_role_union_and_personal_override(db_session): - _mk_user(db_session, "ur3", {"can_add_skill": False}) # personal force-off - _mk_role(db_session, "r_a", {"can_add_skill": True, "can_add_mcp": True}) - _mk_role(db_session, "r_b", {"can_use_api_key": True}) - db_session.add(RoleAssignment(role_id="r_a", principal_type="user", principal_id="ur3")) - db_session.add(RoleAssignment(role_id="r_b", principal_type="user", principal_id="ur3")) - db_session.commit() - caps = resolve_user_capabilities(db_session, "ur3") - assert caps["can_add_skill"] is False # personal override beats role - assert caps["can_add_mcp"] is True # role r_a union - assert caps["can_use_api_key"] is True # role r_b union - - -# ── Service CRUD + assignment ───────────────────────────────────────── -def test_service_create_assign_delete(db_session): - svc = RoleService(db_session) - res = svc.create_role("部门管理员", description="dept admin", permissions={"can_add_skill": True, "junk": 1}) - assert res.ok and res.role_id - rid = res.role_id - roles = svc.list_roles() - assert any(r["role_id"] == rid and r["permissions"] == {"can_add_skill": True} for r in roles) - - # Duplicate-name rejection - assert not svc.create_role("部门管理员").ok - - # Assign to user - _mk_user(db_session, "us1") - db_session.commit() - svc.set_principal_roles("user", "us1", [rid]) - assert [r["role_id"] for r in svc.get_principal_roles("user", "us1")] == [rid] - assert svc.list_assignments(rid) == [{"principal_type": "user", "principal_id": "us1"}] - - # Delete role → cascade clears assignments - assert svc.delete_role(rid).ok - assert role_permissions_for_user(db_session, "us1") == {} - - -def test_seed_default_roles_creates_then_idempotent(db_session): - from core.services.role_service import DEFAULT_ROLES, seed_default_roles - - added = seed_default_roles(db_session) - assert set(added) == {spec["name"] for spec in DEFAULT_ROLES} - # Run again: dedup by name → no duplicate creation - assert seed_default_roles(db_session) == [] - names = {r.name for r in RoleService(db_session).repo.list_all()} - assert {"部门成员", "IT管理员"}.issubset(names) - # Seed capability pack is already normalized (only granted bits + allowed_apps kept) - it = RoleService(db_session).repo.get_by_name("IT管理员") - assert it.permissions.get("can_system_config") is True - assert it.permissions.get("allowed_apps") == ["plan_mode", "automation", "batch_runner"] - - -def test_seed_skips_existing_name(db_session): - from core.services.role_service import seed_default_roles - - # Admin already created a role with the same name (different id, different capabilities) → seed should not overwrite - db_session.add(Role(role_id="r_custom", name="部门成员", permissions={"can_add_skill": True})) - db_session.commit() - added = seed_default_roles(db_session) - assert "部门成员" not in added - assert RoleService(db_session).repo.get_by_name("部门成员").role_id == "r_custom" - - -def test_apply_team_default_roles_idempotent(db_session): - from core.db.models import Team - from core.db.repository import RoleRepository - from core.services.role_service import apply_team_default_roles, seed_default_roles - - seed_default_roles(db_session) # 部门成员=new-team default, IT管理员=no - db_session.add(Team(team_id="t_new", name="t_new")) - db_session.commit() - assert apply_team_default_roles(db_session, "t_new") == 1 - assert RoleRepository(db_session).list_principal_role_ids("team", "t_new") == ["role_seed_dept_member"] - assert apply_team_default_roles(db_session, "t_new") == 0 # idempotent - - -def test_create_team_auto_assigns_default_role(db_session): - from core.db.repository import RoleRepository - from core.services.role_service import seed_default_roles - from core.services.team_service import TeamService - - seed_default_roles(db_session) - res = TeamService(db_session).create_team(name="新团队X") - assert res.ok - ids = RoleRepository(db_session).list_principal_role_ids("team", res.team_id) - assert "role_seed_dept_member" in ids and "role_seed_it_admin" not in ids - - -def test_service_delete_system_role_blocked(db_session): - db_session.add(Role(role_id="r_sys", name="内置", permissions={}, is_system=True)) - db_session.commit() - res = RoleService(db_session).delete_role("r_sys") - assert not res.ok and "内置" in res.message diff --git a/src/backend/tests/test_team_agents.py b/src/backend/tests/test_team_agents.py deleted file mode 100644 index 43765bc3..00000000 --- a/src/backend/tests/test_team_agents.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Team sub-agents: visibility + owner/admin management permission tests.""" - -from __future__ import annotations - -import pytest - -from core.db.models import Team, TeamMember, UserShadow -from core.services.user_agent_service import UserAgentService - - -def _seed(db): - # Users: mgr (team admin), mem (team member), out (non-member) - for uid in ("mgr", "mem", "out"): - db.add(UserShadow(user_id=uid, username=uid, extra_data={})) - db.add(Team(team_id="t1", name="T1")) - db.add(TeamMember(team_id="t1", user_id="mgr", role="admin")) - db.add(TeamMember(team_id="t1", user_id="mem", role="member")) - db.commit() - - -def test_only_manager_can_create_team_agent(db_session): - _seed(db_session) - svc = UserAgentService(db_session) - # member cannot create a team agent - with pytest.raises(PermissionError): - svc.create(user_id="mem", operator_name="mem", owner_type="team", - data={"name": "A", "system_prompt": "x"}, team_id="t1") - # non-member cannot create - with pytest.raises(PermissionError): - svc.create(user_id="out", operator_name="out", owner_type="team", - data={"name": "A", "system_prompt": "x"}, team_id="t1") - # owner/admin can - agent = svc.create(user_id="mgr", operator_name="mgr", owner_type="team", - data={"name": "TeamBot", "system_prompt": "x"}, team_id="t1") - assert agent["owner_type"] == "team" - assert agent["team_id"] == "t1" - assert agent["user_id"] is None - assert agent["created_by"] == "mgr" - - -def test_team_agent_requires_team_id(db_session): - _seed(db_session) - with pytest.raises(ValueError): - UserAgentService(db_session).create( - user_id="mgr", operator_name="mgr", owner_type="team", - data={"name": "A", "system_prompt": "x"}, team_id=None) - - -def test_team_agent_visible_to_all_members_not_outsiders(db_session): - _seed(db_session) - svc = UserAgentService(db_session) - a = svc.create(user_id="mgr", operator_name="mgr", owner_type="team", - data={"name": "TeamBot", "system_prompt": "x"}, team_id="t1") - aid = a["agent_id"] - # both member and manager can see it - assert any(x["agent_id"] == aid for x in svc.list_for_user("mem")) - assert any(x["agent_id"] == aid for x in svc.list_for_user("mgr")) - # non-member cannot see it - assert not any(x["agent_id"] == aid for x in svc.list_for_user("out")) - # non-member is denied access to details - with pytest.raises(PermissionError): - svc.get_by_id(aid, user_id="out") - - -def test_member_can_use_but_not_edit_team_agent(db_session): - _seed(db_session) - svc = UserAgentService(db_session) - aid = svc.create(user_id="mgr", operator_name="mgr", owner_type="team", - data={"name": "TeamBot", "system_prompt": "x"}, team_id="t1")["agent_id"] - # member can read (use) - assert svc.get_by_id(aid, user_id="mem")["agent_id"] == aid - # member cannot edit (owner_type='user' context) - with pytest.raises(PermissionError): - svc.update(aid, user_id="mem", operator_name="mem", owner_type="user", data={"name": "X"}) - # manager can edit - updated = svc.update(aid, user_id="mgr", operator_name="mgr", owner_type="user", data={"name": "X"}) - assert updated["name"] == "X" - # manager can delete - assert svc.delete(aid, user_id="mgr", owner_type="user") is True - - -def test_disabled_team_agent_hidden_from_members_visible_to_manager(db_session): - _seed(db_session) - svc = UserAgentService(db_session) - aid = svc.create(user_id="mgr", operator_name="mgr", owner_type="team", - data={"name": "TeamBot", "system_prompt": "x"}, team_id="t1")["agent_id"] - svc.update(aid, user_id="mgr", operator_name="mgr", owner_type="user", data={"is_enabled": False}) - # member's list does not show disabled team agents - assert not any(x["agent_id"] == aid for x in svc.list_for_user("mem")) - # manager can still access it (to re-enable it) - assert svc.get_by_id(aid, user_id="mgr")["agent_id"] == aid - - -def test_admin_route_cannot_edit_team_agent(db_session): - _seed(db_session) - svc = UserAgentService(db_session) - aid = svc.create(user_id="mgr", operator_name="mgr", owner_type="team", - data={"name": "TeamBot", "system_prompt": "x"}, team_id="t1")["agent_id"] - # Admin backend route (owner_type='admin') can only edit admin agents, not team agents - with pytest.raises(PermissionError): - svc.update(aid, user_id=None, operator_name="管理员", owner_type="admin", data={"name": "X"}) diff --git a/src/frontend/scripts/check-i18n.mjs b/src/frontend/scripts/check-i18n.mjs index 5006f32d..0b94d8a2 100644 --- a/src/frontend/scripts/check-i18n.mjs +++ b/src/frontend/scripts/check-i18n.mjs @@ -7,7 +7,7 @@ * 否则英文界面静默回退中文。 * 动态调用 t(变量)（DB 文案、后端 tag 等）不在检查范围。 */ -import { readFileSync, readdirSync } from 'node:fs'; +import { existsSync, readFileSync, readdirSync } from 'node:fs'; import { join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -18,16 +18,32 @@ const entryRe = /^\s*'((?:[^'\\]|\\.)*)'\s*:/; const keyOwner = new Map(); const errors = []; -for (const f of readdirSync(dictDir).filter((f) => f.endsWith('.ts') && f !== 'index.ts')) { - const lines = readFileSync(join(dictDir, f), 'utf-8').split('\n'); +const dictionaryFiles = readdirSync(dictDir) + .filter((f) => f.endsWith('.ts') && f !== 'index.ts') + .map((f) => ({ owner: f, path: join(dictDir, f) })); +const editionDictionary = join(root, 'edition-ee', 'i18n.ts'); +if (existsSync(editionDictionary)) { + dictionaryFiles.push({ owner: 'edition-ee/i18n.ts', path: editionDictionary }); +} +const editionDictionaryDir = join(root, 'edition-ee', 'i18n'); +if (existsSync(editionDictionaryDir)) { + dictionaryFiles.push( + ...readdirSync(editionDictionaryDir) + .filter((f) => f.endsWith('.ts') && f !== 'index.ts') + .map((f) => ({ owner: `edition-ee/i18n/${f}`, path: join(editionDictionaryDir, f) })), + ); +} + +for (const { owner, path } of dictionaryFiles) { + const lines = readFileSync(path, 'utf-8').split('\n'); lines.forEach((ln, i) => { const m = entryRe.exec(ln); if (!m) return; const key = m[1].replace(/\\'/g, "'"); if (keyOwner.has(key)) { - errors.push(`duplicate key '${key}' in ${f}:${i + 1} (already in ${keyOwner.get(key)})`); + errors.push(`duplicate key '${key}' in ${owner}:${i + 1} (already in ${keyOwner.get(key)})`); } else { - keyOwner.set(key, `${f}:${i + 1}`); + keyOwner.set(key, `${owner}:${i + 1}`); } }); } diff --git a/src/frontend/src/App.tsx b/src/frontend/src/App.tsx index 8716dcae..6e237ba5 100755 --- a/src/frontend/src/App.tsx +++ b/src/frontend/src/App.tsx @@ -155,7 +155,7 @@ export default function App() { void fetchCapabilities(); }, [fetchCapabilities, authChecking, authUserId]); - // Fetch edition / license capability bits at startup (CE hides EE entries such as Teams) + // Fetch edition capabilities at startup; CE has no extension entries. const fetchEdition = useEditionStore((s) => s.fetchEdition); useEffect(() => { void fetchEdition(); diff --git a/src/frontend/src/agentEdition.tsx b/src/frontend/src/agentEdition.tsx new file mode 100644 index 00000000..a7eb03cc --- /dev/null +++ b/src/frontend/src/agentEdition.tsx @@ -0,0 +1,14 @@ +import type { UserAgentItem } from './stores/agentStore'; +import { t } from './i18n'; + +export function useEditionAgentPolicy() { + return { + canManage: (_agent: UserAgentItem) => false, + includeInLibrary: (_agent: UserAgentItem) => false, + creatorLabel: (_agent: UserAgentItem) => t('系统内置'), + }; +} + +export function EditionAgentBadge({ agent: _agent }: { agent: UserAgentItem }) { + return null; +} diff --git a/src/frontend/src/api.ts b/src/frontend/src/api.ts index 8a5a1b26..bdfd0b1d 100644 --- a/src/frontend/src/api.ts +++ b/src/frontend/src/api.ts @@ -5,11 +5,18 @@ */ import type { Catalog, ChatItem, ChatMessage, ChunkPreviewResult, KBChunk, MemoryItem, MemoryProfile, MemoryGraphRelation, ResourceItem, AutomationTask, AutomationRun, AutomationNotification, FileConfirmInfo, FileConfirmDecision, DesignPickInfo, OntologyAssetKind, OntologyTagOption } from './types'; -import type { TeamRole } from './utils/roles'; -import { createApiResponseError, LicenseError, licenseErrorMessage, readErrorMessage } from './utils/apiError'; +import type { EditionAuthUserFields } from './editionApiTypes'; +import type { EditionChatDetailFields, EditionCreateProjectFields } from './editionModelTypes'; +import { createEditionAccessError } from './editionAccessError'; +import { createApiResponseError, readErrorMessage } from './utils/apiError'; import { t } from './i18n'; - -export { LicenseError } from './utils/apiError'; +import { + normalizeSiteEditionFields, + normalizeSiteVisibility, + type SiteEditionFields, + type SiteUpdateEditionFields, + type SiteVisibility, +} from './editionSiteVisibility'; type JsonObject = Record; @@ -117,7 +124,7 @@ function isApiEnvelope(payload: unknown): payload is ApiEnvelope { return !!payload && typeof payload === 'object' && 'code' in payload && 'data' in payload; } -function unwrapData(payload: unknown): T { +export function unwrapData(payload: unknown): T { if (isApiEnvelope(payload)) { return payload.data; } @@ -205,7 +212,7 @@ function throwIfSessionExpired(status: number, payload: unknown): void { throw new Error('Session expired'); } -async function apiRequest(path: string, options?: RequestInit): Promise { +export async function apiRequest(path: string, options?: RequestInit): Promise { const url = `${getApiUrl()}${path}`; const response = await fetch(url, { ...options, @@ -224,9 +231,8 @@ async function apiRequest(path: string, options?: RequestInit): Promise { if (!response.ok) { // 401 → session expired, show login; 403 → insufficient permission, fall through to the generic branch below to surface the backend message throwIfSessionExpired(response.status, payload); - if (response.status === 402) { - throw new LicenseError(licenseErrorMessage(payload)); - } + const editionError = createEditionAccessError(response.status, payload, readErrorMessage); + if (editionError) throw editionError; throw createApiResponseError(response.status, payload, `API Error: ${response.status}`); } return payload as T; @@ -280,25 +286,6 @@ export async function getMainModelCapabilities(): Promise { }; } -export interface EditionInfo { - /** Deployment edition: ce (community) / ee (commercial). */ - edition: string; - /** License state machine: internal / licensed / grace / expired / invalid / missing / ce. */ - mode: string; - /** Feature-flag boolean map (multi_tenancy / audit / billing ...); all false on CE. */ - features: Record; -} - -export async function getEditionInfo(): Promise { - const wrapped = await apiRequest('/v1/meta/edition'); - const data = unwrapData(wrapped); - return { - edition: String(data?.edition || 'ee'), - mode: String(data?.mode || 'internal'), - features: (data?.features as Record | undefined) || {}, - }; -} - export async function getCatalog(): Promise { const wrapped = await apiRequest('/v1/catalog'); const data = unwrapData(wrapped); @@ -398,23 +385,17 @@ export async function deleteSession(chatId: string): Promise { }); } -export interface ChatDetail { +export type ChatDetail = { chat_id: string; title: string; user_id: string; project_id: string | null; - share_scope: 'private' | 'team_read' | 'team_edit'; - owner_user_id: string; - is_owner: boolean; - access_level: 'admin' | 'edit' | 'read'; - /** Whether this chat belongs to a team project — determines whether the owner can set it to shared. */ - is_team_project?: boolean; pinned?: boolean; favorite?: boolean; metadata?: Record; -} +} & EditionChatDetailFields; -/** Fetch chat detail (carries share_scope / is_owner / access_level in shared scenarios). */ +/** Fetch chat detail, extended by the active edition's response contract. */ export async function getChatDetail(chatId: string): Promise { const wrapped = await apiRequest(`/v1/chats/${encodeURIComponent(chatId)}`); return unwrapData(wrapped); @@ -1466,17 +1447,7 @@ export async function resetLarkAppInit(): Promise { // ── Auth API (SSO session) ────────────────────────────────────────────── -export interface TeamMembershipBrief { - team_id: string; - name: string; - role: TeamRole; - source?: 'manual' | 'sso_auto'; - sso_department?: string | null; - description?: string | null; - member_count?: number; -} - -export interface AuthUser { +export interface AuthUser extends EditionAuthUserFields { user_id: string; username: string; email?: string; @@ -1484,7 +1455,6 @@ export interface AuthUser { nickname?: string | null; real_name?: string | null; department?: string | null; - teams?: TeamMembershipBrief[]; expires_at?: string; sso_token?: string | null; /** null/undefined = all enabled apps visible by default; array = only the app IDs in the list are visible */ @@ -1527,22 +1497,6 @@ export interface MyProfile extends AuthUser { auth_source?: 'local' | 'external'; } -export interface TeamMemberBrief { - user_id: string; - username: string; - avatar_url?: string | null; - role: TeamRole; - joined_at?: string | null; - is_self?: boolean; -} - -export interface UserSearchResult { - user_id: string; - username: string; - real_name?: string | null; - avatar_url?: string | null; -} - export async function getMyProfile(): Promise { const wrapped = await apiRequest('/v1/me'); return unwrapData(wrapped); @@ -1626,40 +1580,6 @@ export async function clearMyAvatar(): Promise { return unwrapData(wrapped); } -export async function getMyTeams(): Promise { - const wrapped = await apiRequest('/v1/me/teams'); - const d = unwrapData<{ items?: TeamMembershipBrief[] }>(wrapped); - return Array.isArray(d?.items) ? d.items : []; -} - -export async function getTeamMembers(teamId: string): Promise<{ items: TeamMemberBrief[]; my_role: string }> { - const wrapped = await apiRequest(`/v1/me/teams/${encodeURIComponent(teamId)}/members`); - const d = unwrapData<{ items: TeamMemberBrief[]; my_role: string }>(wrapped); - return { items: d?.items || [], my_role: d?.my_role || 'member' }; -} - -export async function inviteTeamMember( - teamId: string, - body: { user_id?: string; username?: string; role?: Exclude }, -): Promise { - await apiRequest(`/v1/me/teams/${encodeURIComponent(teamId)}/members`, { - method: 'POST', - body: JSON.stringify(body), - }); -} - -export async function removeTeamMember(teamId: string, memberUserId: string): Promise { - await apiRequest(`/v1/me/teams/${encodeURIComponent(teamId)}/members/${encodeURIComponent(memberUserId)}`, { - method: 'DELETE', - }); -} - -export async function searchUsers(q: string, limit = 10): Promise { - const wrapped = await apiRequest(`/v1/me/users/search?q=${encodeURIComponent(q)}&limit=${limit}`); - const d = unwrapData<{ items?: UserSearchResult[] }>(wrapped); - return Array.isArray(d?.items) ? d.items : []; -} - export interface ChatShareRecord { share_id: string; chat_id: string; @@ -1972,202 +1892,6 @@ export async function copyArtifactToPersonalFolder( }); } -// ── Team folders / team files API ──────────────────────────────── -import type { - MyTeamItem, - TeamFolderNode, - TeamFolderFlat, - TeamMemberPermission, - TeamFilePermission, -} from './types/teamFiles'; - -export async function listMyTeamsWithPermissions(): Promise { - const wrapped = await apiRequest('/v1/my-teams'); - const data = unwrapData<{ items: MyTeamItem[] }>(wrapped); - return data.items || []; -} - -export async function listTeamFolderTree(teamId: string): Promise { - const wrapped = await apiRequest(`/v1/teams/${encodeURIComponent(teamId)}/folders?as=tree`); - const data = unwrapData<{ tree: TeamFolderNode[] }>(wrapped); - return data.tree || []; -} - -export async function listTeamFoldersFlat(teamId: string): Promise { - const wrapped = await apiRequest(`/v1/teams/${encodeURIComponent(teamId)}/folders?as=flat`); - const data = unwrapData<{ items: TeamFolderFlat[] }>(wrapped); - return data.items || []; -} - -export async function createTeamFolder( - teamId: string, - name: string, - parentFolderId: string | null, -): Promise<{ folder_id: string }> { - const wrapped = await apiRequest(`/v1/teams/${encodeURIComponent(teamId)}/folders`, { - method: 'POST', - body: JSON.stringify({ name, parent_folder_id: parentFolderId }), - }); - return unwrapData<{ folder_id: string }>(wrapped); -} - -export async function renameTeamFolder( - teamId: string, - folderId: string, - name: string, -): Promise { - await apiRequest(`/v1/teams/${encodeURIComponent(teamId)}/folders/${encodeURIComponent(folderId)}`, { - method: 'PATCH', - body: JSON.stringify({ name }), - }); -} - -export async function moveTeamFolder( - teamId: string, - folderId: string, - newParentFolderId: string | null, -): Promise { - await apiRequest(`/v1/teams/${encodeURIComponent(teamId)}/folders/${encodeURIComponent(folderId)}`, { - method: 'PATCH', - body: JSON.stringify({ parent_folder_id: newParentFolderId }), - }); -} - -export async function deleteTeamFolder( - teamId: string, - folderId: string, -): Promise<{ artifacts_affected: number }> { - const wrapped = await apiRequest( - `/v1/teams/${encodeURIComponent(teamId)}/folders/${encodeURIComponent(folderId)}`, - { method: 'DELETE' }, - ); - return unwrapData<{ artifacts_affected: number }>(wrapped); -} - -export async function getFolderAffectedCount( - teamId: string, - folderId: string, -): Promise { - const wrapped = await apiRequest( - `/v1/teams/${encodeURIComponent(teamId)}/folders/${encodeURIComponent(folderId)}/affected-count`, - ); - const data = unwrapData<{ count: number }>(wrapped); - return data.count || 0; -} - -export async function listTeamFiles(params: { - teamId: string; - folderId: string | null; - type?: 'document' | 'image'; - keyword?: string; - page?: number; - page_size?: number; -}): Promise<{ items: ResourceItem[]; total: number; has_more: boolean }> { - const qs = new URLSearchParams(); - if (params.folderId) qs.set('folder_id', params.folderId); - if (params.type) qs.set('type', params.type); - if (params.keyword) qs.set('keyword', params.keyword); - if (params.page) qs.set('page', String(params.page)); - if (params.page_size) qs.set('page_size', String(params.page_size)); - const q = qs.toString(); - const wrapped = await apiRequest( - `/v1/teams/${encodeURIComponent(params.teamId)}/files${q ? '?' + q : ''}`, - ); - return unwrapData<{ items: ResourceItem[]; total: number; has_more: boolean }>(wrapped); -} - -export async function uploadTeamFile( - teamId: string, - folderId: string | null, - file: File, -): Promise { - const url = `${getApiUrl()}/v1/teams/${encodeURIComponent(teamId)}/files/upload`; - const form = new FormData(); - form.append('file', file); - if (folderId) form.append('folder_id', folderId); - const response = await fetch(url, { method: 'POST', credentials: 'include', body: form }); - if (!response.ok) { - const payload = await response.json().catch(() => ({})); - throw new Error(readErrorMessage(payload, `Upload failed: ${response.status}`)); - } - const payload = await response.json(); - return unwrapData(payload); -} - -export async function deleteTeamFile(teamId: string, artifactId: string): Promise { - await apiRequest( - `/v1/teams/${encodeURIComponent(teamId)}/files/${encodeURIComponent(artifactId)}`, - { method: 'DELETE' }, - ); -} - -export async function moveTeamFile( - teamId: string, - artifactId: string, - targetFolderId: string | null, -): Promise { - const wrapped = await apiRequest( - `/v1/teams/${encodeURIComponent(teamId)}/files/${encodeURIComponent(artifactId)}/move`, - { method: 'POST', body: JSON.stringify({ folder_id: targetFolderId }) }, - ); - return unwrapData(wrapped); -} - -export async function moveArtifactToTeam( - artifactId: string, - teamId: string, - folderId: string | null, -): Promise { - const wrapped = await apiRequest( - `/v1/artifacts/${encodeURIComponent(artifactId)}/move-to-team`, - { method: 'POST', body: JSON.stringify({ team_id: teamId, folder_id: folderId }) }, - ); - return unwrapData(wrapped); -} - -/** Copy a personal file into a team folder (non-destructive; keeps the personal original). */ -export async function copyArtifactToTeam( - artifactId: string, - teamId: string, - folderId: string | null, -): Promise { - const wrapped = await apiRequest( - `/v1/artifacts/${encodeURIComponent(artifactId)}/copy-to-team`, - { method: 'POST', body: JSON.stringify({ team_id: teamId, folder_id: folderId }) }, - ); - return unwrapData(wrapped); -} - -/** Recursively copy a personal folder into a team folder (keeps the personal originals). Returns {folders, files} counts. */ -export async function copyFolderToTeam( - personalFolderId: string, - teamId: string, - folderId: string | null, -): Promise<{ folders: number; files: number }> { - const wrapped = await apiRequest( - `/v1/myspace/folders/${encodeURIComponent(personalFolderId)}/copy-to-team`, - { method: 'POST', body: JSON.stringify({ team_id: teamId, folder_id: folderId }) }, - ); - return unwrapData<{ folders: number; files: number }>(wrapped); -} - -export async function listTeamMemberPermissions(teamId: string): Promise { - const wrapped = await apiRequest(`/v1/teams/${encodeURIComponent(teamId)}/members/permissions`); - const data = unwrapData<{ items: TeamMemberPermission[] }>(wrapped); - return data.items || []; -} - -export async function setTeamMemberPermission( - teamId: string, - userId: string, - permission: TeamFilePermission, -): Promise { - await apiRequest( - `/v1/teams/${encodeURIComponent(teamId)}/members/${encodeURIComponent(userId)}/permission`, - { method: 'PUT', body: JSON.stringify({ file_permission: permission }) }, - ); -} - // ── Plan Mode API ───────────────────────────────────────────────────────── import type { Plan } from './types'; @@ -2462,80 +2186,6 @@ export async function deleteNotifications(ids: string[]): Promise { // ── Skill Distillation (Lab personal skill distillation) ──────────────── -export interface SkillDistillResultMeta { - proposed_skill_id?: string; - display_name?: string; - description?: string; - tags?: string[]; - confidence?: number; - digest_text?: string; - sampled_ratio?: number; - partial?: boolean; - session_count?: number; - useful_digests?: number; -} - -export interface SkillDistillJob { - job_id: string; - kind: string; - status: 'queued' | 'running' | 'completed' | 'failed' | 'cancelled'; - progress_done: number; - progress_total: number; - cost_usd: number; - scope: { chat_ids?: string[] | null; hint?: string; include_project_memories?: boolean }; - result_meta: SkillDistillResultMeta; - saved_skill_id?: string | null; - error?: string | null; - created_at?: string | null; - started_at?: string | null; - finished_at?: string | null; - result_skill_content?: string | null; -} - -export async function createSkillDistillJob(params: { - chat_ids: string[] | 'all'; - hint?: string; - include_project_memories?: boolean; -}): Promise { - const res = await apiRequest('/v1/lab/skill-distill/jobs', { - method: 'POST', - body: JSON.stringify(params), - }); - return unwrapData(res); -} - -export async function listSkillDistillJobs(limit = 20): Promise { - const res = await apiRequest(`/v1/lab/skill-distill/jobs?limit=${limit}`); - const data = unwrapData<{ items: SkillDistillJob[] }>(res); - return Array.isArray(data.items) ? data.items : []; -} - -export async function getSkillDistillJob(jobId: string): Promise { - const res = await apiRequest(`/v1/lab/skill-distill/jobs/${jobId}`); - return unwrapData(res); -} - -export async function saveSkillDistillJob( - jobId: string, - params: { skill_content?: string; enable?: boolean }, -): Promise<{ skill_id: string; display_name: string; is_enabled: boolean; job: SkillDistillJob }> { - const res = await apiRequest(`/v1/lab/skill-distill/jobs/${jobId}/save`, { - method: 'POST', - body: JSON.stringify(params), - }); - return unwrapData<{ skill_id: string; display_name: string; is_enabled: boolean; job: SkillDistillJob }>(res); -} - -export async function cancelSkillDistillJob(jobId: string): Promise { - const res = await apiRequest(`/v1/lab/skill-distill/jobs/${jobId}/cancel`, { - method: 'POST', - }); - return unwrapData(res); -} - -export async function deleteSkillDistillJob(jobId: string): Promise { - await apiRequest(`/v1/lab/skill-distill/jobs/${jobId}`, { method: 'DELETE' }); -} // ── Batch execution API ──────────────────────────────────────────────────── @@ -2673,7 +2323,7 @@ import type { ProjectDetail, ProjectFileItem, ProjectItem, - TeamForProjectCreation, + ProjectKind, } from './types'; export interface ProjectListResponse { @@ -2692,20 +2342,12 @@ export async function listProjects(opts: { q?: string; sort?: string; page?: num return unwrapData(wrapped); } -export async function listMyTeamsForProjects(): Promise { - const wrapped = await apiRequest('/v1/projects/teams'); - const data = unwrapData<{ teams: TeamForProjectCreation[] }>(wrapped); - return data?.teams || []; -} - export async function createProject(body: { name: string; description?: string; - kind: 'personal' | 'team'; - team_id?: string; + kind: ProjectKind; linked_folder_id?: string; - linked_team_folder_id?: string; -}): Promise { +} & EditionCreateProjectFields): Promise { const wrapped = await apiRequest('/v1/projects', { method: 'POST', body: JSON.stringify(body), @@ -2811,25 +2453,6 @@ export async function listProjectChats( return { items: data?.items || [], total: data?.pagination?.total_items || 0 }; } -/** - * Set the current chat's share scope within a team project. - * Only the chat owner may call this; the chat must belong to a ``kind='team'`` - * project to be set to anything other than private. - */ -export async function updateChatShareScope( - chatId: string, - shareScope: 'private' | 'team_read' | 'team_edit', -): Promise { - const wrapped = await apiRequest( - `/v1/chats/${encodeURIComponent(chatId)}/share`, - { - method: 'POST', - body: JSON.stringify({ share_scope: shareScope }), - }, - ); - return unwrapData(wrapped); -} - // ── Third-party integration: DingTalk account connection (dingtalk skill / dws CLI) ── export interface DingTalkStatus { status: 'disconnected' | 'pending' | 'connected' | 'error'; @@ -3246,15 +2869,14 @@ export async function cancelLoop(loopId: string): Promise { // ── Sites (site hosting) ──────────────────────────────────────────────── -export interface SiteItem { +export interface SiteItem extends SiteEditionFields { site_id: string; slug: string; /** In-app relative access URL, of the form /site// */ url: string; title: string; description: string | null; - visibility: 'public' | 'private' | 'team'; - team_id: string | null; + visibility: SiteVisibility; entry_file: string; current_version: number; file_count: number; @@ -3276,8 +2898,8 @@ function toSiteItem(raw: JsonObject): SiteItem { url: String(raw.url ?? `/site/${raw.slug ?? ''}/`), title: String(raw.title ?? ''), description: typeof raw.description === 'string' ? raw.description : null, - visibility: raw.visibility === 'private' ? 'private' : raw.visibility === 'team' ? 'team' : 'public', - team_id: typeof raw.team_id === 'string' ? raw.team_id : null, + visibility: normalizeSiteVisibility(raw.visibility), + ...normalizeSiteEditionFields(raw), entry_file: String(raw.entry_file ?? 'index.html'), current_version: Number(raw.current_version ?? 1), file_count: Number(raw.file_count ?? 0), @@ -3301,7 +2923,12 @@ export async function listSites(page = 1, pageSize = 50): Promise<{ items: SiteI export async function updateSite( siteId: string, - data: { title?: string; visibility?: 'public' | 'private' | 'team'; team_id?: string; slug?: string; description?: string }, + data: { + title?: string; + visibility?: SiteVisibility; + slug?: string; + description?: string; + } & SiteUpdateEditionFields, ): Promise { const wrapped = await apiRequest(`/v1/sites/${encodeURIComponent(siteId)}`, { method: 'PATCH', diff --git a/src/frontend/src/chatEdition.ts b/src/frontend/src/chatEdition.ts new file mode 100644 index 00000000..e07a3783 --- /dev/null +++ b/src/frontend/src/chatEdition.ts @@ -0,0 +1,7 @@ +import type { ChatDetail } from './api'; + +export type ChatAccessLevel = 'admin' | 'edit' | 'read'; + +export function chatAccessLevel(_detail: ChatDetail): ChatAccessLevel | null { + return null; +} diff --git a/src/frontend/src/components/agent/AgentCreatePage.tsx b/src/frontend/src/components/agent/AgentCreatePage.tsx index 38e3084d..d52ec76d 100644 --- a/src/frontend/src/components/agent/AgentCreatePage.tsx +++ b/src/frontend/src/components/agent/AgentCreatePage.tsx @@ -1,45 +1,49 @@ import { useEffect, useState } from 'react'; -import { Button, Form, Segmented, Select, Tag, message } from 'antd'; -import { ArrowLeftOutlined, PlusOutlined, EditOutlined, UserOutlined, TeamOutlined } from '@ant-design/icons'; -import { useAgentStore, type UserAgentItem } from '../../stores/agentStore'; -import { listMyTeamsForProjects } from '../../api'; -import type { TeamForProjectCreation } from '../../types'; +import { + ArrowLeftOutlined, + EditOutlined, + PlusOutlined, + UserOutlined, +} from '@ant-design/icons'; +import { Button, Form, Tag, message } from 'antd'; + +import { t } from '../../i18n'; +import { useAgentStore } from '../../stores/agentStore'; +import type { UserAgentItem } from '../../stores/agentStore'; +import { getOntologyBuildFailure } from '../../utils/apiError'; +import type { OntologyBuildFailure } from '../../utils/apiError'; +import { OntologyBuildValidationModal } from '../common/OntologyBuildValidationModal'; import { AgentFormFields } from './AgentFormFields'; import { getRandomIconUrl } from './AgentPanel'; -import { OntologyBuildValidationModal } from '../common/OntologyBuildValidationModal'; -import { getOntologyBuildFailure, type OntologyBuildFailure } from '../../utils/apiError'; -import { t } from '../../i18n'; interface AgentCreatePageProps { onBack: () => void; onCreated: () => void; - agent?: UserAgentItem | null; // null/undefined = create mode, provided = edit mode + agent?: UserAgentItem | null; } export function AgentCreatePage({ onBack, onCreated, agent }: AgentCreatePageProps) { - const { createAgent, updateAgent, fetchAgents, fetchAvailableResources, availableResources } = useAgentStore(); + const { + createAgent, + updateAgent, + fetchAgents, + fetchAvailableResources, + availableResources, + } = useAgentStore(); const [form] = Form.useForm(); const [saving, setSaving] = useState(false); const [buildFailure, setBuildFailure] = useState(null); const isEdit = !!agent; - // Creation scope: personal / team (team is only open for teams where you are owner/admin) - const [scope, setScope] = useState<'personal' | 'team'>('personal'); - const [teamId, setTeamId] = useState(undefined); - const [managerTeams, setManagerTeams] = useState([]); - const [heroIconUrl] = useState(() => - isEdit && agent ? getRandomIconUrl(agent.agent_id || agent.name) : getRandomIconUrl(String(Date.now())) - ); - - useEffect(() => { - // In create mode, load "teams where I am owner/admin" for team-scope selection - if (isEdit) return; - listMyTeamsForProjects().then(setManagerTeams).catch(() => { /* ignore */ }); - }, [isEdit]); + const [heroIconUrl] = useState(() => ( + isEdit && agent + ? getRandomIconUrl(agent.agent_id || agent.name) + : getRandomIconUrl(String(Date.now())) + )); useEffect(() => { - fetchAvailableResources(); + void fetchAvailableResources(); if (agent) { - const ec = agent.extra_config || {}; + const extraConfig = agent.extra_config || {}; form.setFieldsValue({ name: agent.name, description: agent.description, @@ -50,7 +54,7 @@ export function AgentCreatePage({ onBack, onCreated, agent }: AgentCreatePagePro plugin_ids: agent.plugin_ids || [], ontology_tags: agent.ontology_tags || [], max_iters: agent.max_iters ?? 10, - shared_context: !!ec.shared_context, + shared_context: !!extraConfig.shared_context, }); } else { form.resetFields(); @@ -58,23 +62,15 @@ export function AgentCreatePage({ onBack, onCreated, agent }: AgentCreatePagePro } }, [agent, fetchAvailableResources, form]); - async function handleSubmit() { + const handleSubmit = async () => { try { const values = await form.validateFields(); - // Merge shared_context into extra_config const { shared_context, ...rest } = values; const existingExtra = (isEdit ? agent?.extra_config : {}) || {}; - const extra_config: Record = { - ...existingExtra, - shared_context: !!shared_context, + const payload: Partial = { + ...rest, + extra_config: { ...existingExtra, shared_context: !!shared_context }, }; - const payload: Partial = { ...rest, extra_config }; - // Create a team sub-agent: include team_id (the backend uses it to set owner_type=team and verify management permission) - if (!isEdit && scope === 'team') { - if (!teamId) { message.error(t('请选择要归属的团队')); return; } - payload.team_id = teamId; - } - setSaving(true); if (isEdit) { await updateAgent(agent!.agent_id, payload); @@ -87,16 +83,13 @@ export function AgentCreatePage({ onBack, onCreated, agent }: AgentCreatePagePro onCreated(); } catch (error: unknown) { if (error && typeof error === 'object' && 'errorFields' in error) return; - const ontologyFailure = getOntologyBuildFailure(error); - if (ontologyFailure) { - setBuildFailure(ontologyFailure); - } else { - message.error(error instanceof Error ? error.message : (isEdit ? t('更新失败') : t('创建失败'))); - } + const failure = getOntologyBuildFailure(error); + if (failure) setBuildFailure(failure); + else message.error(error instanceof Error ? error.message : (isEdit ? t('更新失败') : t('创建失败'))); } finally { setSaving(false); } - } + }; return (
@@ -115,51 +108,23 @@ export function AgentCreatePage({ onBack, onCreated, agent }: AgentCreatePagePro
-
{!isEdit && ( -
- setScope(v as 'personal' | 'team')} - options={[ - { label: t('个人'), value: 'personal', icon: }, - { label: t('团队'), value: 'team', icon: , disabled: managerTeams.length === 0 }, - ]} - /> - {scope === 'team' && ( - -
-
-
- - {t('按团队')} -
- -
- -
- )} - - - ); +export function VisibilityScopeModal(_props: Record) { + return null; } diff --git a/src/frontend/src/components/file/MySpaceImportModal.tsx b/src/frontend/src/components/file/MySpaceImportModal.tsx index 316882c6..605af43e 100644 --- a/src/frontend/src/components/file/MySpaceImportModal.tsx +++ b/src/frontend/src/components/file/MySpaceImportModal.tsx @@ -1,10 +1,24 @@ -import { useState, useEffect, useCallback, useMemo } from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; import type { ReactNode } from 'react'; +import { + FileOutlined, + FolderOutlined, + PictureOutlined, + SearchOutlined, + UserOutlined, +} from '@ant-design/icons'; +import { Button, Checkbox, Empty, Input, Modal, Tabs, TreeSelect } from 'antd'; import { AnimatePresence, motion } from 'motion/react'; -import { Modal, Tabs, Checkbox, Input, Empty, Button, TreeSelect } from 'antd'; -import { EASE } from '../../utils/motionTokens'; + +import { getApiUrl, getArtifacts, listPersonalFolderTree } from '../../api'; +import { useDelayedFlag } from '../../hooks'; import { t } from '../../i18n'; -import { SearchOutlined, FileOutlined, PictureOutlined, TeamOutlined, FolderOutlined, UserOutlined } from '@ant-design/icons'; +import { useFileStore } from '../../stores'; +import type { ImportedSpaceFile } from '../../stores/fileStore'; +import type { PersonalFolderNode, ResourceItem } from '../../types'; +import { formatDateKey } from '../../utils/date'; +import { getFileIconSrc } from '../../utils/fileIcon'; +import { EASE } from '../../utils/motionTokens'; import { FilePreviewPane } from './FilePreviewPane'; interface ScopeTreeNode { @@ -13,105 +27,65 @@ interface ScopeTreeNode { icon?: ReactNode; children?: ScopeTreeNode[]; } -import { - getArtifacts, - getApiUrl, - listMyTeamsWithPermissions, - listTeamFolderTree, - listTeamFiles, - listPersonalFolderTree, -} from '../../api'; -import { useFileStore } from '../../stores'; -import { useDelayedFlag } from '../../hooks'; -import type { PersonalFolderNode, ResourceItem } from '../../types'; -import type { ImportedSpaceFile } from '../../stores/fileStore'; -import type { MyTeamItem, TeamFolderNode } from '../../types/teamFiles'; -import { getFileIconSrc } from '../../utils/fileIcon'; -import { formatDateKey } from '../../utils/date'; -/** Project reference selection result —— the onSubmit argument when mode='project' */ export interface ProjectImportSelection { - /** List of selected individual artifact IDs (MySpace personal + team mixed) */ artifactIds: string[]; - /** List of selected whole personal folder IDs */ folderIds: string[]; - /** List of selected whole team folder IDs */ - teamFolderIds: string[]; } interface MySpaceImportModalProps { open: boolean; onClose: () => void; - /** - * 'attach' (default) = goes through the chat attachment flow (writes fileStore.addImportedSpaceFiles). - * 'project' = project reference flow: passes the selection back to the parent via callback, and the parent calls - * ``/v1/projects/{id}/files/reference``. - */ mode?: 'attach' | 'project'; - /** Provided by the parent when mode='project', handles artifact / folder references */ onProjectSubmit?: (selection: ProjectImportSelection) => Promise | void; - /** Custom modal title (defaults: mode='attach' uses "从我的空间导入", 'project' uses "从我的空间引用") */ title?: string; } -type ImportScope = - | { kind: 'personal'; folderId: string | null } - | { kind: 'team'; teamId: string; folderId: string | null }; +const ROOT_KEY = 'personal::__root__'; +const folderKey = (folderId: string) => `personal::${folderId}`; -const PERSONAL_ROOT_KEY = 'personal::__root__'; -const PERSONAL_FOLDER_KEY = (folderId: string) => `personal::${folderId}`; -const TEAM_ROOT_KEY = (teamId: string) => `team::${teamId}::__root__`; -const FOLDER_KEY = (teamId: string, folderId: string) => `team::${teamId}::${folderId}`; - -function keyOfScope(s: ImportScope): string { - if (s.kind === 'personal') { - return s.folderId ? PERSONAL_FOLDER_KEY(s.folderId) : PERSONAL_ROOT_KEY; - } - return s.folderId ? FOLDER_KEY(s.teamId, s.folderId) : TEAM_ROOT_KEY(s.teamId); +function parseFolderKey(key: string): string | null | undefined { + if (key === ROOT_KEY || key === 'personal') return null; + if (!key.startsWith('personal::')) return undefined; + const folderId = key.slice('personal::'.length); + return folderId && folderId !== '__root__' ? folderId : null; } -function parseScopeKey(key: string): ImportScope | null { - // Compatible with the old PERSONAL_KEY = 'personal' - if (key === 'personal' || key === PERSONAL_ROOT_KEY) { - return { kind: 'personal', folderId: null }; - } - if (key.startsWith('personal::')) { - const folderIdRaw = key.slice('personal::'.length); - return { kind: 'personal', folderId: folderIdRaw === '__root__' ? null : folderIdRaw }; - } - const parts = key.split('::'); - if (parts[0] !== 'team' || parts.length < 3) return null; - const teamId = parts[1]; - const folderIdRaw = parts[2]; - return { kind: 'team', teamId, folderId: folderIdRaw === '__root__' ? null : folderIdRaw }; -} +const IMAGE_MIMES = new Set([ + 'image/png', + 'image/jpeg', + 'image/jpg', + 'image/gif', + 'image/webp', + 'image/bmp', + 'image/svg+xml', +]); -const IMAGE_MIMES = new Set(['image/png', 'image/jpeg', 'image/jpg', 'image/gif', 'image/webp', 'image/bmp', 'image/svg+xml']); - -function isImageItem(item: ResourceItem) { - return item.type === 'image' || (item.mime_type ? IMAGE_MIMES.has(item.mime_type) : false); +function isImageItem(item: ResourceItem): boolean { + return item.type === 'image' || (!!item.mime_type && IMAGE_MIMES.has(item.mime_type)); } -function formatSize(bytes?: number) { +function formatSize(bytes?: number): string { if (!bytes) return ''; if (bytes < 1024) return `${bytes} B`; if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; return `${(bytes / 1024 / 1024).toFixed(1)} MB`; } -interface FileListProps { - items: ResourceItem[]; - selected: Set; - previewId: string | null; - onToggle: (id: string) => void; - onPreview: (item: ResourceItem) => void; +function foldersToOptions(folders: PersonalFolderNode[]): ScopeTreeNode[] { + return folders.map((folder) => ({ + value: folderKey(folder.folder_id), + title: folder.name, + icon: , + children: folder.children?.length ? foldersToOptions(folder.children) : undefined, + })); } -function FileListSkeleton({ count = 5 }: { count?: number }) { +function FileListSkeleton() { return (