diff --git a/CubeDB/migrate/migrations/mysql/20260813120000_component_warehouse.sql b/CubeDB/migrate/migrations/mysql/20260813120000_component_warehouse.sql new file mode 100644 index 000000000..47dce8e51 --- /dev/null +++ b/CubeDB/migrate/migrations/mysql/20260813120000_component_warehouse.sql @@ -0,0 +1,85 @@ +-- Copyright (c) 2026 Tencent Inc. +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Component warehouse: CubeOps-owned inventory of versioned component trees +-- extracted from one-click packages. Coverage uses t_component_node_install, +-- not the live heartbeat matrix. + +-- +goose NO TRANSACTION +-- +goose Up + +CALL cubemaster_acquire_migration_lock('cubemaster_migration_20260813120000_warehouse', 60); + +CREATE TABLE IF NOT EXISTS `t_component_warehouse` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `arch` varchar(16) NOT NULL, + `component` varchar(64) NOT NULL, + `version` varchar(128) NOT NULL, + `source` varchar(32) NOT NULL DEFAULT '', + `source_ref` varchar(256) NOT NULL DEFAULT '', + `rel_path` varchar(512) NOT NULL DEFAULT '', + `size_bytes` bigint NOT NULL DEFAULT 0, + `checksum` varchar(128) NOT NULL DEFAULT '', + `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `uk_wh_arch_comp_ver` (`arch`, `component`, `version`), + KEY `idx_wh_component` (`component`, `version`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS `t_component_import_job` ( + `id` varchar(36) NOT NULL, + `source` varchar(32) NOT NULL, + `source_ref` varchar(256) NOT NULL DEFAULT '', + `tag` varchar(128) NOT NULL DEFAULT '', + `arch` varchar(16) NOT NULL DEFAULT '', + `status` varchar(32) NOT NULL DEFAULT 'pending', + `error` text, + `bytes_total` bigint NOT NULL DEFAULT 0, + `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY `idx_import_status` (`status`, `created_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS `t_component_preinstall_job` ( + `id` varchar(36) NOT NULL, + `node_id` varchar(128) NOT NULL, + `arch` varchar(16) NOT NULL, + `component` varchar(64) NOT NULL, + `version` varchar(128) NOT NULL, + `status` varchar(32) NOT NULL DEFAULT 'pending', + `error` text, + `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY `idx_preinstall_node_status` (`node_id`, `status`), + KEY `idx_preinstall_comp` (`arch`, `component`, `version`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS `t_component_node_install` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `node_id` varchar(128) NOT NULL, + `arch` varchar(16) NOT NULL, + `component` varchar(64) NOT NULL, + `version` varchar(128) NOT NULL, + `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `uk_node_install` (`node_id`, `arch`, `component`, `version`), + KEY `idx_node_install_node` (`node_id`), + KEY `idx_node_install_comp` (`arch`, `component`, `version`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +SELECT RELEASE_LOCK('cubemaster_migration_20260813120000_warehouse'); + +-- +goose Down + +CALL cubemaster_acquire_migration_lock('cubemaster_migration_20260813120000_warehouse', 60); + +DROP TABLE IF EXISTS `t_component_node_install`; +DROP TABLE IF EXISTS `t_component_preinstall_job`; +DROP TABLE IF EXISTS `t_component_import_job`; +DROP TABLE IF EXISTS `t_component_warehouse`; + +SELECT RELEASE_LOCK('cubemaster_migration_20260813120000_warehouse'); diff --git a/CubeDB/migrate/migrations/postgres/20260813120000_component_warehouse.sql b/CubeDB/migrate/migrations/postgres/20260813120000_component_warehouse.sql new file mode 100644 index 000000000..19cfe6d89 --- /dev/null +++ b/CubeDB/migrate/migrations/postgres/20260813120000_component_warehouse.sql @@ -0,0 +1,83 @@ +-- Copyright (c) 2026 Tencent Inc. +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Component warehouse (PostgreSQL). + +-- +goose NO TRANSACTION +-- +goose Up + +SELECT cubemaster_acquire_migration_lock('cubemaster_migration_20260813120000_warehouse', 60); + +CREATE TABLE IF NOT EXISTS t_component_warehouse ( + id bigserial NOT NULL, + arch varchar(16) NOT NULL, + component varchar(64) NOT NULL, + version varchar(128) NOT NULL, + source varchar(32) NOT NULL DEFAULT '', + source_ref varchar(256) NOT NULL DEFAULT '', + rel_path varchar(512) NOT NULL DEFAULT '', + size_bytes bigint NOT NULL DEFAULT 0, + checksum varchar(128) NOT NULL DEFAULT '', + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id), + CONSTRAINT uk_wh_arch_comp_ver UNIQUE (arch, component, version) +); +CREATE INDEX IF NOT EXISTS idx_wh_component ON t_component_warehouse (component, version); + +CREATE TABLE IF NOT EXISTS t_component_import_job ( + id varchar(36) NOT NULL, + source varchar(32) NOT NULL, + source_ref varchar(256) NOT NULL DEFAULT '', + tag varchar(128) NOT NULL DEFAULT '', + arch varchar(16) NOT NULL DEFAULT '', + status varchar(32) NOT NULL DEFAULT 'pending', + error text, + bytes_total bigint NOT NULL DEFAULT 0, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id) +); +CREATE INDEX IF NOT EXISTS idx_import_status ON t_component_import_job (status, created_at); + +CREATE TABLE IF NOT EXISTS t_component_preinstall_job ( + id varchar(36) NOT NULL, + node_id varchar(128) NOT NULL, + arch varchar(16) NOT NULL, + component varchar(64) NOT NULL, + version varchar(128) NOT NULL, + status varchar(32) NOT NULL DEFAULT 'pending', + error text, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id) +); +CREATE INDEX IF NOT EXISTS idx_preinstall_node_status ON t_component_preinstall_job (node_id, status); +CREATE INDEX IF NOT EXISTS idx_preinstall_comp ON t_component_preinstall_job (arch, component, version); + +CREATE TABLE IF NOT EXISTS t_component_node_install ( + id bigserial NOT NULL, + node_id varchar(128) NOT NULL, + arch varchar(16) NOT NULL, + component varchar(64) NOT NULL, + version varchar(128) NOT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id), + CONSTRAINT uk_node_install UNIQUE (node_id, arch, component, version) +); +CREATE INDEX IF NOT EXISTS idx_node_install_node ON t_component_node_install (node_id); +CREATE INDEX IF NOT EXISTS idx_node_install_comp ON t_component_node_install (arch, component, version); + +SELECT pg_advisory_unlock(hashtext('cubemaster_migration_20260813120000_warehouse')); + +-- +goose Down + +SELECT cubemaster_acquire_migration_lock('cubemaster_migration_20260813120000_warehouse', 60); + +DROP TABLE IF EXISTS t_component_node_install; +DROP TABLE IF EXISTS t_component_preinstall_job; +DROP TABLE IF EXISTS t_component_import_job; +DROP TABLE IF EXISTS t_component_warehouse; + +SELECT pg_advisory_unlock(hashtext('cubemaster_migration_20260813120000_warehouse')); diff --git a/CubeMaster/conf.yaml b/CubeMaster/conf.yaml index c62dade38..a9b2061bf 100644 --- a/CubeMaster/conf.yaml +++ b/CubeMaster/conf.yaml @@ -23,8 +23,8 @@ cubelet_conf: create_image_timeout_insec: 300 # See docs/guide/lifecycle.md — Timeout semantics (canonical). default_timeout_insec: -1 - # Create-path RPC/scheduling deadline (seconds). - create_timeout_insec: 300 + # Create-path RPC/scheduling deadline (seconds). Must cover warehouse download. + create_timeout_insec: 600 create_concurrent_limit: 100 destroy_concurent_limit: 100 enable_exposed_port: true diff --git a/CubeMaster/pkg/base/config/config.go b/CubeMaster/pkg/base/config/config.go index 4fdfe45ad..e7eed5b09 100644 --- a/CubeMaster/pkg/base/config/config.go +++ b/CubeMaster/pkg/base/config/config.go @@ -944,7 +944,7 @@ func preHandleCubeletConf(config *Config) error { } // DefaultTimeoutInsec is left untouched — see docs/guide/lifecycle.md. if config.CubeletConf.CreateTimeoutInsec <= 0 { - config.CubeletConf.CreateTimeoutInsec = 300 + config.CubeletConf.CreateTimeoutInsec = 600 } if config.CubeletConf.MaxRetries == 0 { config.CubeletConf.MaxRetries = 5 diff --git a/CubeMaster/pkg/templatecenter/compat.go b/CubeMaster/pkg/templatecenter/compat.go index c589e4945..d95cd5724 100644 --- a/CubeMaster/pkg/templatecenter/compat.go +++ b/CubeMaster/pkg/templatecenter/compat.go @@ -86,15 +86,14 @@ func ScheduleCompatScanForNode(nodeID string) { } // ScanNodeCompat updates replica compat_status for the compat matrix only. -// It does not touch scheduling caches: READY replicas remain schedulable regardless of STALE. +// It does not touch scheduling caches: READY replicas remain schedulable +// regardless of historical STALE rows. Pinned replicas stay OK even when the +// node has not reported live versions yet. func ScanNodeCompat(ctx context.Context, nodeID string) error { if !isReady() { return ErrTemplateStoreNotInitialized } - current, ok := nodemeta.GetNodeComponentVersions(ctx, nodeID) - if !ok { - return markNodeReadyReplicasCompat(ctx, nodeID, CompatStatusUnknown) - } + current, _ := nodemeta.GetNodeComponentVersions(ctx, nodeID) replicas := make([]models.TemplateReplica, 0) if err := store.db.WithContext(ctx).Table(constants.TemplateReplicaTableName). @@ -115,28 +114,6 @@ func ScanNodeCompat(ctx context.Context, nodeID string) error { return nil } -func markNodeReadyReplicasCompat(ctx context.Context, nodeID, status string) error { - replicas := make([]models.TemplateReplica, 0) - if err := store.db.WithContext(ctx).Table(constants.TemplateReplicaTableName). - Where("node_id = ? AND status = ?", nodeID, ReplicaStatusReady). - Find(&replicas).Error; err != nil { - return err - } - for _, row := range replicas { - current := normalizeCompatStatus(row.CompatStatus) - if current == CompatStatusStale && status == CompatStatusUnknown { - continue - } - if current == status { - continue - } - if err := updateReplicaCompat(ctx, row.TemplateID, nodeID, status); err != nil { - return err - } - } - return nil -} - func updateReplicaCompat(ctx context.Context, templateID, nodeID, status string) error { now := time.Now() return store.db.WithContext(ctx).Table(constants.TemplateReplicaTableName). diff --git a/CubeMaster/pkg/templatecenter/compat_test.go b/CubeMaster/pkg/templatecenter/compat_test.go index 6d6c79106..0dbe5e907 100644 --- a/CubeMaster/pkg/templatecenter/compat_test.go +++ b/CubeMaster/pkg/templatecenter/compat_test.go @@ -24,6 +24,7 @@ func TestEvaluateCompat(t *testing.T) { GuestImageVersion: "v1", AgentVersion: "a1", KernelVersion: "k1", + ShimVersion: "s1", CompatPolicy: CompatPolicyStrict, }, guest: "v1", @@ -32,17 +33,67 @@ func TestEvaluateCompat(t *testing.T) { want: CompatStatusOK, }, { - name: "guest mismatch is stale", + name: "pinned replica stays ok when live guest differs", replica: ReplicaStatus{ GuestImageVersion: "v1", AgentVersion: "a1", KernelVersion: "k1", + ShimVersion: "s1", CompatPolicy: CompatPolicyStrict, }, guest: "v2", agent: "a1", kernel: "k1", - want: CompatStatusStale, + want: CompatStatusOK, + }, + { + name: "unpinned replica is unknown", + replica: ReplicaStatus{ + CompatPolicy: CompatPolicyStrict, + }, + guest: "v2", + agent: "a2", + kernel: "k2", + want: CompatStatusUnknown, + }, + { + name: "empty guest pin is unknown even when agent is set", + replica: ReplicaStatus{ + AgentVersion: "a1", + KernelVersion: "k1", + ShimVersion: "s1", + CompatPolicy: CompatPolicyStrict, + }, + guest: "v2", + agent: "a1", + kernel: "k1", + want: CompatStatusUnknown, + }, + { + name: "pre-multiversion guest agent kernel without shim is unknown", + replica: ReplicaStatus{ + GuestImageVersion: "v0.6.0", + AgentVersion: "v0.6.0", + KernelVersion: "k1", + CompatPolicy: CompatPolicyStrict, + }, + guest: "v0.6.0-test7", + agent: "v0.6.0-test7", + kernel: "k2", + want: CompatStatusUnknown, + }, + { + name: "guest agent shim without kernel is unknown", + replica: ReplicaStatus{ + GuestImageVersion: "v1", + AgentVersion: "a1", + ShimVersion: "s1", + CompatPolicy: CompatPolicyStrict, + }, + guest: "v2", + agent: "a2", + kernel: "k2", + want: CompatStatusUnknown, }, { name: "kernel mismatch does not require redo", @@ -50,6 +101,7 @@ func TestEvaluateCompat(t *testing.T) { GuestImageVersion: "v1", AgentVersion: "a1", KernelVersion: "k1", + ShimVersion: "s1", CompatPolicy: CompatPolicyStrict, }, guest: "v1", @@ -58,17 +110,18 @@ func TestEvaluateCompat(t *testing.T) { want: CompatStatusOK, }, { - name: "missing current agent is unknown, not ok", + name: "pinned replica stays ok when current versions are empty", replica: ReplicaStatus{ GuestImageVersion: "v1", AgentVersion: "a1", KernelVersion: "k1", + ShimVersion: "s1", CompatPolicy: CompatPolicyStrict, }, - guest: "v1", + guest: "", agent: "", - kernel: "k1", - want: CompatStatusUnknown, + kernel: "", + want: CompatStatusOK, }, { name: "unknown literal is treated as missing", @@ -76,6 +129,7 @@ func TestEvaluateCompat(t *testing.T) { GuestImageVersion: "v1", AgentVersion: "unknown", KernelVersion: "k1", + ShimVersion: "s1", CompatPolicy: CompatPolicyStrict, }, guest: "v1", @@ -84,11 +138,12 @@ func TestEvaluateCompat(t *testing.T) { want: CompatStatusUnknown, }, { - name: "guest only policy ignores agent mismatch", + name: "guest only policy with restore pins stays ok when agent differs", replica: ReplicaStatus{ GuestImageVersion: "v1", AgentVersion: "a1", KernelVersion: "k1", + ShimVersion: "s1", CompatPolicy: CompatPolicyGuestOnly, }, guest: "v1", @@ -102,6 +157,7 @@ func TestEvaluateCompat(t *testing.T) { GuestImageVersion: "v1", AgentVersion: "a1", KernelVersion: "k1", + ShimVersion: "s1", CompatPolicy: CompatPolicyStrict, }, guest: "v1", diff --git a/CubeMaster/pkg/templatecenter/store.go b/CubeMaster/pkg/templatecenter/store.go index 8783ffa50..abbc139b7 100644 --- a/CubeMaster/pkg/templatecenter/store.go +++ b/CubeMaster/pkg/templatecenter/store.go @@ -1159,42 +1159,24 @@ func normalizeCompatPolicy(policy string) string { } } -func compareCompatDimension(bound, current string) (stale bool, unknown bool) { - bound = normalizeComponentVersion(bound) - current = normalizeComponentVersion(current) - if bound == "" || current == "" { - return false, true - } - return bound != current, false -} - -func evaluateCompat(replica ReplicaStatus, currentGuestImage, currentAgent, _ string) string { - policy := normalizeCompatPolicy(replica.CompatPolicy) - dimensions := []struct { - bound string - current string - active bool - }{ - {replica.GuestImageVersion, currentGuestImage, true}, - {replica.AgentVersion, currentAgent, policy != CompatPolicyGuestOnly}, - } - seenUnknown := false - for _, dim := range dimensions { - if !dim.active { - continue - } - stale, unknown := compareCompatDimension(dim.bound, dim.current) - if stale { - return CompatStatusStale - } - if unknown { - seenUnknown = true - } - } - if seenUnknown { - return CompatStatusUnknown +// hasRestorePin reports whether the replica froze guest-image, cube-agent, +// kernel, and cube-shim. guest-image and cube-agent alone were already stored +// for the 0.4.0 compat matrix; without kernel and shim the replica was not +// created with component multi-version restore, and create would follow live +// toolbox paths for those components. +func hasRestorePin(replica ReplicaStatus) bool { + guest := normalizeComponentVersion(replica.GuestImageVersion) + agent := normalizeComponentVersion(replica.AgentVersion) + kernel := normalizeComponentVersion(replica.KernelVersion) + shim := normalizeComponentVersion(replica.ShimVersion) + return guest != "" && agent != "" && kernel != "" && shim != "" +} + +func evaluateCompat(replica ReplicaStatus, _, _, _ string) string { + if hasRestorePin(replica) { + return CompatStatusOK } - return CompatStatusOK + return CompatStatusUnknown } func isReplicaSchedulable(replica ReplicaStatus) bool { @@ -1202,8 +1184,8 @@ func isReplicaSchedulable(replica ReplicaStatus) bool { } // bindGuestVersionToReplica records pin versions on the replica. CompatStatus -// still compares guest[+agent] only; kernel/shim are stored for create inject -// and do not participate in evaluateCompat. +// is OK only when guest, agent, kernel, and shim are all pinned so live +// toolbox drift does not mark a multi-version replica as needing rebuild. func bindGuestVersionToReplica(replica *ReplicaStatus, guestImageVersion, agentVersion, kernelVersion, shimVersion string) { if replica == nil { return diff --git a/CubeOps/Dockerfile b/CubeOps/Dockerfile index 2cd099f99..e65db452c 100644 --- a/CubeOps/Dockerfile +++ b/CubeOps/Dockerfile @@ -41,6 +41,9 @@ RUN apk add --no-cache ca-certificates docker-cli tzdata COPY --from=builder /usr/local/bin/cubeops /usr/local/bin/cubeops +RUN mkdir -p /data/cubeops/warehouse +VOLUME ["/data/cubeops/warehouse"] + EXPOSE 3010 ENTRYPOINT ["cubeops"] diff --git a/CubeOps/README.md b/CubeOps/README.md index 8fa30540b..43f502cef 100644 --- a/CubeOps/README.md +++ b/CubeOps/README.md @@ -18,11 +18,16 @@ content-fingerprint tamper detection and cluster-wide locking. CubeOps exposes two API groups: 1. **Admin/Ops API** (`/api/v1/auth`, `/api/v1/cluster`, `/api/v1/agenthub`, - `/api/v1/store`, `/api/v1/config`) — used by the WebUI for cluster - management, digital assistant (AgentHub) lifecycle, and store operations. + `/api/v1/store`, `/api/v1/config`, `/api/v1/warehouse`) — used by the WebUI for cluster + management, digital assistant (AgentHub) lifecycle, store operations, and the + component warehouse. 2. **SDK API** (`/api/v1/sdk/*`) — used by the WebUI for sandbox/template/ snapshot CRUD. These endpoints call CubeMaster HTTP REST API directly (replacing the former CubeAPI reverse proxy). +3. **Node warehouse API** (`/internal/warehouse/*`) — unauthenticated, same + isolation model as CubeMaster `/internal/meta`. Compute nodes download + pinned component versions here. Do not publish this prefix on the public + WebUI nginx. ## Quick Start @@ -96,35 +101,14 @@ curl -s http://127.0.0.1:3010/health # → ok ``` -The systemd unit reads environment variables from `.one-click.env` via the -start script at `deploy/one-click/scripts/systemd/cubeops-start.sh`. - ## Configuration -CubeOps supports two configuration methods, which can be combined: - -### Option 1: YAML config file (recommended) - -Copy the example and edit: - -```bash -cp config.example.yaml /etc/cube/ops.yaml -vi /etc/cube/ops.yaml -``` - -Or point to a custom path: - -```bash -export CUBE_OPS_CONFIG=/path/to/your/config.yaml -``` - -See [`config.example.yaml`](./config.example.yaml) for all available fields -with inline comments. - -### Option 2: Environment variables (legacy, still supported) +One-click and Helm configure CubeOps with environment variables. Nested YAML +keys map to `CUBE_OPS_
_` (for example `warehouse.dir` → +`CUBE_OPS_WAREHOUSE_DIR`). Env wins over a YAML file, which wins over built-in +defaults. -Environment variables take precedence over YAML — use this to override -individual fields without editing the YAML file. +### Environment variables (one-click / Helm) | Variable | Default | Description | |----------|---------|-------------| @@ -140,8 +124,34 @@ individual fields without editing the YAML file. | `CUBE_MASTER_ADDR` | `http://127.0.0.1:8089` | CubeMaster base URL | | `CUBE_API_SANDBOX_DOMAIN` | `cube.app` | Sandbox domain (used by SDK handler for sandbox URL construction) | | `REDIS_URL` | *(optional)* | Redis for JWT blacklist | +| `CUBE_OPS_WAREHOUSE_DIR` | `/data/cubeops/warehouse` | On-disk component warehouse root (`warehouse.dir`) | +| `CUBE_OPS_WAREHOUSE_WRITE_TIMEOUT` | `30m` | HTTP write timeout for node blob + admin uploads (`warehouse.write_timeout`) | +| `CUBE_OPS_WAREHOUSE_FETCH_TIMEOUT` | `30m` | GitHub/CNB download timeout for one-click imports (`warehouse.fetch_timeout`) | +| `CUBE_OPS_WAREHOUSE_GITHUB_REPOS` | `TencentCloud/CubeSandbox` | Comma-separated GitHub owner/repo allow-list (`warehouse.github_repos`) | +| `CUBE_OPS_WAREHOUSE_CNB_REPOS` | `CubeSandbox/CubeSandbox` | Comma-separated CNB owner/repo allow-list (`warehouse.cnb_repos`) | +| `CUBE_OPS_WAREHOUSE_GITHUB_TOKEN` | *(optional)* | Token for private GitHub release downloads (`warehouse.github_token`) | +| `CUBE_OPS_WAREHOUSE_CNB_TOKEN` | *(optional)* | Token for private CNB release downloads (`warehouse.cnb_token`) | -**Resolution order**: environment variables > YAML file > built-in defaults. +The systemd unit reads environment variables from `.one-click.env` via the +start script at `deploy/one-click/scripts/systemd/cubeops-start.sh`. + +### YAML file (optional, manual installs) + +Copy the example and edit: + +```bash +cp config.example.yaml /etc/cube/ops.yaml +vi /etc/cube/ops.yaml +``` + +Or point to a custom path: + +```bash +export CUBE_OPS_CONFIG=/path/to/your/config.yaml +``` + +See [`config.example.yaml`](./config.example.yaml) for all available fields +with inline comments. One-click does not install this file. ## Authentication @@ -169,6 +179,29 @@ RBAC is reserved for future use — currently any valid JWT grants full access. - `GET /api/v1/nodes` — Node list - `GET /api/v1/nodes/{nodeID}` — Node detail +### Warehouse + +The warehouse catalog is a closed set of four components (`cube-shim`, +`cube-image`, `cube-agent`, `cube-kernel-scf`). Artifacts stay keyed by +`(arch, component, version)`. `GET /warehouse/components` always returns +those four names (empty warehouse: `versionCount: 0`). Unknown names on +`GET /warehouse/components/{component}` are `400`. If CubeMaster cannot +list nodes, `nodesMissing` is omitted rather than reported as zero. + +- `GET /api/v1/warehouse/components` — Catalog summaries (`name`, `versionCount`, `arches`, `sizeBytes`, optional `nodesMissing`) +- `GET /api/v1/warehouse/components/{component}` — Versions grouped with per-arch artifacts and node coverage +- `DELETE /api/v1/warehouse/components/{component}/versions/{version}?arch=` — Delete a warehouse copy (not node-local inventory) +- `POST /api/v1/warehouse/uploads` — Upload a one-click `.tar.gz` +- `POST /api/v1/warehouse/imports` — Start an async import (github / cnb / upload) +- `GET /api/v1/warehouse/imports` — List import jobs (`?limit=&offset=`; response `{jobs, total}`) +- `GET /api/v1/warehouse/imports/{id}` — Import job status +- `POST /api/v1/warehouse/preinstall` — Create per-node pull jobs +- `GET /api/v1/warehouse/preinstall` — List preinstall jobs (`?limit=&offset=&node_id=&status=`; response `{jobs, total}`) +- `GET /internal/warehouse/blob` — Node download of one version tree (no JWT) +- `GET /internal/warehouse/jobs` — Pending preinstall jobs for this node +- `POST /internal/warehouse/jobs/{id}/ack` — Ack running/succeeded/failed +- `PUT /internal/warehouse/inventory` — Replace this node's inventoried versions for one arch (empty `items` clears that node+arch) + ### AgentHub - `GET /api/v1/agenthub/instances` — List agent instances - `POST /api/v1/agenthub/instances` — Create agent instance diff --git a/CubeOps/config.example.yaml b/CubeOps/config.example.yaml index 7a10bf1c3..8f67593e9 100644 --- a/CubeOps/config.example.yaml +++ b/CubeOps/config.example.yaml @@ -1,15 +1,16 @@ -# CubeOps configuration example. +# CubeOps configuration example (optional; one-click and Helm use env vars). # # Copy to /etc/cube/ops.yaml and edit, or set the path in CUBE_OPS_CONFIG. # Any field here can be overridden by an environment variable; env wins. +# Nested keys map to CUBE_OPS_
_, e.g. warehouse.dir → CUBE_OPS_WAREHOUSE_DIR. # --- HTTP server --- -bind: "127.0.0.1:3010" -log_level: "info" -log_dir: "/data/log/CubeOps" # log file directory -log_file_num: 10 # number of rotated log files to retain -log_file_size: 100 # max size in MB per log file before rotation -jwt_secret: "" # leave empty to auto-generate on first start +bind: "127.0.0.1:3010" # CUBE_OPS_BIND +log_level: "info" # CUBE_OPS_LOG_LEVEL +log_dir: "/data/log/CubeOps" # CUBE_OPS_LOG_DIR +log_file_num: 10 # CUBE_OPS_LOG_FILE_NUM +log_file_size: 100 # CUBE_OPS_LOG_FILE_SIZE +jwt_secret: "" # JWT_SECRET; leave empty to auto-generate on first start # --- Database (MySQL) --- # IMPORTANT: Replace the placeholder values below with your real credentials @@ -19,31 +20,44 @@ jwt_secret: "" # leave empty to auto-generate on first start # Two equivalent ways to configure: # # (a) Full URL (recommended): -# database_url: "mysql://USER:PASSWORD@HOST:PORT/DBNAME" +# database_url: "mysql://USER:PASSWORD@HOST:PORT/DBNAME" # DATABASE_URL # # (b) Individual fields (URL takes precedence if both are set): -# mysql_host: "127.0.0.1" -# mysql_port: 3306 -# mysql_user: "REPLACE_ME" -# mysql_password: "REPLACE_ME" -# mysql_db: "REPLACE_ME" +# mysql_host: "127.0.0.1" # CUBE_SANDBOX_MYSQL_HOST +# mysql_port: 3306 # CUBE_SANDBOX_MYSQL_PORT +# mysql_user: "REPLACE_ME" # CUBE_SANDBOX_MYSQL_USER +# mysql_password: "REPLACE_ME" # CUBE_SANDBOX_MYSQL_PASSWORD +# mysql_db: "REPLACE_ME" # CUBE_SANDBOX_MYSQL_DB # # When neither is set, CubeOps falls back to the CUBE_SANDBOX_MYSQL_* # environment variables (legacy behavior). # --- JWT --- # Access tokens are short-lived; refresh tokens are long-lived. -access_ttl: "15m" -refresh_ttl: "168h" +access_ttl: "15m" # JWT_ACCESS_TTL +refresh_ttl: "168h" # JWT_REFRESH_TTL # --- CubeMaster (the actual sandbox / cluster control plane) --- -cubemaster_addr: "http://127.0.0.1:8089" +cubemaster_addr: "http://127.0.0.1:8089" # CUBE_MASTER_ADDR # --- CubeAPI (deprecated SDK proxy) --- -cubeapi_url: "http://127.0.0.1:3000" +cubeapi_url: "http://127.0.0.1:3000" # CUBE_API_URL # --- Redis (optional, only needed if you add rate-limiting later) --- -redis_url: "" +redis_url: "" # REDIS_URL # --- Sandbox domain exposed to SDK clients via /config --- -sandbox_domain: "cube.app" +sandbox_domain: "cube.app" # CUBE_API_SANDBOX_DOMAIN + +# --- Component warehouse --- +# Disk tree: //// +warehouse: + dir: "/data/cubeops/warehouse" # CUBE_OPS_WAREHOUSE_DIR + write_timeout: "30m" # CUBE_OPS_WAREHOUSE_WRITE_TIMEOUT + fetch_timeout: "30m" # CUBE_OPS_WAREHOUSE_FETCH_TIMEOUT + github_repos: # CUBE_OPS_WAREHOUSE_GITHUB_REPOS (comma-separated) + - "TencentCloud/CubeSandbox" + cnb_repos: # CUBE_OPS_WAREHOUSE_CNB_REPOS (comma-separated) + - "CubeSandbox/CubeSandbox" + # github_token / cnb_token: leave empty for public releases. + # CUBE_OPS_WAREHOUSE_GITHUB_TOKEN / CUBE_OPS_WAREHOUSE_CNB_TOKEN diff --git a/CubeOps/internal/config/config.go b/CubeOps/internal/config/config.go index 17337df2d..b4797d600 100644 --- a/CubeOps/internal/config/config.go +++ b/CubeOps/internal/config/config.go @@ -10,13 +10,15 @@ // manifests keep using env vars without changes. // // 2. YAML file at the path in CUBE_OPS_CONFIG (or /etc/cube/ops.yaml if -// unset). YAML is the recommended way to configure CubeOps going forward -// because it groups all knobs in one place and supports comments. +// unset). One-click and Helm use environment variables; the YAML file is +// optional for manual installs. Nested keys map to CUBE_OPS_
_ +// (for example warehouse.dir → CUBE_OPS_WAREHOUSE_DIR). // // 3. Built-in defaults. // -// The YAML schema is intentionally flat — one section per top-level -// component. See config.example.yaml for a fully commented example. +// The YAML schema groups related knobs under a section per component +// (for example warehouse:). See config.example.yaml for a fully +// commented example. package config import ( @@ -65,6 +67,19 @@ type Config struct { // Sandbox domain exposed to SDK clients; matches SDK handler's // CUBE_API_SANDBOX_DOMAIN env so the /config endpoint stays in sync. SandboxDomain string `yaml:"sandbox_domain"` + + Warehouse WarehouseConfig `yaml:"warehouse"` +} + +// WarehouseConfig is the on-disk component warehouse and its import sources. +type WarehouseConfig struct { + Dir string `yaml:"dir"` + WriteTimeout time.Duration `yaml:"write_timeout"` + FetchTimeout time.Duration `yaml:"fetch_timeout"` + GitHubRepos []string `yaml:"github_repos"` + CNBRepos []string `yaml:"cnb_repos"` + GitHubToken string `yaml:"github_token"` + CNBToken string `yaml:"cnb_token"` } // Load reads configuration from YAML + environment variables (env wins). @@ -113,6 +128,21 @@ func Load() (*Config, error) { if cfg.SandboxDomain == "" { cfg.SandboxDomain = "cube.app" } + if cfg.Warehouse.Dir == "" { + cfg.Warehouse.Dir = "/data/cubeops/warehouse" + } + if cfg.Warehouse.WriteTimeout == 0 { + cfg.Warehouse.WriteTimeout = 30 * time.Minute + } + if cfg.Warehouse.FetchTimeout == 0 { + cfg.Warehouse.FetchTimeout = 30 * time.Minute + } + if len(cfg.Warehouse.GitHubRepos) == 0 { + cfg.Warehouse.GitHubRepos = []string{"TencentCloud/CubeSandbox"} + } + if len(cfg.Warehouse.CNBRepos) == 0 { + cfg.Warehouse.CNBRepos = []string{"CubeSandbox/CubeSandbox"} + } // JWT_SECRET is optional — if not set, it will be auto-generated and // persisted to the DB on first startup (see store.bootstrapJWTSecret). @@ -324,4 +354,41 @@ func overrideFromEnv(cfg *Config) { cfg.RefreshTTL = d } } + if v := os.Getenv("CUBE_OPS_WAREHOUSE_DIR"); v != "" { + cfg.Warehouse.Dir = v + } + if v := os.Getenv("CUBE_OPS_WAREHOUSE_WRITE_TIMEOUT"); v != "" { + if d, err := time.ParseDuration(v); err == nil { + cfg.Warehouse.WriteTimeout = d + } + } + if v := os.Getenv("CUBE_OPS_WAREHOUSE_FETCH_TIMEOUT"); v != "" { + if d, err := time.ParseDuration(v); err == nil { + cfg.Warehouse.FetchTimeout = d + } + } + if v := os.Getenv("CUBE_OPS_WAREHOUSE_GITHUB_TOKEN"); v != "" { + cfg.Warehouse.GitHubToken = v + } + if v := os.Getenv("CUBE_OPS_WAREHOUSE_CNB_TOKEN"); v != "" { + cfg.Warehouse.CNBToken = v + } + if v := os.Getenv("CUBE_OPS_WAREHOUSE_GITHUB_REPOS"); v != "" { + cfg.Warehouse.GitHubRepos = splitCSV(v) + } + if v := os.Getenv("CUBE_OPS_WAREHOUSE_CNB_REPOS"); v != "" { + cfg.Warehouse.CNBRepos = splitCSV(v) + } +} + +func splitCSV(v string) []string { + parts := strings.Split(v, ",") + out := make([]string, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + out = append(out, p) + } + } + return out } diff --git a/CubeOps/internal/config/config_test.go b/CubeOps/internal/config/config_test.go index 7aa8d24f1..913402e06 100644 --- a/CubeOps/internal/config/config_test.go +++ b/CubeOps/internal/config/config_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "testing" + "time" ) // TestLoad_FromYAML proves that config.Load() reads values from the YAML @@ -15,7 +16,7 @@ import ( // YAML path is wired up, not just documented. func TestLoad_FromYAML(t *testing.T) { dir := t.TempDir() - yamlPath := filepath.Join(dir, "ops.yaml") + yamlPath := filepath.Join(dir, "config.yaml") yamlContent := []byte(`bind: "0.0.0.0:9999" log_level: "debug" cubemaster_addr: "http://1.2.3.4:8089" @@ -55,7 +56,7 @@ refresh_ttl: "336h" // precedence over YAML values — the documented resolution order. func TestLoad_EnvOverridesYAML(t *testing.T) { dir := t.TempDir() - yamlPath := filepath.Join(dir, "ops.yaml") + yamlPath := filepath.Join(dir, "config.yaml") yamlContent := []byte(`bind: "0.0.0.0:9999" database_url: "mysql://root:pass@127.0.0.1:3306/yamldb" `) @@ -77,7 +78,7 @@ database_url: "mysql://root:pass@127.0.0.1:3306/yamldb" // TestLoad_NoYAML_UsesEnvAndDefaults proves the system still works without // a YAML file — existing deployments using only env vars are unaffected. func TestLoad_NoYAML_UsesEnvAndDefaults(t *testing.T) { - t.Setenv("CUBE_OPS_CONFIG", "/nonexistent/path/ops.yaml") + t.Setenv("CUBE_OPS_CONFIG", "/nonexistent/path/config.yaml") t.Setenv("DATABASE_URL", "mysql://root:pass@127.0.0.1:3306/envdb") cfg, err := Load() @@ -90,11 +91,14 @@ func TestLoad_NoYAML_UsesEnvAndDefaults(t *testing.T) { if cfg.Bind != "127.0.0.1:3010" { t.Errorf("Bind = %q, want default 127.0.0.1:3010", cfg.Bind) } + if cfg.Warehouse.Dir != "/data/cubeops/warehouse" { + t.Errorf("Warehouse.Dir = %q, want default /data/cubeops/warehouse", cfg.Warehouse.Dir) + } } // TestLoad_MissingDB_Fails proves we still require a database URL. func TestLoad_MissingDB_Fails(t *testing.T) { - t.Setenv("CUBE_OPS_CONFIG", "/nonexistent/path/ops.yaml") + t.Setenv("CUBE_OPS_CONFIG", "/nonexistent/path/config.yaml") t.Setenv("DATABASE_URL", "") // Also clear individual MySQL env vars so buildMySQLURL returns "". t.Setenv("CUBE_SANDBOX_MYSQL_HOST", "") @@ -104,3 +108,88 @@ func TestLoad_MissingDB_Fails(t *testing.T) { t.Error("Load with no DB config = nil err, want error") } } + +func TestLoad_WarehouseSection(t *testing.T) { + dir := t.TempDir() + yamlPath := filepath.Join(dir, "config.yaml") + yamlContent := []byte(`database_url: "mysql://root:pass@127.0.0.1:3306/testdb" +warehouse: + dir: "/tmp/wh" + write_timeout: "5m" + fetch_timeout: "10m" + github_repos: + - "acme/box" + cnb_repos: + - "acme/cnb" +`) + if err := os.WriteFile(yamlPath, yamlContent, 0o644); err != nil { + t.Fatalf("write yaml: %v", err) + } + t.Setenv("CUBE_OPS_CONFIG", yamlPath) + + cfg, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.Warehouse.Dir != "/tmp/wh" { + t.Errorf("Warehouse.Dir = %q, want /tmp/wh", cfg.Warehouse.Dir) + } + if cfg.Warehouse.WriteTimeout != 5*time.Minute { + t.Errorf("Warehouse.WriteTimeout = %v, want 5m", cfg.Warehouse.WriteTimeout) + } + if cfg.Warehouse.FetchTimeout != 10*time.Minute { + t.Errorf("Warehouse.FetchTimeout = %v, want 10m", cfg.Warehouse.FetchTimeout) + } + if len(cfg.Warehouse.GitHubRepos) != 1 || cfg.Warehouse.GitHubRepos[0] != "acme/box" { + t.Errorf("Warehouse.GitHubRepos = %v, want [acme/box]", cfg.Warehouse.GitHubRepos) + } + if len(cfg.Warehouse.CNBRepos) != 1 || cfg.Warehouse.CNBRepos[0] != "acme/cnb" { + t.Errorf("Warehouse.CNBRepos = %v, want [acme/cnb]", cfg.Warehouse.CNBRepos) + } +} + +func TestLoad_WarehouseEnvOverridesSection(t *testing.T) { + dir := t.TempDir() + yamlPath := filepath.Join(dir, "config.yaml") + yamlContent := []byte(`database_url: "mysql://root:pass@127.0.0.1:3306/testdb" +warehouse: + dir: "/tmp/wh" +`) + if err := os.WriteFile(yamlPath, yamlContent, 0o644); err != nil { + t.Fatalf("write yaml: %v", err) + } + t.Setenv("CUBE_OPS_CONFIG", yamlPath) + t.Setenv("CUBE_OPS_WAREHOUSE_DIR", "/env/wh") + t.Setenv("CUBE_OPS_WAREHOUSE_GITHUB_REPOS", "env/one,env/two") + t.Setenv("CUBE_OPS_WAREHOUSE_WRITE_TIMEOUT", "7m") + t.Setenv("CUBE_OPS_WAREHOUSE_FETCH_TIMEOUT", "11m") + t.Setenv("CUBE_OPS_WAREHOUSE_CNB_REPOS", "env/cnb") + t.Setenv("CUBE_OPS_WAREHOUSE_GITHUB_TOKEN", "gh-secret") + t.Setenv("CUBE_OPS_WAREHOUSE_CNB_TOKEN", "cnb-secret") + + cfg, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.Warehouse.Dir != "/env/wh" { + t.Errorf("Warehouse.Dir = %q, want /env/wh (env should override YAML)", cfg.Warehouse.Dir) + } + if cfg.Warehouse.WriteTimeout != 7*time.Minute { + t.Errorf("Warehouse.WriteTimeout = %v, want 7m", cfg.Warehouse.WriteTimeout) + } + if cfg.Warehouse.FetchTimeout != 11*time.Minute { + t.Errorf("Warehouse.FetchTimeout = %v, want 11m", cfg.Warehouse.FetchTimeout) + } + if len(cfg.Warehouse.GitHubRepos) != 2 || cfg.Warehouse.GitHubRepos[0] != "env/one" || cfg.Warehouse.GitHubRepos[1] != "env/two" { + t.Errorf("Warehouse.GitHubRepos = %v, want [env/one env/two]", cfg.Warehouse.GitHubRepos) + } + if len(cfg.Warehouse.CNBRepos) != 1 || cfg.Warehouse.CNBRepos[0] != "env/cnb" { + t.Errorf("Warehouse.CNBRepos = %v, want [env/cnb]", cfg.Warehouse.CNBRepos) + } + if cfg.Warehouse.GitHubToken != "gh-secret" { + t.Errorf("Warehouse.GitHubToken = %q, want gh-secret", cfg.Warehouse.GitHubToken) + } + if cfg.Warehouse.CNBToken != "cnb-secret" { + t.Errorf("Warehouse.CNBToken = %q, want cnb-secret", cfg.Warehouse.CNBToken) + } +} diff --git a/CubeOps/internal/config/daoconfig_test.go b/CubeOps/internal/config/daoconfig_test.go index 94917bf22..844b708cd 100644 --- a/CubeOps/internal/config/daoconfig_test.go +++ b/CubeOps/internal/config/daoconfig_test.go @@ -121,7 +121,7 @@ func TestDaoConfig_DefaultPort(t *testing.T) { // dao.Config handed to store.New(). func TestDaoConfig_FullLoadToDaoConfig(t *testing.T) { dir := t.TempDir() - yamlPath := dir + "/ops.yaml" + yamlPath := dir + "/config.yaml" yamlContent := []byte(`database_url: "mysql://loader:loaderpass@192.168.1.10:3306/loaderdb" `) if err := os.WriteFile(yamlPath, yamlContent, 0o644); err != nil { diff --git a/CubeOps/internal/handler/warehouse.go b/CubeOps/internal/handler/warehouse.go new file mode 100644 index 000000000..8b485a0e4 --- /dev/null +++ b/CubeOps/internal/handler/warehouse.go @@ -0,0 +1,543 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package handler + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/tencentcloud/CubeSandbox/CubeOps/internal/httputil" + "github.com/tencentcloud/CubeSandbox/CubeOps/internal/store" + "github.com/tencentcloud/CubeSandbox/CubeOps/internal/warehouse" +) + +const nodeIDHeader = "X-Cube-Node-ID" + +var nodeIDRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +// WarehouseHandler serves admin (JWT) and node (internal, no token) warehouse APIs. +type WarehouseHandler struct { + store *store.Store + layout *warehouse.Layout + importer *warehouse.Importer + cm CubeMasterClient +} + +func NewWarehouseHandler(s *store.Store, layout *warehouse.Layout, importer *warehouse.Importer, cm CubeMasterClient) *WarehouseHandler { + return &WarehouseHandler{store: s, layout: layout, importer: importer, cm: cm} +} + +func (h *WarehouseHandler) RegisterAdmin(r *gin.RouterGroup) { + r.GET("/warehouse/components", h.ListComponents) + r.GET("/warehouse/components/:component", h.GetComponent) + r.DELETE("/warehouse/components/:component/versions/:version", h.DeleteVersion) + r.POST("/warehouse/uploads", h.Upload) + r.GET("/warehouse/imports", h.ListImports) + r.POST("/warehouse/imports", h.CreateImport) + r.GET("/warehouse/imports/:id", h.GetImport) + r.POST("/warehouse/preinstall", h.CreatePreinstall) + r.GET("/warehouse/preinstall", h.ListPreinstall) +} + +func (h *WarehouseHandler) RegisterInternal(r *gin.RouterGroup) { + r.GET("/blob", h.GetBlob) + r.GET("/jobs", h.ListNodeJobs) + r.POST("/jobs/:id/ack", h.AckJob) + r.PUT("/inventory", h.PutInventory) +} + +func (h *WarehouseHandler) ListComponents(c *gin.Context) { + items, err := h.store.ListWarehouseItems(c.Request.Context()) + if err != nil { + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + installs, err := h.store.ListNodeInstalls(c.Request.Context()) + if err != nil { + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + nodeIDs, coverageOK := h.coverageNodeIDs(c.Request.Context()) + summaries := warehouse.SummarizeComponents(items, installs, nodeIDs, coverageOK) + httputil.WriteJSON(c, http.StatusOK, gin.H{"components": summaries}) +} + +func (h *WarehouseHandler) GetComponent(c *gin.Context) { + component, err := warehouse.NormalizeComponent(c.Param("component")) + if err != nil { + httputil.WriteError(c, http.StatusBadRequest, err.Error()) + return + } + items, err := h.store.ListWarehouseItemsByComponent(c.Request.Context(), component) + if err != nil { + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + installs, err := h.store.ListNodeInstalls(c.Request.Context()) + if err != nil { + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + nodeIDs, coverageOK := h.coverageNodeIDs(c.Request.Context()) + detail := warehouse.GroupComponent(component, items, installs, nodeIDs, coverageOK) + httputil.WriteJSON(c, http.StatusOK, detail) +} + +func (h *WarehouseHandler) coverageNodeIDs(ctx context.Context) (ids []string, ok bool) { + if h.cm == nil { + return nil, true + } + raw, err := h.cm.GetNodes(ctx) + if err != nil { + return nil, false + } + var wrap cmNodesResponse + if err := json.Unmarshal(raw, &wrap); err != nil { + return nil, false + } + for _, n := range wrap.Data { + if n.NodeID != "" { + ids = append(ids, n.NodeID) + } + } + return ids, true +} + +func (h *WarehouseHandler) DeleteVersion(c *gin.Context) { + component, err := warehouse.NormalizeComponent(c.Param("component")) + if err != nil { + httputil.WriteError(c, http.StatusBadRequest, err.Error()) + return + } + version, err := warehouse.NormalizeVersion(c.Param("version")) + if err != nil { + httputil.WriteError(c, http.StatusBadRequest, err.Error()) + return + } + arch, err := warehouse.NormalizeArch(c.Query("arch")) + if err != nil { + httputil.WriteError(c, http.StatusBadRequest, "arch query is required (amd64 or arm64)") + return + } + if err := h.store.DeleteWarehouseItem(c.Request.Context(), arch, component, version); err != nil { + if errors.Is(err, sql.ErrNoRows) { + httputil.WriteErrorCode(c, http.StatusNotFound, warehouse.CodeNotFound, "warehouse version not found") + return + } + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + _ = h.layout.RemoveDir(arch, component, version) + _ = h.store.CancelPendingPreinstallForVersion(c.Request.Context(), arch, component, version) + httputil.WriteNoContent(c) +} + +func (h *WarehouseHandler) Upload(c *gin.Context) { + if err := h.layout.EnsureRoot(); err != nil { + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + file, err := c.FormFile("file") + if err != nil { + httputil.WriteError(c, http.StatusBadRequest, "multipart field file is required") + return + } + if !strings.HasSuffix(strings.ToLower(file.Filename), ".tar.gz") && + !strings.HasSuffix(strings.ToLower(file.Filename), ".tgz") { + httputil.WriteError(c, http.StatusBadRequest, "upload must be a .tar.gz one-click package") + return + } + id := uuid.NewString() + dest := filepath.Join(h.layout.UploadsDir(), id+".tar.gz") + src, err := file.Open() + if err != nil { + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + defer src.Close() + out, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + if _, err := io.Copy(out, src); err != nil { + _ = out.Close() + _ = os.Remove(dest) + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + if err := out.Sync(); err != nil { + _ = out.Close() + _ = os.Remove(dest) + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + if err := out.Close(); err != nil { + _ = os.Remove(dest) + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + httputil.WriteJSON(c, http.StatusCreated, gin.H{"uploadId": id, "filename": filepath.Base(file.Filename)}) +} + +type createImportRequest struct { + Source string `json:"source"` + Repo string `json:"repo"` + Tag string `json:"tag"` + UploadID string `json:"uploadId"` + Arch []string `json:"arch"` +} + +func (h *WarehouseHandler) CreateImport(c *gin.Context) { + var req createImportRequest + if err := c.ShouldBindJSON(&req); err != nil { + httputil.WriteError(c, http.StatusBadRequest, "invalid JSON body") + return + } + arches, err := normalizeArchList(req.Arch) + if err != nil { + httputil.WriteError(c, http.StatusBadRequest, err.Error()) + return + } + source := strings.ToLower(strings.TrimSpace(req.Source)) + sourceRef, err := h.importSourceRef(source, req) + if err != nil { + httputil.WriteError(c, http.StatusBadRequest, err.Error()) + return + } + var jobs []store.ImportJob + for _, arch := range arches { + job := store.ImportJob{ + ID: uuid.NewString(), + Source: source, + SourceRef: sourceRef, + Arch: arch, + Status: store.ImportPending, + Tag: strings.TrimSpace(req.Tag), + } + if err := h.store.CreateImportJob(c.Request.Context(), job); err != nil { + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + jobs = append(jobs, job) + } + if h.importer != nil { + h.importer.Kick() + } + httputil.WriteJSON(c, http.StatusAccepted, gin.H{"jobs": jobs}) +} + +func (h *WarehouseHandler) importSourceRef(source string, req createImportRequest) (string, error) { + switch source { + case warehouse.SourceUpload: + if strings.TrimSpace(req.UploadID) == "" { + return "", fmt.Errorf("uploadId is required") + } + path := filepath.Join(h.layout.UploadsDir(), filepath.Base(req.UploadID)+".tar.gz") + if st, err := os.Stat(path); err != nil || !st.Mode().IsRegular() { + return "", fmt.Errorf("upload not found") + } + return path, nil + case warehouse.SourceGitHub, warehouse.SourceCNB: + if strings.TrimSpace(req.Repo) == "" || strings.TrimSpace(req.Tag) == "" { + return "", fmt.Errorf("repo and tag are required") + } + return strings.TrimSpace(req.Repo), nil + default: + return "", fmt.Errorf("source must be github, cnb, or upload") + } +} + +func (h *WarehouseHandler) ListImports(c *gin.Context) { + limit, offset := parsePagination(c) + jobs, total, err := h.store.ListImportJobs(c.Request.Context(), limit, offset) + if err != nil { + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + if jobs == nil { + jobs = []store.ImportJob{} + } + httputil.WriteJSON(c, http.StatusOK, gin.H{"jobs": jobs, "total": total}) +} + +func (h *WarehouseHandler) GetImport(c *gin.Context) { + job, err := h.store.GetImportJob(c.Request.Context(), c.Param("id")) + if err != nil { + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + if job == nil { + httputil.WriteError(c, http.StatusNotFound, "import job not found") + return + } + httputil.WriteJSON(c, http.StatusOK, job) +} + +type createPreinstallRequest struct { + NodeIDs []string `json:"nodeIds"` + Arch string `json:"arch"` + Component string `json:"component"` + Version string `json:"version"` +} + +func (h *WarehouseHandler) CreatePreinstall(c *gin.Context) { + var req createPreinstallRequest + if err := c.ShouldBindJSON(&req); err != nil { + httputil.WriteError(c, http.StatusBadRequest, "invalid JSON body") + return + } + arch, err := warehouse.NormalizeArch(req.Arch) + if err != nil { + httputil.WriteError(c, http.StatusBadRequest, err.Error()) + return + } + component, err := warehouse.NormalizeComponent(req.Component) + if err != nil { + httputil.WriteError(c, http.StatusBadRequest, err.Error()) + return + } + version, err := warehouse.NormalizeVersion(req.Version) + if err != nil { + httputil.WriteError(c, http.StatusBadRequest, err.Error()) + return + } + item, err := h.store.GetWarehouseItem(c.Request.Context(), arch, component, version) + if err != nil { + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + if item == nil { + httputil.WriteErrorCode(c, http.StatusNotFound, warehouse.CodeNotFound, "warehouse version not found") + return + } + if len(req.NodeIDs) == 0 { + httputil.WriteError(c, http.StatusBadRequest, "nodeIds is required") + return + } + var jobs []store.PreinstallJob + for _, rawID := range req.NodeIDs { + nodeID, err := normalizeNodeID(rawID) + if err != nil { + httputil.WriteError(c, http.StatusBadRequest, err.Error()) + return + } + jobs = append(jobs, store.PreinstallJob{ + ID: uuid.NewString(), + NodeID: nodeID, + Arch: arch, + Component: component, + Version: version, + Status: store.PreinstallPending, + }) + } + if err := h.store.CreatePreinstallJobs(c.Request.Context(), jobs); err != nil { + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + httputil.WriteJSON(c, http.StatusAccepted, gin.H{"jobs": jobs}) +} + +func (h *WarehouseHandler) ListPreinstall(c *gin.Context) { + limit, offset := parsePagination(c) + jobs, total, err := h.store.ListPreinstallJobs(c.Request.Context(), c.Query("node_id"), c.Query("status"), limit, offset) + if err != nil { + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + if jobs == nil { + jobs = []store.PreinstallJob{} + } + httputil.WriteJSON(c, http.StatusOK, gin.H{"jobs": jobs, "total": total}) +} + +func (h *WarehouseHandler) GetBlob(c *gin.Context) { + arch, err := warehouse.NormalizeArch(c.Query("arch")) + if err != nil { + httputil.WriteErrorCode(c, http.StatusBadRequest, warehouse.CodeInvalidRequest, err.Error()) + return + } + component, err := warehouse.NormalizeComponent(c.Query("component")) + if err != nil { + httputil.WriteErrorCode(c, http.StatusBadRequest, warehouse.CodeInvalidRequest, err.Error()) + return + } + version, err := warehouse.NormalizeVersion(c.Query("version")) + if err != nil { + httputil.WriteErrorCode(c, http.StatusBadRequest, warehouse.CodeInvalidRequest, err.Error()) + return + } + item, err := h.store.GetWarehouseItem(c.Request.Context(), arch, component, version) + if err != nil { + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + dir := h.layout.Abs(arch, component, version) + if item == nil || !dirExists(dir) { + httputil.WriteErrorCode(c, http.StatusNotFound, warehouse.CodeNotFound, + fmt.Sprintf("warehouse does not have %s %s for arch %s", component, version, arch)) + return + } + c.Header("Content-Type", "application/gzip") + c.Header("Content-Disposition", fmt.Sprintf(`attachment; filename="%s-%s-%s.tar.gz"`, arch, component, version)) + c.Status(http.StatusOK) + if err := warehouse.WriteTarGz(c.Writer, dir); err != nil { + slog.Error("warehouse blob stream failed", "error", err, "dir", dir) + } +} + +func (h *WarehouseHandler) ListNodeJobs(c *gin.Context) { + nodeID, err := requireNodeID(c) + if err != nil { + httputil.WriteErrorCode(c, http.StatusBadRequest, warehouse.CodeInvalidRequest, err.Error()) + return + } + jobs, err := h.store.ListNodePreinstallWork(c.Request.Context(), nodeID, 15*time.Minute) + if err != nil { + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + if jobs == nil { + jobs = []store.PreinstallJob{} + } + httputil.WriteJSON(c, http.StatusOK, gin.H{"jobs": jobs}) +} + +type ackJobRequest struct { + Status string `json:"status"` + Error string `json:"error"` +} + +func (h *WarehouseHandler) AckJob(c *gin.Context) { + nodeID, err := requireNodeID(c) + if err != nil { + httputil.WriteErrorCode(c, http.StatusBadRequest, warehouse.CodeInvalidRequest, err.Error()) + return + } + var req ackJobRequest + if err := c.ShouldBindJSON(&req); err != nil { + httputil.WriteError(c, http.StatusBadRequest, "invalid JSON body") + return + } + status := strings.ToLower(strings.TrimSpace(req.Status)) + switch status { + case store.PreinstallRunning, store.PreinstallSucceeded, store.PreinstallFailed: + default: + httputil.WriteError(c, http.StatusBadRequest, "status must be running, succeeded, or failed") + return + } + if err := h.store.AckPreinstallJob(c.Request.Context(), c.Param("id"), nodeID, status, req.Error); err != nil { + if errors.Is(err, sql.ErrNoRows) { + httputil.WriteErrorCode(c, http.StatusConflict, warehouse.CodeUnauthorizedJob, "job not found for this node") + return + } + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + httputil.WriteJSON(c, http.StatusOK, gin.H{"ok": true}) +} + +type inventoryItemRequest struct { + Component string `json:"component"` + Version string `json:"version"` +} + +type inventoryRequest struct { + Arch string `json:"arch"` + Items []inventoryItemRequest `json:"items"` +} + +func (h *WarehouseHandler) PutInventory(c *gin.Context) { + nodeID, err := requireNodeID(c) + if err != nil { + httputil.WriteErrorCode(c, http.StatusBadRequest, warehouse.CodeInvalidRequest, err.Error()) + return + } + var req inventoryRequest + if err := c.ShouldBindJSON(&req); err != nil { + httputil.WriteError(c, http.StatusBadRequest, "invalid JSON body") + return + } + arch, err := warehouse.NormalizeArch(req.Arch) + if err != nil { + httputil.WriteError(c, http.StatusBadRequest, err.Error()) + return + } + items := make([]store.NodeInstall, 0, len(req.Items)) + for _, raw := range req.Items { + component, err := warehouse.NormalizeComponent(raw.Component) + if err != nil { + httputil.WriteError(c, http.StatusBadRequest, err.Error()) + return + } + version, err := warehouse.NormalizeVersion(raw.Version) + if err != nil { + httputil.WriteError(c, http.StatusBadRequest, err.Error()) + return + } + items = append(items, store.NodeInstall{Component: component, Version: version}) + } + if err := h.store.ReplaceNodeInstalls(c.Request.Context(), nodeID, arch, items); err != nil { + httputil.WriteError(c, http.StatusInternalServerError, err.Error()) + return + } + httputil.WriteJSON(c, http.StatusOK, gin.H{"ok": true}) +} + +func requireNodeID(c *gin.Context) (string, error) { + raw := strings.TrimSpace(c.GetHeader(nodeIDHeader)) + if raw == "" { + raw = strings.TrimSpace(c.Query("node_id")) + } + return normalizeNodeID(raw) +} + +func normalizeNodeID(raw string) (string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", fmt.Errorf("node_id is required") + } + if !nodeIDRe.MatchString(raw) { + return "", fmt.Errorf("invalid node_id") + } + return raw, nil +} + +func normalizeArchList(arches []string) ([]string, error) { + if len(arches) == 0 { + return nil, fmt.Errorf("arch is required") + } + seen := map[string]struct{}{} + var out []string + for _, a := range arches { + n, err := warehouse.NormalizeArch(a) + if err != nil { + return nil, err + } + if _, ok := seen[n]; ok { + continue + } + seen[n] = struct{}{} + out = append(out, n) + } + return out, nil +} + +func dirExists(path string) bool { + st, err := os.Stat(path) + return err == nil && st.IsDir() +} diff --git a/CubeOps/internal/handler/warehouse_catalog_test.go b/CubeOps/internal/handler/warehouse_catalog_test.go new file mode 100644 index 000000000..922096533 --- /dev/null +++ b/CubeOps/internal/handler/warehouse_catalog_test.go @@ -0,0 +1,172 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package handler_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/tencentcloud/CubeSandbox/CubeOps/internal/handler" + "github.com/tencentcloud/CubeSandbox/CubeOps/internal/store" + "github.com/tencentcloud/CubeSandbox/CubeOps/internal/warehouse" +) + +func TestWarehouseCatalogHTTP_EmptyAndVersions(t *testing.T) { + env := newTestEnv(t) + defer env.teardown() + + wh := handler.NewWarehouseHandler(env.store, warehouse.NewLayout(t.TempDir()), nil, env.fakeCM) + r := gin.New() + wh.RegisterAdmin(r.Group("/api/v1")) + + list := getJSON(t, r, "/api/v1/warehouse/components") + comps, _ := list["components"].([]any) + if len(comps) != 4 { + t.Fatalf("components=%d want 4 body=%s", len(comps), mustJSON(list)) + } + for _, raw := range comps { + row := raw.(map[string]any) + if row["versionCount"].(float64) != 0 { + t.Errorf("%s versionCount=%v want 0", row["name"], row["versionCount"]) + } + if _, ok := row["nodesMissing"]; ok { + t.Errorf("%s nodesMissing should be omitted when CubeMaster nodes fail, got %v", row["name"], row["nodesMissing"]) + } + } + + ctx := context.Background() + mustInsert := func(item store.WarehouseItem) { + t.Helper() + ok, err := env.store.InsertWarehouseItem(ctx, item) + if err != nil || !ok { + t.Fatalf("insert %+v: ok=%v err=%v", item, ok, err) + } + } + mustInsert(store.WarehouseItem{ + Arch: "amd64", Component: "cube-shim", Version: "v0.6.0", + Source: "github", SourceRef: "TencentCloud/CubeSandbox", + RelPath: "amd64/cube-shim/v0.6.0", SizeBytes: 100, Checksum: "sha256:a", + }) + mustInsert(store.WarehouseItem{ + Arch: "arm64", Component: "cube-shim", Version: "v0.6.0", + Source: "github", SourceRef: "TencentCloud/CubeSandbox", + RelPath: "arm64/cube-shim/v0.6.0", SizeBytes: 80, Checksum: "sha256:b", + }) + mustInsert(store.WarehouseItem{ + Arch: "amd64", Component: "cube-shim", Version: "v0.5.0", + Source: "github", SourceRef: "TencentCloud/CubeSandbox", + RelPath: "amd64/cube-shim/v0.5.0", SizeBytes: 90, Checksum: "sha256:c", + }) + + list = getJSON(t, r, "/api/v1/warehouse/components") + shim := findComponent(t, list, "cube-shim") + if shim["versionCount"].(float64) != 2 { + t.Errorf("versionCount=%v want 2", shim["versionCount"]) + } + if shim["sizeBytes"].(float64) != 270 { + t.Errorf("sizeBytes=%v want 270", shim["sizeBytes"]) + } + + detail := getJSON(t, r, "/api/v1/warehouse/components/cube-shim") + if detail["name"] != "cube-shim" { + t.Fatalf("name=%v", detail["name"]) + } + versions, _ := detail["versions"].([]any) + if len(versions) != 2 { + t.Fatalf("versions=%d want 2: %s", len(versions), mustJSON(detail)) + } + first := versions[0].(map[string]any) + if first["version"] != "v0.6.0" { + t.Errorf("first version=%v want v0.6.0", first["version"]) + } + arts, _ := first["artifacts"].([]any) + if len(arts) != 2 { + t.Fatalf("v0.6.0 artifacts=%d want 2", len(arts)) + } + if arts[0].(map[string]any)["arch"] != "amd64" { + t.Errorf("arch order=%v want amd64 first", arts[0].(map[string]any)["arch"]) + } + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/warehouse/components/cubelet", nil) + r.ServeHTTP(w, req) + if w.Code != http.StatusBadRequest { + t.Fatalf("unknown component status=%d want 400 body=%s", w.Code, w.Body.String()) + } +} + +func getJSON(t *testing.T, r *gin.Engine, path string) map[string]any { + t.Helper() + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, path, nil) + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("%s status=%d body=%s", path, w.Code, w.Body.String()) + } + var out map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil { + t.Fatalf("json %s: %v body=%s", path, err, w.Body.String()) + } + return out +} + +func findComponent(t *testing.T, list map[string]any, name string) map[string]any { + t.Helper() + comps, _ := list["components"].([]any) + for _, raw := range comps { + row := raw.(map[string]any) + if row["name"] == name { + return row + } + } + t.Fatalf("component %s not found in %s", name, mustJSON(list)) + return nil +} + +func mustJSON(v any) string { + b, _ := json.Marshal(v) + return string(b) +} + +func TestListImportsHTTP(t *testing.T) { + env := newTestEnv(t) + defer env.teardown() + + wh := handler.NewWarehouseHandler(env.store, warehouse.NewLayout(t.TempDir()), nil, env.fakeCM) + r := gin.New() + wh.RegisterAdmin(r.Group("/api/v1")) + + empty := getJSON(t, r, "/api/v1/warehouse/imports") + jobs, _ := empty["jobs"].([]any) + if len(jobs) != 0 { + t.Fatalf("empty jobs=%v", jobs) + } + if empty["total"].(float64) != 0 { + t.Fatalf("empty total=%v", empty["total"]) + } + + if err := env.store.CreateImportJob(context.Background(), store.ImportJob{ + ID: "imp-1", Source: "github", SourceRef: "TencentCloud/CubeSandbox", + Tag: "v0.6.0", Arch: "amd64", Status: store.ImportPending, + }); err != nil { + t.Fatalf("create: %v", err) + } + + listed := getJSON(t, r, "/api/v1/warehouse/imports") + jobs, _ = listed["jobs"].([]any) + if len(jobs) != 1 { + t.Fatalf("jobs=%d body=%s", len(jobs), mustJSON(listed)) + } + if listed["total"].(float64) != 1 { + t.Fatalf("total=%v body=%s", listed["total"], mustJSON(listed)) + } + row := jobs[0].(map[string]any) + if row["id"] != "imp-1" || row["status"] != store.ImportPending { + t.Fatalf("row=%s", mustJSON(row)) + } +} diff --git a/CubeOps/internal/handler/warehouse_test.go b/CubeOps/internal/handler/warehouse_test.go new file mode 100644 index 000000000..e51d7502b --- /dev/null +++ b/CubeOps/internal/handler/warehouse_test.go @@ -0,0 +1,111 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package handler + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/tencentcloud/CubeSandbox/CubeOps/internal/auth" + "github.com/tencentcloud/CubeSandbox/CubeOps/internal/warehouse" +) + +func TestWarehouseInternalBlob_BadArch(t *testing.T) { + gin.SetMode(gin.TestMode) + h := NewWarehouseHandler(nil, warehouse.NewLayout(t.TempDir()), nil, nil) + r := gin.New() + g := r.Group("/internal/warehouse") + h.RegisterInternal(g) + + w := httptestRecorder(t, r, "GET", "/internal/warehouse/blob?arch=ppc64&component=cube-shim&version=v0.6.0") + if w.Code != http.StatusBadRequest { + t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) + } +} + +func TestWarehouseInternalRequiresNodeID(t *testing.T) { + gin.SetMode(gin.TestMode) + h := NewWarehouseHandler(nil, warehouse.NewLayout(t.TempDir()), nil, nil) + r := gin.New() + g := r.Group("/internal/warehouse") + h.RegisterInternal(g) + w := httptestRecorder(t, r, "GET", "/internal/warehouse/jobs") + if w.Code != http.StatusBadRequest { + t.Fatalf("status=%d want 400 body=%s", w.Code, w.Body.String()) + } + w = httptestRecorder(t, r, "PUT", "/internal/warehouse/inventory", `{"arch":"amd64","items":[]}`) + if w.Code != http.StatusBadRequest { + t.Fatalf("inventory without node: status=%d want 400 body=%s", w.Code, w.Body.String()) + } +} + +func TestWarehouseInternalInventory_BadJSON(t *testing.T) { + gin.SetMode(gin.TestMode) + h := NewWarehouseHandler(nil, warehouse.NewLayout(t.TempDir()), nil, nil) + r := gin.New() + h.RegisterInternal(r.Group("/internal/warehouse")) + req := httptest.NewRequest("PUT", "/internal/warehouse/inventory", strings.NewReader(`{`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Cube-Node-ID", "node-1") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusBadRequest { + t.Fatalf("status=%d want 400 body=%s", w.Code, w.Body.String()) + } +} + +func TestWarehouseInternalInventory_BadComponent(t *testing.T) { + gin.SetMode(gin.TestMode) + h := NewWarehouseHandler(nil, warehouse.NewLayout(t.TempDir()), nil, nil) + r := gin.New() + h.RegisterInternal(r.Group("/internal/warehouse")) + req := httptest.NewRequest("PUT", "/internal/warehouse/inventory", strings.NewReader( + `{"arch":"amd64","items":[{"component":"cubelet","version":"v1"}]}`, + )) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Cube-Node-ID", "node-1") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusBadRequest { + t.Fatalf("status=%d want 400 body=%s", w.Code, w.Body.String()) + } +} + +func TestWarehouseAdminRequiresJWT(t *testing.T) { + gin.SetMode(gin.TestMode) + jm := auth.NewJWTManager("test-secret-32-bytes-long-enough!", time.Minute, time.Hour) + h := NewWarehouseHandler(nil, warehouse.NewLayout(t.TempDir()), nil, nil) + r := gin.New() + authed := r.Group("/api/v1", auth.Middleware(jm)) + h.RegisterAdmin(authed) + + w := httptestRecorder(t, r, "GET", "/api/v1/warehouse/components") + if w.Code != http.StatusUnauthorized { + t.Fatalf("status=%d want 401 body=%s", w.Code, w.Body.String()) + } +} + +func TestWarehouseGetComponent_UnknownName(t *testing.T) { + gin.SetMode(gin.TestMode) + h := NewWarehouseHandler(nil, warehouse.NewLayout(t.TempDir()), nil, nil) + r := gin.New() + h.RegisterAdmin(r.Group("/api/v1")) + w := httptestRecorder(t, r, "GET", "/api/v1/warehouse/components/cubelet") + if w.Code != http.StatusBadRequest { + t.Fatalf("status=%d want 400 body=%s", w.Code, w.Body.String()) + } +} + +func TestKnownComponentCatalog(t *testing.T) { + if !warehouse.KnownComponent("cube-shim") { + t.Fatal("shim should be known") + } + if warehouse.KnownComponent("not-a-component") { + t.Fatal("unknown should be rejected") + } +} diff --git a/CubeOps/internal/httputil/response.go b/CubeOps/internal/httputil/response.go index a4c3594da..ebd8d0fbd 100644 --- a/CubeOps/internal/httputil/response.go +++ b/CubeOps/internal/httputil/response.go @@ -26,6 +26,10 @@ func WriteError(c *gin.Context, status int, msg string) { c.JSON(status, model.APIError{Error: msg}) } +func WriteErrorCode(c *gin.Context, status int, code, msg string) { + c.JSON(status, model.APIError{Error: msg, Code: code}) +} + // WriteRawJSON writes a pre-encoded JSON body verbatim (used when proxying // CubeMaster responses without re-marshalling). func WriteRawJSON(c *gin.Context, status int, raw json.RawMessage) { diff --git a/CubeOps/internal/model/types.go b/CubeOps/internal/model/types.go index f49664732..dc35342a7 100644 --- a/CubeOps/internal/model/types.go +++ b/CubeOps/internal/model/types.go @@ -45,4 +45,5 @@ type RefreshResponse struct { // APIError is a generic error response. type APIError struct { Error string `json:"error"` + Code string `json:"code,omitempty"` } diff --git a/CubeOps/internal/server/server.go b/CubeOps/internal/server/server.go index 6c274ea7e..8b5fbfbc1 100644 --- a/CubeOps/internal/server/server.go +++ b/CubeOps/internal/server/server.go @@ -20,27 +20,42 @@ import ( "github.com/tencentcloud/CubeSandbox/CubeOps/internal/logging" "github.com/tencentcloud/CubeSandbox/CubeOps/internal/service" "github.com/tencentcloud/CubeSandbox/CubeOps/internal/store" + "github.com/tencentcloud/CubeSandbox/CubeOps/internal/warehouse" cubelog "github.com/tencentcloud/CubeSandbox/cubelog" ) // Server is the CubeOps HTTP server. type Server struct { - cfg *config.Config - store *store.Store - jm *auth.JWTManager - httpSrv *http.Server - cm *cubemaster.Client + cfg *config.Config + store *store.Store + jm *auth.JWTManager + httpSrv *http.Server + cm *cubemaster.Client + importer *warehouse.Importer + layout *warehouse.Layout + cancel context.CancelFunc } // New creates a new CubeOps server. func New(cfg *config.Config, s *store.Store) *Server { jm := auth.NewJWTManager(cfg.JWTSecret, cfg.AccessTTL, cfg.RefreshTTL) cm := cubemaster.New(cfg.CubeMasterAddr) + layout := warehouse.NewLayout(cfg.Warehouse.Dir) + fetch := warehouse.FetchConfig{ + GitHubRepos: cfg.Warehouse.GitHubRepos, + CNBRepos: cfg.Warehouse.CNBRepos, + GitHubToken: cfg.Warehouse.GitHubToken, + CNBToken: cfg.Warehouse.CNBToken, + Timeout: cfg.Warehouse.FetchTimeout, + } + importer := warehouse.NewImporter(s, layout, fetch) return &Server{ - cfg: cfg, - store: s, - jm: jm, - cm: cm, + cfg: cfg, + store: s, + jm: jm, + cm: cm, + importer: importer, + layout: layout, } } @@ -48,11 +63,19 @@ func New(cfg *config.Config, s *store.Store) *Server { func (s *Server) Start() error { engine := s.buildRouter() + writeTimeout := s.cfg.Warehouse.WriteTimeout + if writeTimeout <= 0 { + writeTimeout = 30 * time.Minute + } + impCtx, cancel := context.WithCancel(context.Background()) + s.cancel = cancel + go s.importer.Run(impCtx) + s.httpSrv = &http.Server{ Addr: s.cfg.Bind, Handler: engine, - ReadHeaderTimeout: 10 * time.Second, // mitigate Slowloris attacks - WriteTimeout: 300 * time.Second, // match nginx proxy_read_timeout + ReadHeaderTimeout: 10 * time.Second, // mitigate Slowloris attacks + WriteTimeout: writeTimeout, // blob download / large upload response IdleTimeout: 120 * time.Second, // ReadTimeout is intentionally NOT set. Go's http.Server.ReadTimeout // covers the entire request body read AND cancels the request context @@ -71,6 +94,9 @@ func (s *Server) Shutdown(ctx context.Context) error { return nil } logging.G(ctx).Info("CubeOps shutting down") + if s.cancel != nil { + s.cancel() + } return s.httpSrv.Shutdown(ctx) } @@ -81,6 +107,7 @@ func (s *Server) buildRouter() *gin.Engine { // to stdout and bypasses any logger the operator has configured. gin.SetMode(gin.ReleaseMode) r := gin.New() + r.MaxMultipartMemory = 64 << 20 r.Use(requestLogger()) r.Use(cubeopsRecovery()) @@ -115,6 +142,11 @@ func (s *Server) buildRouter() *gin.Engine { storeH.Register(authed) agenthubH.Register(authed) + warehouseH := handler.NewWarehouseHandler(s.store, s.layout, s.importer, s.cm) + warehouseH.RegisterAdmin(authed) + internalWH := r.Group("/internal/warehouse") + warehouseH.RegisterInternal(internalWH) + // SDK routes — mounted at both /api/v1/sdk and /api/v1/sdk/v2 because // the WebUI and the E2B-compatible clients hit different prefixes. sdkGroup := authed.Group("/sdk") diff --git a/CubeOps/internal/store/dialect.go b/CubeOps/internal/store/dialect.go index a13e086d8..282cb6e9f 100644 --- a/CubeOps/internal/store/dialect.go +++ b/CubeOps/internal/store/dialect.go @@ -166,3 +166,13 @@ func formatTimestamp(col string) string { } return "DATE_FORMAT(" + col + ", '%Y-%m-%dT%H:%i:%sZ')" } + +// olderThanDurationSQL is true when col is older than ? seconds on the +// database clock. Passing a Go time.Time cutoff is wrong: driver location +// vs DATETIME/timestamp without time zone makes a live row look stale. +func olderThanDurationSQL(col string) string { + if IsPostgres() { + return "EXTRACT(EPOCH FROM (NOW() - " + col + ")) > ?" + } + return "TIMESTAMPDIFF(SECOND, " + col + ", NOW()) > ?" +} diff --git a/CubeOps/internal/store/dockertest_fixture_test.go b/CubeOps/internal/store/dockertest_fixture_test.go index a6913849e..5dc677e29 100644 --- a/CubeOps/internal/store/dockertest_fixture_test.go +++ b/CubeOps/internal/store/dockertest_fixture_test.go @@ -17,24 +17,29 @@ import ( "github.com/tencentcloud/CubeSandbox/CubeDB/dao" "github.com/tencentcloud/CubeSandbox/CubeOps/internal/crypto" "github.com/tencentcloud/CubeSandbox/CubeOps/internal/store" + + _ "github.com/jackc/pgx/v5/stdlib" ) -// dockertest_fixture_test.go provides a shared MySQL test environment for -// CubeOps integration tests. It mirrors the pattern used by -// cubedb/migrate/dockertest_fixture_test.go: spin up a throwaway MySQL 8.0 -// container, run migrations, and hand the caller a fully-initialised *store.Store. +// dockertest_fixture_test.go provides shared MySQL and PostgreSQL test +// environments for CubeOps store tests. It mirrors CubeDB/migrate's +// dockertest: throwaway containers, goose migrations, ready *store.Store. // // Docker missing → skip locally; CUBEOPS_REQUIRE_DOCKER_TESTS=1 or CI=true → Fatal. const ( mysqlImageTag = "8.0" + postgresImageTag = "16-alpine" requireDockerTestsEnv = "CUBEOPS_REQUIRE_DOCKER_TESTS" containerProbeTimeout = 90 * time.Second + engineMySQL = "mysql" + enginePostgres = "postgres" ) // testStoreEnv holds the test database connection + teardown function. type testStoreEnv struct { store *store.Store + engine string dsn string teardown func() } @@ -62,6 +67,11 @@ func abortOrSkipDocker(t *testing.T, format string, args ...any) { // env.teardown(). func newTestStore(t *testing.T) *testStoreEnv { t.Helper() + return newMySQLTestStore(t) +} + +func newMySQLTestStore(t *testing.T) *testStoreEnv { + t.Helper() pool, err := dockertest.NewPool("") if err != nil { @@ -87,9 +97,8 @@ func newTestStore(t *testing.T) *testStoreEnv { } port := resource.GetPort("3306/tcp") - // The DAO config uses individual fields, not a DSN string. cfg := dao.Config{ - Driver: "mysql", + Driver: engineMySQL, User: "root", Pwd: "root", Addr: fmt.Sprintf("127.0.0.1:%s", port), @@ -97,10 +106,11 @@ func newTestStore(t *testing.T) *testStoreEnv { MaxIdleConns: 5, MaxOpenConns: 10, } + dsn := fmt.Sprintf("root:root@tcp(127.0.0.1:%s)/cubeops_test?charset=utf8&parseTime=true", port) pool.MaxWait = containerProbeTimeout if err := pool.Retry(func() error { - db, err := sql.Open("mysql", fmt.Sprintf("root:root@tcp(127.0.0.1:%s)/cubeops_test?charset=utf8&parseTime=true", port)) + db, err := sql.Open("mysql", dsn) if err != nil { return err } @@ -111,6 +121,67 @@ func newTestStore(t *testing.T) *testStoreEnv { t.Fatalf("mysql container never became reachable: %v", err) } + return bootstrapTestStore(t, pool, resource, cfg, engineMySQL, dsn) +} + +func newPostgresTestStore(t *testing.T) *testStoreEnv { + t.Helper() + + pool, err := dockertest.NewPool("") + if err != nil { + abortOrSkipDocker(t, "dockertest not available (%v)", err) + } + if err := pool.Client.Ping(); err != nil { + abortOrSkipDocker(t, "docker daemon not reachable (%v)", err) + } + + resource, err := pool.RunWithOptions(&dockertest.RunOptions{ + Repository: "postgres", + Tag: postgresImageTag, + Env: []string{ + "POSTGRES_USER=cube", + "POSTGRES_PASSWORD=cube_pass", + "POSTGRES_DB=cubeops_test", + }, + }, func(hostConfig *docker.HostConfig) { + hostConfig.AutoRemove = true + hostConfig.RestartPolicy = docker.RestartPolicy{Name: "no"} + }) + if err != nil { + abortOrSkipDocker(t, "could not start postgres container (%v)", err) + } + + port := resource.GetPort("5432/tcp") + cfg := dao.Config{ + Driver: enginePostgres, + User: "cube", + Pwd: "cube_pass", + Addr: fmt.Sprintf("127.0.0.1:%s", port), + DBName: "cubeops_test", + MaxIdleConns: 5, + MaxOpenConns: 10, + } + dsn := fmt.Sprintf("host=127.0.0.1 port=%s user=cube password=cube_pass dbname=cubeops_test sslmode=disable", port) + + pool.MaxWait = containerProbeTimeout + if err := pool.Retry(func() error { + db, err := sql.Open("pgx", dsn) + if err != nil { + return err + } + defer db.Close() + return db.Ping() + }); err != nil { + _ = pool.Purge(resource) + t.Fatalf("postgres container never became reachable: %v", err) + } + + return bootstrapTestStore(t, pool, resource, cfg, enginePostgres, dsn) +} + +func bootstrapTestStore(t *testing.T, pool *dockertest.Pool, resource *dockertest.Resource, cfg dao.Config, engine, dsn string) *testStoreEnv { + t.Helper() + // store.New() handles the full bootstrap: // 1. dao.Open → connects + runs goose migrations // 2. bootstrapMasterKey → generates + installs the crypto master key @@ -127,15 +198,32 @@ func newTestStore(t *testing.T) *testStoreEnv { s, err := store.New(ctx, cfg) if err != nil { _ = pool.Purge(resource) - t.Fatalf("store.New: %v", err) + t.Fatalf("store.New (%s): %v", engine, err) } return &testStoreEnv{ - store: s, - dsn: fmt.Sprintf("root:root@tcp(127.0.0.1:%s)/cubeops_test", port), + store: s, + engine: engine, + dsn: dsn, teardown: func() { _ = s.Close() _ = pool.Purge(resource) }, } } + +// withWarehouseStores runs fn against a fresh MySQL store and a fresh +// PostgreSQL store so warehouse SELECT/INSERT paths are exercised on both. +func withWarehouseStores(t *testing.T, fn func(t *testing.T, s *store.Store)) { + t.Helper() + t.Run(engineMySQL, func(t *testing.T) { + env := newMySQLTestStore(t) + defer env.teardown() + fn(t, env.store) + }) + t.Run(enginePostgres, func(t *testing.T) { + env := newPostgresTestStore(t) + defer env.teardown() + fn(t, env.store) + }) +} diff --git a/CubeOps/internal/store/warehouse.go b/CubeOps/internal/store/warehouse.go new file mode 100644 index 000000000..5dc8ca353 --- /dev/null +++ b/CubeOps/internal/store/warehouse.go @@ -0,0 +1,547 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" + + "gorm.io/gorm" +) + +const ( + ImportPending = "pending" + ImportRunning = "running" + ImportSucceeded = "succeeded" + ImportFailed = "failed" + + PreinstallPending = "pending" + PreinstallRunning = "running" + PreinstallSucceeded = "succeeded" + PreinstallFailed = "failed" + PreinstallCancelled = "cancelled" +) + +// WarehouseItem is one archived component version. +type WarehouseItem struct { + Arch string `json:"arch"` + Component string `json:"component"` + Version string `json:"version"` + Source string `json:"source"` + SourceRef string `json:"sourceRef"` + RelPath string `json:"relPath"` + SizeBytes int64 `json:"sizeBytes"` + Checksum string `json:"checksum"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +// ImportJob is an asynchronous one-click import. +type ImportJob struct { + ID string `json:"id"` + Source string `json:"source"` + SourceRef string `json:"sourceRef"` + Tag string `json:"tag"` + Arch string `json:"arch"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + BytesTotal int64 `json:"bytesTotal"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +// PreinstallJob is a node-side install request. +type PreinstallJob struct { + ID string `json:"id"` + NodeID string `json:"nodeId"` + Arch string `json:"arch"` + Component string `json:"component"` + Version string `json:"version"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +// NodeInstall is one locally inventoried version on a node. +type NodeInstall struct { + NodeID string `json:"nodeId"` + Arch string `json:"arch"` + Component string `json:"component"` + Version string `json:"version"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func (s *Store) InsertWarehouseItem(ctx context.Context, item WarehouseItem) (inserted bool, err error) { + q := insertIgnorePrefix() + + ` INTO t_component_warehouse (arch, component, version, source, source_ref, rel_path, size_bytes, checksum) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + onConflictDoNothing() + res := s.db.WithContext(ctx).Exec(q, + item.Arch, item.Component, item.Version, item.Source, item.SourceRef, + item.RelPath, item.SizeBytes, item.Checksum, + ) + if res.Error != nil { + return false, fmt.Errorf("insert warehouse item: %w", res.Error) + } + return res.RowsAffected > 0, nil +} + +func (s *Store) GetWarehouseItem(ctx context.Context, arch, component, version string) (*WarehouseItem, error) { + var item WarehouseItem + err := s.db.WithContext(ctx).Raw( + `SELECT arch, component, version, source, source_ref, rel_path, size_bytes, checksum, created_at, updated_at + FROM t_component_warehouse WHERE arch = ? AND component = ? AND version = ? LIMIT 1`, + arch, component, version, + ).Row().Scan( + &item.Arch, &item.Component, &item.Version, &item.Source, &item.SourceRef, + &item.RelPath, &item.SizeBytes, &item.Checksum, &item.CreatedAt, &item.UpdatedAt, + ) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return nil, fmt.Errorf("get warehouse item: %w", err) + } + return &item, nil +} + +func (s *Store) ListWarehouseItems(ctx context.Context) ([]WarehouseItem, error) { + rows, err := s.db.WithContext(ctx).Raw( + `SELECT arch, component, version, source, source_ref, rel_path, size_bytes, checksum, created_at, updated_at + FROM t_component_warehouse ORDER BY component, version, arch`, + ).Rows() + if err != nil { + return nil, fmt.Errorf("list warehouse: %w", err) + } + defer rows.Close() + return scanWarehouseItems(rows) +} + +func (s *Store) ListWarehouseItemsByComponent(ctx context.Context, component string) ([]WarehouseItem, error) { + rows, err := s.db.WithContext(ctx).Raw( + `SELECT arch, component, version, source, source_ref, rel_path, size_bytes, checksum, created_at, updated_at + FROM t_component_warehouse WHERE component = ? ORDER BY version, arch`, + component, + ).Rows() + if err != nil { + return nil, fmt.Errorf("list warehouse by component: %w", err) + } + defer rows.Close() + return scanWarehouseItems(rows) +} + +func scanWarehouseItems(rows *sql.Rows) ([]WarehouseItem, error) { + var out []WarehouseItem + for rows.Next() { + var item WarehouseItem + if err := rows.Scan( + &item.Arch, &item.Component, &item.Version, &item.Source, &item.SourceRef, + &item.RelPath, &item.SizeBytes, &item.Checksum, &item.CreatedAt, &item.UpdatedAt, + ); err != nil { + return nil, err + } + out = append(out, item) + } + return out, rows.Err() +} + +func (s *Store) DeleteWarehouseItem(ctx context.Context, arch, component, version string) error { + res := s.db.WithContext(ctx).Exec( + `DELETE FROM t_component_warehouse WHERE arch = ? AND component = ? AND version = ?`, + arch, component, version, + ) + if res.Error != nil { + return fmt.Errorf("delete warehouse item: %w", res.Error) + } + if res.RowsAffected == 0 { + return sql.ErrNoRows + } + return nil +} + +func (s *Store) CreateImportJob(ctx context.Context, job ImportJob) error { + res := s.db.WithContext(ctx).Exec( + `INSERT INTO t_component_import_job (id, source, source_ref, tag, arch, status, error, bytes_total) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + job.ID, job.Source, job.SourceRef, job.Tag, job.Arch, job.Status, nullIfEmpty(job.Error), job.BytesTotal, + ) + if res.Error != nil { + return fmt.Errorf("create import job: %w", res.Error) + } + return nil +} + +func (s *Store) GetImportJob(ctx context.Context, id string) (*ImportJob, error) { + var job ImportJob + var errMsg sql.NullString + err := s.db.WithContext(ctx).Raw( + `SELECT id, source, source_ref, tag, arch, status, error, bytes_total, created_at, updated_at + FROM t_component_import_job WHERE id = ? LIMIT 1`, id, + ).Row().Scan( + &job.ID, &job.Source, &job.SourceRef, &job.Tag, &job.Arch, &job.Status, + &errMsg, &job.BytesTotal, &job.CreatedAt, &job.UpdatedAt, + ) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return nil, fmt.Errorf("get import job: %w", err) + } + job.Error = errMsg.String + return &job, nil +} + +func staleAfterSeconds(d time.Duration) int64 { + sec := int64(d / time.Second) + if d > 0 && sec < 1 { + sec = 1 + } + return sec +} + +// ListImportWork returns pending jobs plus running jobs whose updated_at is +// older than staleAfter, so another replica can reclaim a dead worker. +func (s *Store) ListImportWork(ctx context.Context, staleAfter time.Duration) ([]ImportJob, error) { + rows, err := s.db.WithContext(ctx).Raw( + `SELECT id, source, source_ref, tag, arch, status, error, bytes_total, created_at, updated_at + FROM t_component_import_job + WHERE status = ? OR (status = ? AND `+olderThanDurationSQL("updated_at")+`) + ORDER BY created_at ASC`, + ImportPending, ImportRunning, staleAfterSeconds(staleAfter), + ).Rows() + if err != nil { + return nil, fmt.Errorf("list import work: %w", err) + } + defer rows.Close() + return scanImportJobs(rows) +} + +// ClaimImportJob atomically takes a pending job, or a running job that has +// been stale longer than staleAfter. Returns true when this caller owns it. +func (s *Store) ClaimImportJob(ctx context.Context, id string, staleAfter time.Duration) (bool, error) { + res := s.db.WithContext(ctx).Exec( + `UPDATE t_component_import_job + SET status = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND ( + status = ? + OR (status = ? AND `+olderThanDurationSQL("updated_at")+`) + )`, + ImportRunning, id, ImportPending, ImportRunning, staleAfterSeconds(staleAfter), + ) + if res.Error != nil { + return false, fmt.Errorf("claim import job: %w", res.Error) + } + return res.RowsAffected == 1, nil +} + +func clampListLimit(limit, offset int) (int, int) { + if limit <= 0 { + limit = DefaultListLimit + } + if limit > MaxListLimit { + limit = MaxListLimit + } + if offset < 0 { + offset = 0 + } + return limit, offset +} + +func (s *Store) ListImportJobs(ctx context.Context, limit, offset int) ([]ImportJob, int, error) { + limit, offset = clampListLimit(limit, offset) + var total int64 + if err := s.db.WithContext(ctx).Raw( + `SELECT COUNT(*) FROM t_component_import_job`, + ).Scan(&total).Error; err != nil { + return nil, 0, fmt.Errorf("count import jobs: %w", err) + } + rows, err := s.db.WithContext(ctx).Raw( + `SELECT id, source, source_ref, tag, arch, status, error, bytes_total, created_at, updated_at + FROM t_component_import_job ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?`, + limit, offset, + ).Rows() + if err != nil { + return nil, 0, fmt.Errorf("list import jobs: %w", err) + } + defer rows.Close() + jobs, err := scanImportJobs(rows) + if err != nil { + return nil, 0, err + } + return jobs, int(total), nil +} + +func (s *Store) UpdateImportJob(ctx context.Context, id, status, errMsg string, bytesTotal int64) error { + res := s.db.WithContext(ctx).Exec( + `UPDATE t_component_import_job SET status = ?, error = ?, bytes_total = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?`, + status, nullIfEmpty(errMsg), bytesTotal, id, + ) + if res.Error != nil { + return fmt.Errorf("update import job: %w", res.Error) + } + return nil +} + +func (s *Store) CountLiveImportJobsBySourceRef(ctx context.Context, sourceRef string) (int, error) { + var n int64 + if err := s.db.WithContext(ctx).Raw( + `SELECT COUNT(*) FROM t_component_import_job + WHERE source_ref = ? AND status IN (?, ?)`, + sourceRef, ImportPending, ImportRunning, + ).Scan(&n).Error; err != nil { + return 0, fmt.Errorf("count live import jobs: %w", err) + } + return int(n), nil +} + +func (s *Store) CreatePreinstallJobs(ctx context.Context, jobs []PreinstallJob) error { + if len(jobs) == 0 { + return nil + } + return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + for _, job := range jobs { + if err := tx.Exec( + `INSERT INTO t_component_preinstall_job (id, node_id, arch, component, version, status, error) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + job.ID, job.NodeID, job.Arch, job.Component, job.Version, job.Status, nullIfEmpty(job.Error), + ).Error; err != nil { + return fmt.Errorf("create preinstall job: %w", err) + } + } + return nil + }) +} + +func (s *Store) ListPreinstallJobs(ctx context.Context, nodeID, status string, limit, offset int) ([]PreinstallJob, int, error) { + limit, offset = clampListLimit(limit, offset) + where := ` WHERE 1=1` + args := []any{} + if nodeID != "" { + where += ` AND node_id = ?` + args = append(args, nodeID) + } + if status != "" { + where += ` AND status = ?` + args = append(args, status) + } + var total int64 + if err := s.db.WithContext(ctx).Raw( + `SELECT COUNT(*) FROM t_component_preinstall_job`+where, args..., + ).Scan(&total).Error; err != nil { + return nil, 0, fmt.Errorf("count preinstall jobs: %w", err) + } + rows, err := s.db.WithContext(ctx).Raw( + `SELECT id, node_id, arch, component, version, status, error, created_at, updated_at + FROM t_component_preinstall_job`+where+` ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?`, + append(append([]any{}, args...), limit, offset)..., + ).Rows() + if err != nil { + return nil, 0, fmt.Errorf("list preinstall jobs: %w", err) + } + defer rows.Close() + jobs, err := scanPreinstallJobs(rows) + if err != nil { + return nil, 0, err + } + return jobs, int(total), nil +} + +func (s *Store) ListNodePreinstallWork(ctx context.Context, nodeID string, staleAfter time.Duration) ([]PreinstallJob, error) { + rows, err := s.db.WithContext(ctx).Raw( + `SELECT id, node_id, arch, component, version, status, error, created_at, updated_at + FROM t_component_preinstall_job + WHERE node_id = ? AND (status = ? OR (status = ? AND `+olderThanDurationSQL("updated_at")+`)) + ORDER BY created_at ASC`, + nodeID, PreinstallPending, PreinstallRunning, staleAfterSeconds(staleAfter), + ).Rows() + if err != nil { + return nil, fmt.Errorf("list node preinstall work: %w", err) + } + defer rows.Close() + return scanPreinstallJobs(rows) +} + +func (s *Store) AckPreinstallJob(ctx context.Context, id, nodeID, status, errMsg string) error { + res := s.db.WithContext(ctx).Exec( + `UPDATE t_component_preinstall_job SET status = ?, error = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND node_id = ?`, + status, nullIfEmpty(errMsg), id, nodeID, + ) + if res.Error != nil { + return fmt.Errorf("ack preinstall job: %w", res.Error) + } + if res.RowsAffected == 0 { + return sql.ErrNoRows + } + return nil +} + +func (s *Store) CancelPendingPreinstallForVersion(ctx context.Context, arch, component, version string) error { + res := s.db.WithContext(ctx).Exec( + `UPDATE t_component_preinstall_job SET status = ?, error = ?, updated_at = CURRENT_TIMESTAMP + WHERE arch = ? AND component = ? AND version = ? AND status IN (?, ?, ?)`, + PreinstallCancelled, "warehouse version deleted", arch, component, version, + PreinstallPending, PreinstallFailed, PreinstallRunning, + ) + if res.Error != nil { + return fmt.Errorf("cancel preinstall jobs: %w", res.Error) + } + return nil +} + +// ReplaceNodeInstalls atomically replaces the inventory snapshot for +// (nodeID, arch). If the incoming set matches what is already stored, the +// transaction commits without DELETE/INSERT. +func (s *Store) ReplaceNodeInstalls(ctx context.Context, nodeID, arch string, items []NodeInstall) error { + items = uniqueNodeInstalls(items) + txErr := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + rows, err := tx.Raw( + `SELECT component, version FROM t_component_node_install + WHERE node_id = ? AND arch = ? FOR UPDATE`, + nodeID, arch, + ).Rows() + if err != nil { + return fmt.Errorf("lock node installs: %w", err) + } + existing := map[string]struct{}{} + for rows.Next() { + var component, version string + if err := rows.Scan(&component, &version); err != nil { + _ = rows.Close() + return err + } + existing[nodeInstallKey(component, version)] = struct{}{} + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return err + } + if err := rows.Close(); err != nil { + return err + } + if sameNodeInstallSet(existing, items) { + return nil + } + if err := tx.Exec( + `DELETE FROM t_component_node_install WHERE node_id = ? AND arch = ?`, + nodeID, arch, + ).Error; err != nil { + return fmt.Errorf("delete node installs: %w", err) + } + for _, it := range items { + if err := tx.Exec( + `INSERT INTO t_component_node_install (node_id, arch, component, version) VALUES (?, ?, ?, ?)`, + nodeID, arch, it.Component, it.Version, + ).Error; err != nil { + return fmt.Errorf("insert node install: %w", err) + } + } + return nil + }) + if txErr != nil { + return fmt.Errorf("replace node installs: %w", txErr) + } + return nil +} + +func (s *Store) ListNodeInstalls(ctx context.Context) ([]NodeInstall, error) { + rows, err := s.db.WithContext(ctx).Raw( + `SELECT node_id, arch, component, version, updated_at FROM t_component_node_install`, + ).Rows() + if err != nil { + return nil, fmt.Errorf("list node installs: %w", err) + } + defer rows.Close() + return scanNodeInstalls(rows) +} + +func scanNodeInstalls(rows *sql.Rows) ([]NodeInstall, error) { + var out []NodeInstall + for rows.Next() { + var item NodeInstall + if err := rows.Scan(&item.NodeID, &item.Arch, &item.Component, &item.Version, &item.UpdatedAt); err != nil { + return nil, err + } + out = append(out, item) + } + return out, rows.Err() +} + +func nodeInstallKey(component, version string) string { + return component + "\x00" + version +} + +func uniqueNodeInstalls(items []NodeInstall) []NodeInstall { + if len(items) == 0 { + return nil + } + seen := make(map[string]struct{}, len(items)) + out := make([]NodeInstall, 0, len(items)) + for _, it := range items { + key := nodeInstallKey(it.Component, it.Version) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, it) + } + return out +} + +func sameNodeInstallSet(existing map[string]struct{}, items []NodeInstall) bool { + if len(existing) != len(items) { + return false + } + for _, it := range items { + if _, ok := existing[nodeInstallKey(it.Component, it.Version)]; !ok { + return false + } + } + return true +} + +func scanImportJobs(rows *sql.Rows) ([]ImportJob, error) { + var out []ImportJob + for rows.Next() { + var job ImportJob + var errMsg sql.NullString + if err := rows.Scan( + &job.ID, &job.Source, &job.SourceRef, &job.Tag, &job.Arch, &job.Status, + &errMsg, &job.BytesTotal, &job.CreatedAt, &job.UpdatedAt, + ); err != nil { + return nil, err + } + job.Error = errMsg.String + out = append(out, job) + } + return out, rows.Err() +} + +func scanPreinstallJobs(rows *sql.Rows) ([]PreinstallJob, error) { + var out []PreinstallJob + for rows.Next() { + var job PreinstallJob + var errMsg sql.NullString + if err := rows.Scan( + &job.ID, &job.NodeID, &job.Arch, &job.Component, &job.Version, &job.Status, + &errMsg, &job.CreatedAt, &job.UpdatedAt, + ); err != nil { + return nil, err + } + job.Error = errMsg.String + out = append(out, job) + } + return out, rows.Err() +} + +func nullIfEmpty(s string) any { + if strings.TrimSpace(s) == "" { + return nil + } + return s +} diff --git a/CubeOps/internal/store/warehouse_test.go b/CubeOps/internal/store/warehouse_test.go new file mode 100644 index 000000000..09861bf47 --- /dev/null +++ b/CubeOps/internal/store/warehouse_test.go @@ -0,0 +1,603 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package store_test + +import ( + "context" + "testing" + "time" + + "github.com/tencentcloud/CubeSandbox/CubeOps/internal/store" +) + +func TestWarehouseItemQueries(t *testing.T) { + withWarehouseStores(t, testWarehouseItemQueries) +} + +func testWarehouseItemQueries(t *testing.T, s *store.Store) { + t.Helper() + ctx := context.Background() + + missing, err := s.GetWarehouseItem(ctx, "amd64", "cube-shim", "v0.6.0") + if err != nil { + t.Fatalf("get missing: %v", err) + } + if missing != nil { + t.Fatalf("get missing: got %+v, want nil", missing) + } + + emptyAll, err := s.ListWarehouseItems(ctx) + if err != nil { + t.Fatalf("empty list all: %v", err) + } + if len(emptyAll) != 0 { + t.Fatalf("empty list all got %d rows", len(emptyAll)) + } + + emptyShim, err := s.ListWarehouseItemsByComponent(ctx, "cube-shim") + if err != nil { + t.Fatalf("empty list shim: %v", err) + } + if len(emptyShim) != 0 { + t.Fatalf("empty warehouse got %d rows", len(emptyShim)) + } + + mustInsert := func(item store.WarehouseItem) { + t.Helper() + ok, err := s.InsertWarehouseItem(ctx, item) + if err != nil { + t.Fatalf("insert %+v: %v", item, err) + } + if !ok { + t.Fatalf("insert %+v: not inserted", item) + } + } + shimAMD := store.WarehouseItem{ + Arch: "amd64", Component: "cube-shim", Version: "v0.6.0", + Source: "github", SourceRef: "TencentCloud/CubeSandbox", + RelPath: "amd64/cube-shim/v0.6.0", SizeBytes: 100, Checksum: "sha256:a", + } + mustInsert(shimAMD) + mustInsert(store.WarehouseItem{ + Arch: "arm64", Component: "cube-shim", Version: "v0.6.0", + Source: "github", SourceRef: "TencentCloud/CubeSandbox", + RelPath: "arm64/cube-shim/v0.6.0", SizeBytes: 80, Checksum: "sha256:b", + }) + mustInsert(store.WarehouseItem{ + Arch: "amd64", Component: "cube-shim", Version: "v0.5.0", + Source: "github", SourceRef: "TencentCloud/CubeSandbox", + RelPath: "amd64/cube-shim/v0.5.0", SizeBytes: 90, Checksum: "sha256:c", + }) + mustInsert(store.WarehouseItem{ + Arch: "amd64", Component: "cube-image", Version: "v0.6.0", + Source: "github", SourceRef: "TencentCloud/CubeSandbox", + RelPath: "amd64/cube-image/v0.6.0", SizeBytes: 1, Checksum: "sha256:d", + }) + + got, err := s.GetWarehouseItem(ctx, "amd64", "cube-shim", "v0.6.0") + if err != nil { + t.Fatalf("get hit: %v", err) + } + if got == nil { + t.Fatal("get hit: nil") + } + if got.Arch != shimAMD.Arch || got.Component != shimAMD.Component || got.Version != shimAMD.Version { + t.Fatalf("get identity=%s/%s/%s want %s/%s/%s", got.Arch, got.Component, got.Version, shimAMD.Arch, shimAMD.Component, shimAMD.Version) + } + if got.Source != shimAMD.Source || got.SourceRef != shimAMD.SourceRef || got.RelPath != shimAMD.RelPath { + t.Fatalf("get refs source=%s sourceRef=%s relPath=%s", got.Source, got.SourceRef, got.RelPath) + } + if got.SizeBytes != shimAMD.SizeBytes || got.Checksum != shimAMD.Checksum { + t.Fatalf("get size/checksum=%d %s", got.SizeBytes, got.Checksum) + } + if got.CreatedAt.IsZero() || got.UpdatedAt.IsZero() { + t.Fatalf("get timestamps created=%v updated=%v", got.CreatedAt, got.UpdatedAt) + } + + stillMissing, err := s.GetWarehouseItem(ctx, "amd64", "cube-shim", "no-such") + if err != nil { + t.Fatalf("get unknown version: %v", err) + } + if stillMissing != nil { + t.Fatalf("get unknown version: got %+v", stillMissing) + } + + all, err := s.ListWarehouseItems(ctx) + if err != nil { + t.Fatalf("list all: %v", err) + } + if len(all) != 4 { + t.Fatalf("list all rows=%d want 4", len(all)) + } + wantOrder := []string{ + "cube-image/v0.6.0/amd64", + "cube-shim/v0.5.0/amd64", + "cube-shim/v0.6.0/amd64", + "cube-shim/v0.6.0/arm64", + } + for i, item := range all { + key := item.Component + "/" + item.Version + "/" + item.Arch + if key != wantOrder[i] { + t.Errorf("list all [%d]=%s want %s", i, key, wantOrder[i]) + } + } + + shim, err := s.ListWarehouseItemsByComponent(ctx, "cube-shim") + if err != nil { + t.Fatalf("list shim: %v", err) + } + if len(shim) != 3 { + t.Fatalf("shim rows=%d want 3", len(shim)) + } + for _, item := range shim { + if item.Component != "cube-shim" { + t.Errorf("unexpected component %s", item.Component) + } + } + wantShim := []string{"v0.5.0/amd64", "v0.6.0/amd64", "v0.6.0/arm64"} + for i, item := range shim { + key := item.Version + "/" + item.Arch + if key != wantShim[i] { + t.Errorf("list shim [%d]=%s want %s", i, key, wantShim[i]) + } + } + + image, err := s.ListWarehouseItemsByComponent(ctx, "cube-image") + if err != nil { + t.Fatalf("list image: %v", err) + } + if len(image) != 1 || image[0].Version != "v0.6.0" { + t.Fatalf("image=%+v", image) + } + + unknown, err := s.ListWarehouseItemsByComponent(ctx, "cube-kernel-scf") + if err != nil { + t.Fatalf("list unknown component: %v", err) + } + if len(unknown) != 0 { + t.Fatalf("unknown component rows=%d", len(unknown)) + } +} + +func TestListImportJobs(t *testing.T) { + withWarehouseStores(t, testListImportJobs) +} + +func testListImportJobs(t *testing.T, s *store.Store) { + t.Helper() + ctx := context.Background() + + empty, total, err := s.ListImportJobs(ctx, 50, 0) + if err != nil { + t.Fatalf("empty list: %v", err) + } + if len(empty) != 0 || total != 0 { + t.Fatalf("empty got %d total=%d", len(empty), total) + } + + for _, job := range []store.ImportJob{ + {ID: "imp-1", Source: "github", SourceRef: "TencentCloud/CubeSandbox", Tag: "v0.5.0", Arch: "amd64", Status: store.ImportSucceeded}, + {ID: "imp-2", Source: "cnb", SourceRef: "CubeSandbox/CubeSandbox", Tag: "v0.6.0", Arch: "arm64", Status: store.ImportRunning}, + {ID: "imp-3", Source: "upload", SourceRef: "/tmp/x.tar.gz", Tag: "v0.6.1", Arch: "amd64", Status: store.ImportPending}, + } { + if err := s.CreateImportJob(ctx, job); err != nil { + t.Fatalf("create %s: %v", job.ID, err) + } + } + + all, total, err := s.ListImportJobs(ctx, 50, 0) + if err != nil { + t.Fatalf("list all: %v", err) + } + if total != 3 || len(all) != 3 { + t.Fatalf("all len=%d total=%d want 3", len(all), total) + } + + page0, total, err := s.ListImportJobs(ctx, 1, 0) + if err != nil || total != 3 || len(page0) != 1 { + t.Fatalf("page0: jobs=%d total=%d err=%v", len(page0), total, err) + } + page1, _, err := s.ListImportJobs(ctx, 1, 1) + if err != nil || len(page1) != 1 { + t.Fatalf("page1: jobs=%d err=%v", len(page1), err) + } + if page0[0].ID == page1[0].ID { + t.Fatalf("pages overlap id=%s", page0[0].ID) + } + + capped, total, err := s.ListImportJobs(ctx, 0, 0) + if err != nil || total != 3 || len(capped) != 3 { + t.Fatalf("default limit: jobs=%d total=%d err=%v", len(capped), total, err) + } +} + +func TestListPreinstallJobs(t *testing.T) { + withWarehouseStores(t, testListPreinstallJobs) +} + +func testListPreinstallJobs(t *testing.T, s *store.Store) { + t.Helper() + ctx := context.Background() + mustCreate := func(id, nodeID, status string) { + t.Helper() + if err := s.CreatePreinstallJobs(ctx, []store.PreinstallJob{{ + ID: id, NodeID: nodeID, Arch: "amd64", Component: "cube-shim", Version: "v1", Status: status, + }}); err != nil { + t.Fatalf("create %s: %v", id, err) + } + } + mustCreate("pre-1", "node-a", store.PreinstallSucceeded) + mustCreate("pre-2", "node-a", store.PreinstallRunning) + mustCreate("pre-3", "node-b", store.PreinstallPending) + + all, total, err := s.ListPreinstallJobs(ctx, "", "", 50, 0) + if err != nil || total != 3 || len(all) != 3 { + t.Fatalf("all: jobs=%d total=%d err=%v", len(all), total, err) + } + page0, total, err := s.ListPreinstallJobs(ctx, "", "", 1, 0) + if err != nil || total != 3 || len(page0) != 1 { + t.Fatalf("page0: jobs=%d total=%d err=%v", len(page0), total, err) + } + page1, _, err := s.ListPreinstallJobs(ctx, "", "", 1, 1) + if err != nil || len(page1) != 1 || page0[0].ID == page1[0].ID { + t.Fatalf("page1 overlap or err: %v %v %v", page0, page1, err) + } + filtered, total, err := s.ListPreinstallJobs(ctx, "node-a", "", 50, 0) + if err != nil || total != 2 || len(filtered) != 2 { + t.Fatalf("node-a: jobs=%d total=%d err=%v", len(filtered), total, err) + } + running, total, err := s.ListPreinstallJobs(ctx, "", store.PreinstallRunning, 50, 0) + if err != nil || total != 1 || len(running) != 1 || running[0].ID != "pre-2" { + t.Fatalf("running: %+v total=%d err=%v", running, total, err) + } +} + +func TestCreatePreinstallJobsRollsBackOnConflict(t *testing.T) { + withWarehouseStores(t, testCreatePreinstallJobsRollsBackOnConflict) +} + +func testCreatePreinstallJobsRollsBackOnConflict(t *testing.T, s *store.Store) { + t.Helper() + ctx := context.Background() + err := s.CreatePreinstallJobs(ctx, []store.PreinstallJob{ + {ID: "pre-dup", NodeID: "node-a", Arch: "amd64", Component: "cube-shim", Version: "v1", Status: store.PreinstallPending}, + {ID: "pre-dup", NodeID: "node-b", Arch: "amd64", Component: "cube-shim", Version: "v1", Status: store.PreinstallPending}, + }) + if err == nil { + t.Fatal("expected duplicate-id error") + } + jobs, total, listErr := s.ListPreinstallJobs(ctx, "", "", 50, 0) + if listErr != nil || total != 0 || len(jobs) != 0 { + t.Fatalf("partial insert leaked: jobs=%d total=%d err=%v", len(jobs), total, listErr) + } +} + +func TestClaimImportJob(t *testing.T) { + withWarehouseStores(t, testClaimImportJob) +} + +func testClaimImportJob(t *testing.T, s *store.Store) { + t.Helper() + ctx := context.Background() + stale := 30 * time.Minute + + mustCreate := func(id, status string) { + t.Helper() + err := s.CreateImportJob(ctx, store.ImportJob{ + ID: id, Source: "upload", SourceRef: "/tmp/x.tar.gz", + Tag: "v0.6.0", Arch: "amd64", Status: status, + }) + if err != nil { + t.Fatalf("create %s: %v", id, err) + } + } + + mustCreate("claim-pending", store.ImportPending) + work, err := s.ListImportWork(ctx, stale) + if err != nil { + t.Fatalf("list work: %v", err) + } + if len(work) != 1 || work[0].ID != "claim-pending" { + t.Fatalf("list pending work=%v", work) + } + + ok, err := s.ClaimImportJob(ctx, "claim-pending", stale) + if err != nil || !ok { + t.Fatalf("claim pending: ok=%v err=%v", ok, err) + } + got, err := s.GetImportJob(ctx, "claim-pending") + if err != nil { + t.Fatalf("get after claim: %v", err) + } + if got == nil || got.Status != store.ImportRunning { + t.Fatalf("after claim status=%v", got) + } + + ok, err = s.ClaimImportJob(ctx, "claim-pending", stale) + if err != nil { + t.Fatalf("second claim: %v", err) + } + if ok { + t.Fatal("second claim of running job succeeded") + } + + mustCreate("claim-fresh-running", store.ImportRunning) + ok, err = s.ClaimImportJob(ctx, "claim-fresh-running", stale) + if err != nil { + t.Fatalf("fresh running claim: %v", err) + } + if ok { + t.Fatal("fresh running job was reclaimed") + } + + mustCreate("claim-stale-running", store.ImportRunning) + past := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + if err := s.DB().WithContext(ctx).Exec( + `UPDATE t_component_import_job SET updated_at = ? WHERE id = ?`, + past, "claim-stale-running", + ).Error; err != nil { + t.Fatalf("backdate stale running: %v", err) + } + work, err = s.ListImportWork(ctx, stale) + if err != nil { + t.Fatalf("list stale work: %v", err) + } + foundStale := false + for _, job := range work { + if job.ID == "claim-stale-running" { + foundStale = true + } + if job.ID == "claim-fresh-running" || job.ID == "claim-pending" { + t.Fatalf("list work included non-stale %s", job.ID) + } + } + if !foundStale { + t.Fatal("list work missing stale running job") + } + + ok, err = s.ClaimImportJob(ctx, "claim-stale-running", stale) + if err != nil || !ok { + t.Fatalf("reclaim stale running: ok=%v err=%v", ok, err) + } +} + +func TestReplaceNodeInstalls(t *testing.T) { + withWarehouseStores(t, testReplaceNodeInstalls) +} + +func testReplaceNodeInstalls(t *testing.T, s *store.Store) { + t.Helper() + ctx := context.Background() + node := "10.10.0.8" + other := "10.10.0.9" + arch := "amd64" + + mustReplace := func(nodeID string, items []store.NodeInstall) { + t.Helper() + if err := s.ReplaceNodeInstalls(ctx, nodeID, arch, items); err != nil { + t.Fatalf("replace %s: %v", nodeID, err) + } + } + keysOf := func(nodeID string) map[string]int64 { + t.Helper() + rows, err := s.DB().WithContext(ctx).Raw( + `SELECT id, component, version FROM t_component_node_install WHERE node_id = ? AND arch = ?`, + nodeID, arch, + ).Rows() + if err != nil { + t.Fatalf("select ids: %v", err) + } + defer rows.Close() + out := map[string]int64{} + for rows.Next() { + var id int64 + var component, version string + if err := rows.Scan(&id, &component, &version); err != nil { + t.Fatalf("scan id: %v", err) + } + out[component+"/"+version] = id + } + if err := rows.Err(); err != nil { + t.Fatalf("ids rows: %v", err) + } + return out + } + + mustReplace(node, []store.NodeInstall{ + {Component: "cube-shim", Version: "vA"}, + {Component: "cube-agent", Version: "vB"}, + }) + mustReplace(other, []store.NodeInstall{ + {Component: "cube-shim", Version: "vKeep"}, + }) + + got, err := s.ListNodeInstalls(ctx) + if err != nil { + t.Fatalf("list: %v", err) + } + if !hasInstall(got, node, "cube-shim", "vA") || !hasInstall(got, node, "cube-agent", "vB") { + t.Fatalf("missing A/B after first replace: %+v", got) + } + + before := keysOf(node) + mustReplace(node, []store.NodeInstall{ + {Component: "cube-agent", Version: "vB"}, + {Component: "cube-shim", Version: "vA"}, + }) + after := keysOf(node) + if len(after) != 2 { + t.Fatalf("same-set rows=%d want 2", len(after)) + } + for k, id := range before { + if after[k] != id { + t.Fatalf("same-set rewrote %s id %d -> %d", k, id, after[k]) + } + } + + mustReplace(node, []store.NodeInstall{ + {Component: "cube-shim", Version: "vA"}, + }) + got, err = s.ListNodeInstalls(ctx) + if err != nil { + t.Fatalf("list after shrink: %v", err) + } + if !hasInstall(got, node, "cube-shim", "vA") { + t.Fatal("missing A after shrink") + } + if hasInstall(got, node, "cube-agent", "vB") { + t.Fatal("B still present after shrink") + } + if !hasInstall(got, other, "cube-shim", "vKeep") { + t.Fatal("other node row was cleared") + } + + mustReplace(node, nil) + got, err = s.ListNodeInstalls(ctx) + if err != nil { + t.Fatalf("list after empty: %v", err) + } + for _, inst := range got { + if inst.NodeID == node { + t.Fatalf("empty replace left %+v", inst) + } + } + if !hasInstall(got, other, "cube-shim", "vKeep") { + t.Fatal("other node row missing after empty replace") + } +} + +func hasInstall(items []store.NodeInstall, nodeID, component, version string) bool { + for _, inst := range items { + if inst.NodeID == nodeID && inst.Component == component && inst.Version == version { + return true + } + } + return false +} + +func TestListNodePreinstallWorkStaleRunning(t *testing.T) { + withWarehouseStores(t, testListNodePreinstallWorkStaleRunning) +} + +func testListNodePreinstallWorkStaleRunning(t *testing.T, s *store.Store) { + t.Helper() + ctx := context.Background() + stale := 15 * time.Minute + mustCreate := func(id, nodeID, status string) { + t.Helper() + if err := s.CreatePreinstallJobs(ctx, []store.PreinstallJob{{ + ID: id, NodeID: nodeID, Arch: "amd64", Component: "cube-shim", Version: "v1", Status: status, + }}); err != nil { + t.Fatalf("create %s: %v", id, err) + } + } + mustCreate("pre-pending", "node-a", store.PreinstallPending) + mustCreate("pre-fresh", "node-a", store.PreinstallRunning) + mustCreate("pre-stale", "node-a", store.PreinstallRunning) + mustCreate("pre-other", "node-b", store.PreinstallPending) + + past := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + if err := s.DB().WithContext(ctx).Exec( + `UPDATE t_component_preinstall_job SET updated_at = ? WHERE id = ?`, + past, "pre-stale", + ).Error; err != nil { + t.Fatalf("backdate: %v", err) + } + + work, err := s.ListNodePreinstallWork(ctx, "node-a", stale) + if err != nil { + t.Fatalf("list: %v", err) + } + found := map[string]bool{} + for _, job := range work { + found[job.ID] = true + } + if !found["pre-pending"] || !found["pre-stale"] { + t.Fatalf("work=%v want pending+stale", found) + } + if found["pre-fresh"] || found["pre-other"] { + t.Fatalf("work included non-stale or other node: %v", found) + } +} + +func TestCancelPendingPreinstallForVersionIncludesRunning(t *testing.T) { + withWarehouseStores(t, testCancelPendingPreinstallForVersionIncludesRunning) +} + +func testCancelPendingPreinstallForVersionIncludesRunning(t *testing.T, s *store.Store) { + t.Helper() + ctx := context.Background() + mustCreate := func(id, status, version string) { + t.Helper() + if err := s.CreatePreinstallJobs(ctx, []store.PreinstallJob{{ + ID: id, NodeID: "node-a", Arch: "amd64", Component: "cube-shim", Version: version, Status: status, + }}); err != nil { + t.Fatalf("create %s: %v", id, err) + } + } + mustCreate("c-pending", store.PreinstallPending, "v1") + mustCreate("c-running", store.PreinstallRunning, "v1") + mustCreate("c-failed", store.PreinstallFailed, "v1") + mustCreate("c-ok", store.PreinstallSucceeded, "v1") + mustCreate("c-other", store.PreinstallRunning, "v2") + + if err := s.CancelPendingPreinstallForVersion(ctx, "amd64", "cube-shim", "v1"); err != nil { + t.Fatalf("cancel: %v", err) + } + jobs, _, err := s.ListPreinstallJobs(ctx, "", "", 50, 0) + if err != nil { + t.Fatalf("list: %v", err) + } + statusOf := map[string]string{} + for _, job := range jobs { + statusOf[job.ID] = job.Status + } + for _, id := range []string{"c-pending", "c-running", "c-failed"} { + if statusOf[id] != store.PreinstallCancelled { + t.Errorf("%s status=%s want cancelled", id, statusOf[id]) + } + } + if statusOf["c-ok"] != store.PreinstallSucceeded { + t.Errorf("succeeded job was cancelled") + } + if statusOf["c-other"] != store.PreinstallRunning { + t.Errorf("other version running job status=%s", statusOf["c-other"]) + } +} + +func TestCountLiveImportJobsBySourceRef(t *testing.T) { + withWarehouseStores(t, testCountLiveImportJobsBySourceRef) +} + +func testCountLiveImportJobsBySourceRef(t *testing.T, s *store.Store) { + t.Helper() + ctx := context.Background() + ref := "/data/cubeops/warehouse/_uploads/a.tar.gz" + mustCreate := func(id, status string) { + t.Helper() + if err := s.CreateImportJob(ctx, store.ImportJob{ + ID: id, Source: "upload", SourceRef: ref, Tag: "v0.6.0", Arch: "amd64", Status: status, + }); err != nil { + t.Fatalf("create %s: %v", id, err) + } + } + mustCreate("live-pending", store.ImportPending) + mustCreate("live-running", store.ImportRunning) + mustCreate("done", store.ImportSucceeded) + if err := s.CreateImportJob(ctx, store.ImportJob{ + ID: "other-ref", Source: "upload", SourceRef: "/tmp/other.tar.gz", + Tag: "v0.6.0", Arch: "arm64", Status: store.ImportPending, + }); err != nil { + t.Fatalf("create other: %v", err) + } + + n, err := s.CountLiveImportJobsBySourceRef(ctx, ref) + if err != nil || n != 2 { + t.Fatalf("live=%d err=%v want 2", n, err) + } + n, err = s.CountLiveImportJobsBySourceRef(ctx, "/tmp/missing.tar.gz") + if err != nil || n != 0 { + t.Fatalf("missing=%d err=%v want 0", n, err) + } +} diff --git a/CubeOps/internal/warehouse/catalog.go b/CubeOps/internal/warehouse/catalog.go new file mode 100644 index 000000000..54b526568 --- /dev/null +++ b/CubeOps/internal/warehouse/catalog.go @@ -0,0 +1,181 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package warehouse + +import ( + "sort" + "time" + + "github.com/tencentcloud/CubeSandbox/CubeOps/internal/store" +) + +// ComponentSummary is the catalog row for GET /warehouse/components. +type ComponentSummary struct { + Name string `json:"name"` + VersionCount int `json:"versionCount"` + Arches []string `json:"arches"` + SizeBytes int64 `json:"sizeBytes"` + NodesMissing *int `json:"nodesMissing,omitempty"` +} + +// ArtifactView is one arch copy inside a version group. +type ArtifactView struct { + Arch string `json:"arch"` + SizeBytes int64 `json:"sizeBytes"` + Source string `json:"source"` + SourceRef string `json:"sourceRef"` + Checksum string `json:"checksum"` + CreatedAt time.Time `json:"createdAt"` + NodesInstalled []string `json:"nodesInstalled,omitempty"` + NodesMissing []string `json:"nodesMissing,omitempty"` +} + +// VersionGroup is one version and its per-arch artifacts. +type VersionGroup struct { + Version string `json:"version"` + Artifacts []ArtifactView `json:"artifacts"` +} + +// ComponentDetail is GET /warehouse/components/:component. +type ComponentDetail struct { + Name string `json:"name"` + Versions []VersionGroup `json:"versions"` +} + +// SummarizeComponents returns one row per catalog component. Empty warehouse +// still lists all four names. When coverageOK is false, NodesMissing is omitted. +func SummarizeComponents(items []store.WarehouseItem, installs []store.NodeInstall, nodeIDs []string, coverageOK bool) []ComponentSummary { + byComp := map[string][]store.WarehouseItem{} + for _, item := range items { + byComp[item.Component] = append(byComp[item.Component], item) + } + installed := installIndex(installs) + out := make([]ComponentSummary, 0, len(Catalog())) + for _, info := range Catalog() { + group := byComp[info.Name] + sum := ComponentSummary{ + Name: info.Name, + Arches: []string{}, + } + versions := map[string]struct{}{} + arches := map[string]struct{}{} + for _, item := range group { + versions[item.Version] = struct{}{} + arches[item.Arch] = struct{}{} + sum.SizeBytes += item.SizeBytes + } + sum.VersionCount = len(versions) + sum.Arches = sortedKeys(arches) + if coverageOK { + n := nodesMissingAny(group, installed, nodeIDs) + sum.NodesMissing = &n + } + out = append(out, sum) + } + return out +} + +// GroupComponent versions a single catalog component. Unknown names should be +// rejected before calling. When coverageOK is false, node lists are omitted. +func GroupComponent(name string, items []store.WarehouseItem, installs []store.NodeInstall, nodeIDs []string, coverageOK bool) ComponentDetail { + installed := installIndex(installs) + var order []string + seen := map[string][]store.WarehouseItem{} + for _, item := range items { + if item.Component != name { + continue + } + if _, ok := seen[item.Version]; !ok { + order = append(order, item.Version) + } + seen[item.Version] = append(seen[item.Version], item) + } + groups := make([]VersionGroup, 0, len(order)) + for _, ver := range order { + arts := seen[ver] + sort.Slice(arts, func(i, j int) bool { return arts[i].Arch < arts[j].Arch }) + views := make([]ArtifactView, 0, len(arts)) + for _, item := range arts { + view := ArtifactView{ + Arch: item.Arch, + SizeBytes: item.SizeBytes, + Source: item.Source, + SourceRef: item.SourceRef, + Checksum: item.Checksum, + CreatedAt: item.CreatedAt, + } + if coverageOK { + view.NodesInstalled, view.NodesMissing = splitCoverage(item, installed, nodeIDs) + } + views = append(views, view) + } + groups = append(groups, VersionGroup{Version: ver, Artifacts: views}) + } + return ComponentDetail{Name: name, Versions: groups} +} + +type coverageKey struct { + arch, component, version string +} + +func itemKey(item store.WarehouseItem) coverageKey { + return coverageKey{item.Arch, item.Component, item.Version} +} + +func installIndex(installs []store.NodeInstall) map[coverageKey]map[string]struct{} { + out := map[coverageKey]map[string]struct{}{} + for _, inst := range installs { + key := coverageKey{inst.Arch, inst.Component, inst.Version} + if out[key] == nil { + out[key] = map[string]struct{}{} + } + out[key][inst.NodeID] = struct{}{} + } + return out +} + +func splitCoverage(item store.WarehouseItem, installed map[coverageKey]map[string]struct{}, nodeIDs []string) (present, missing []string) { + have := installed[itemKey(item)] + present, missing = []string{}, []string{} + seen := map[string]struct{}{} + for _, n := range nodeIDs { + seen[n] = struct{}{} + if _, ok := have[n]; ok { + present = append(present, n) + } else { + missing = append(missing, n) + } + } + for n := range have { + if _, ok := seen[n]; !ok { + present = append(present, n) + } + } + return present, missing +} + +func nodesMissingAny(items []store.WarehouseItem, installed map[coverageKey]map[string]struct{}, nodeIDs []string) int { + if len(items) == 0 { + return 0 + } + count := 0 + for _, n := range nodeIDs { + for _, item := range items { + if _, ok := installed[itemKey(item)][n]; !ok { + count++ + break + } + } + } + return count +} + +func sortedKeys(set map[string]struct{}) []string { + out := make([]string, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/CubeOps/internal/warehouse/catalog_test.go b/CubeOps/internal/warehouse/catalog_test.go new file mode 100644 index 000000000..63ecfe3e2 --- /dev/null +++ b/CubeOps/internal/warehouse/catalog_test.go @@ -0,0 +1,122 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package warehouse + +import ( + "reflect" + "testing" + "time" + + "github.com/tencentcloud/CubeSandbox/CubeOps/internal/store" +) + +func TestCatalog(t *testing.T) { + got := Catalog() + if len(got) != 4 { + t.Fatalf("len=%d want 4", len(got)) + } + want := []string{ComponentShim, ComponentImage, ComponentAgent, ComponentKernel} + for i, name := range want { + if got[i].Name != name { + t.Errorf("Catalog[%d]=%s want %s", i, got[i].Name, name) + } + if !KnownComponent(name) { + t.Errorf("KnownComponent(%s)=false", name) + } + } + if KnownComponent("cubelet") { + t.Fatal("cubelet should not be in warehouse catalog") + } +} + +func TestSummarizeComponents_EmptyWarehouse(t *testing.T) { + out := SummarizeComponents(nil, nil, []string{"n1"}, true) + if len(out) != 4 { + t.Fatalf("len=%d want 4", len(out)) + } + for _, row := range out { + if row.VersionCount != 0 || row.SizeBytes != 0 || len(row.Arches) != 0 { + t.Errorf("%s: %+v", row.Name, row) + } + if row.NodesMissing == nil || *row.NodesMissing != 0 { + t.Errorf("%s nodesMissing=%v want 0", row.Name, row.NodesMissing) + } + } +} + +func TestSummarizeComponents_OmitsCoverageWhenUnavailable(t *testing.T) { + items := []store.WarehouseItem{{Arch: ArchAMD64, Component: ComponentShim, Version: "v0.6.0", SizeBytes: 10}} + out := SummarizeComponents(items, nil, []string{"n1"}, false) + shim := out[0] + if shim.Name != ComponentShim || shim.VersionCount != 1 || shim.SizeBytes != 10 { + t.Fatalf("shim=%+v", shim) + } + if shim.NodesMissing != nil { + t.Fatalf("nodesMissing should be omitted, got %v", *shim.NodesMissing) + } +} + +func TestSummarizeComponents_CountsArchVersionAndMissing(t *testing.T) { + items := []store.WarehouseItem{ + {Arch: ArchAMD64, Component: ComponentShim, Version: "v0.6.0", SizeBytes: 100}, + {Arch: ArchARM64, Component: ComponentShim, Version: "v0.6.0", SizeBytes: 80}, + {Arch: ArchAMD64, Component: ComponentShim, Version: "v0.5.0", SizeBytes: 90}, + } + installs := []store.NodeInstall{ + {NodeID: "n1", Arch: ArchAMD64, Component: ComponentShim, Version: "v0.6.0"}, + {NodeID: "n1", Arch: ArchARM64, Component: ComponentShim, Version: "v0.6.0"}, + {NodeID: "n1", Arch: ArchAMD64, Component: ComponentShim, Version: "v0.5.0"}, + } + out := SummarizeComponents(items, installs, []string{"n1", "n2"}, true) + shim := out[0] + if shim.VersionCount != 2 { + t.Errorf("versionCount=%d want 2", shim.VersionCount) + } + if shim.SizeBytes != 270 { + t.Errorf("sizeBytes=%d want 270", shim.SizeBytes) + } + if !reflect.DeepEqual(shim.Arches, []string{ArchAMD64, ArchARM64}) { + t.Errorf("arches=%v", shim.Arches) + } + if shim.NodesMissing == nil || *shim.NodesMissing != 1 { + t.Errorf("nodesMissing=%v want 1", shim.NodesMissing) + } +} + +func TestGroupComponent(t *testing.T) { + ts := time.Date(2026, 8, 13, 0, 0, 0, 0, time.UTC) + items := []store.WarehouseItem{ + {Arch: ArchARM64, Component: ComponentShim, Version: "v0.6.0", SizeBytes: 80, Source: "github", CreatedAt: ts}, + {Arch: ArchAMD64, Component: ComponentShim, Version: "v0.6.0", SizeBytes: 100, Source: "github", CreatedAt: ts}, + {Arch: ArchAMD64, Component: ComponentShim, Version: "v0.5.0", SizeBytes: 90, Source: "github", CreatedAt: ts}, + {Arch: ArchAMD64, Component: ComponentImage, Version: "v0.6.0", SizeBytes: 1, Source: "github"}, + } + installs := []store.NodeInstall{ + {NodeID: "n1", Arch: ArchAMD64, Component: ComponentShim, Version: "v0.6.0"}, + } + got := GroupComponent(ComponentShim, items, installs, []string{"n1", "n2"}, true) + if got.Name != ComponentShim { + t.Fatalf("name=%s", got.Name) + } + if len(got.Versions) != 2 { + t.Fatalf("versions=%d want 2: %#v", len(got.Versions), got.Versions) + } + if got.Versions[0].Version != "v0.6.0" || len(got.Versions[0].Artifacts) != 2 { + t.Fatalf("first group=%+v", got.Versions[0]) + } + if got.Versions[0].Artifacts[0].Arch != ArchAMD64 { + t.Fatalf("arch order=%s want amd64 first", got.Versions[0].Artifacts[0].Arch) + } + amd := got.Versions[0].Artifacts[0] + if !reflect.DeepEqual(amd.NodesInstalled, []string{"n1"}) { + t.Errorf("installed=%v", amd.NodesInstalled) + } + if !reflect.DeepEqual(amd.NodesMissing, []string{"n2"}) { + t.Errorf("missing=%v", amd.NodesMissing) + } + empty := GroupComponent(ComponentAgent, items, nil, []string{"n1"}, true) + if empty.Name != ComponentAgent || len(empty.Versions) != 0 { + t.Fatalf("empty agent=%+v", empty) + } +} diff --git a/CubeOps/internal/warehouse/disk.go b/CubeOps/internal/warehouse/disk.go new file mode 100644 index 000000000..5c6bc2255 --- /dev/null +++ b/CubeOps/internal/warehouse/disk.go @@ -0,0 +1,463 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package warehouse + +import ( + "archive/tar" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "syscall" +) + +const ( + DefaultDir = "/data/cubeops/warehouse" + uploadsSubdir = "_uploads" + dirPerm = 0o755 + filePerm = 0o644 + maxTarEntryBytes = 8 << 30 // 8 GiB per file inside a streamed blob + maxExtractedBytes = 16 << 30 +) + +// Layout is the on-disk warehouse tree. +type Layout struct { + Root string +} + +func NewLayout(root string) *Layout { + if strings.TrimSpace(root) == "" { + root = DefaultDir + } + return &Layout{Root: root} +} + +func (l *Layout) Abs(arch, component, version string) string { + return filepath.Join(l.Root, arch, component, version) +} + +func (l *Layout) UploadsDir() string { + return filepath.Join(l.Root, uploadsSubdir) +} + +func (l *Layout) EnsureRoot() error { + if err := os.MkdirAll(l.Root, dirPerm); err != nil { + return err + } + return os.MkdirAll(l.UploadsDir(), dirPerm) +} + +// renameDir is os.Rename; tests replace it to simulate NFS EXDEV. +var renameDir = os.Rename + +func (l *Layout) InstallDir(arch, component, version, srcDir string) (exists bool, err error) { + dst := l.Abs(arch, component, version) + if err := os.MkdirAll(filepath.Dir(dst), dirPerm); err != nil { + return false, err + } + if st, err := os.Stat(dst); err == nil && st.IsDir() { + _ = os.RemoveAll(srcDir) + return true, nil + } + if err := renameDir(srcDir, dst); err != nil { + if isCrossDevice(err) { + return l.installDirCopy(srcDir, dst) + } + _ = os.RemoveAll(srcDir) + if st, statErr := os.Stat(dst); statErr == nil && st.IsDir() { + return true, nil + } + return false, fmt.Errorf("rename into warehouse: %w", err) + } + return false, nil +} + +func (l *Layout) installDirCopy(srcDir, dst string) (bool, error) { + tmp := dst + ".install-tmp" + _ = os.RemoveAll(tmp) + if err := copyTree(srcDir, tmp); err != nil { + _ = os.RemoveAll(tmp) + _ = os.RemoveAll(srcDir) + return false, fmt.Errorf("copy into warehouse: %w", err) + } + if err := os.Rename(tmp, dst); err != nil { + _ = os.RemoveAll(tmp) + _ = os.RemoveAll(srcDir) + if st, statErr := os.Stat(dst); statErr == nil && st.IsDir() { + return true, nil + } + return false, fmt.Errorf("rename copy into warehouse: %w", err) + } + _ = os.RemoveAll(srcDir) + return false, nil +} + +func isCrossDevice(err error) bool { + if err == nil { + return false + } + if errors.Is(err, syscall.EXDEV) { + return true + } + var linkErr *os.LinkError + if errors.As(err, &linkErr) { + return errors.Is(linkErr.Err, syscall.EXDEV) + } + var pathErr *os.PathError + if errors.As(err, &pathErr) { + return errors.Is(pathErr.Err, syscall.EXDEV) + } + return false +} + +// RemoveDir deletes a version directory. Missing is not an error. +func (l *Layout) RemoveDir(arch, component, version string) error { + return os.RemoveAll(l.Abs(arch, component, version)) +} + +// DirSizeAndChecksum walks a version directory: size is the sum of regular +// files; checksum is sha256 of the primary artifact (or empty). +func DirSizeAndChecksum(dir, component string) (size int64, checksum string, err error) { + err = filepath.WalkDir(dir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + if info.Mode().IsRegular() { + size += info.Size() + } + return nil + }) + if err != nil { + return 0, "", err + } + primary := primaryArtifact(dir, component) + if primary == "" { + return size, "", nil + } + sum, err := fileSHA256(primary) + if err != nil { + return size, "", err + } + return size, "sha256:" + sum, nil +} + +func primaryArtifact(dir, component string) string { + switch component { + case ComponentShim: + p := filepath.Join(dir, "bin", "containerd-shim-cube-rs") + if fileExists(p) { + return p + } + case ComponentImage: + p := filepath.Join(dir, "cube-guest-image-cpu.img") + if fileExists(p) { + return p + } + case ComponentAgent: + p := filepath.Join(dir, "cube-agent.ext4") + if fileExists(p) { + return p + } + case ComponentKernel: + for _, name := range []string{"vmlinux", "vmlinux-bm", "vmlinux-pvm"} { + p := filepath.Join(dir, name) + if fileExists(p) { + return p + } + } + } + return "" +} + +func fileExists(p string) bool { + st, err := os.Stat(p) + return err == nil && st.Mode().IsRegular() +} + +func fileSHA256(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// ValidateInstalledTree checks the required files for a component version. +func ValidateInstalledTree(dir, component string) error { + switch component { + case ComponentShim: + for _, rel := range []string{ + filepath.Join("bin", "containerd-shim-cube-rs"), + filepath.Join("bin", "cube-runtime"), + } { + if !fileExists(filepath.Join(dir, rel)) { + return fmt.Errorf("missing %s", rel) + } + } + case ComponentImage: + if !fileExists(filepath.Join(dir, "cube-guest-image-cpu.img")) { + return fmt.Errorf("missing cube-guest-image-cpu.img") + } + case ComponentAgent: + if !fileExists(filepath.Join(dir, "cube-agent.ext4")) { + return fmt.Errorf("missing cube-agent.ext4") + } + case ComponentKernel: + if !fileExists(filepath.Join(dir, "vmlinux")) && + !fileExists(filepath.Join(dir, "vmlinux-bm")) && + !fileExists(filepath.Join(dir, "vmlinux-pvm")) { + return fmt.Errorf("missing vmlinux") + } + default: + return fmt.Errorf("unsupported component %s", component) + } + return nil +} + +// WriteTarGz streams dir as a gzip-compressed tar to w. Paths are relative +// to dir; symlinks are stored as links (not followed). +func WriteTarGz(w io.Writer, dir string) error { + gz, err := gzip.NewWriterLevel(w, gzip.BestSpeed) + if err != nil { + return err + } + defer gz.Close() + tw := tar.NewWriter(gz) + defer tw.Close() + + return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + if rel == "." { + return nil + } + rel = filepath.ToSlash(rel) + if strings.HasPrefix(rel, "../") || strings.Contains(rel, "/../") { + return fmt.Errorf("refusing to archive path %q", rel) + } + + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + header.Name = rel + if info.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(path) + if err != nil { + return err + } + header.Linkname = target + header.Typeflag = tar.TypeSymlink + } + if err := tw.WriteHeader(header); err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + if info.Size() > maxTarEntryBytes { + return fmt.Errorf("file %s too large", rel) + } + f, err := os.Open(path) + if err != nil { + return err + } + _, copyErr := io.Copy(tw, f) + closeErr := f.Close() + if copyErr != nil { + return copyErr + } + return closeErr + }) +} + +// ExtractTarGz unpacks a tar.gz into destDir with path-traversal guards. +func ExtractTarGz(r io.Reader, destDir string) error { + gz, err := gzip.NewReader(r) + if err != nil { + return fmt.Errorf("gzip: %w", err) + } + defer gz.Close() + return extractTar(gz, destDir) +} + +func extractTar(r io.Reader, destDir string) error { + if err := os.MkdirAll(destDir, dirPerm); err != nil { + return err + } + tr := tar.NewReader(r) + var written int64 + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + target, err := safeJoin(destDir, hdr.Name) + if err != nil { + return err + } + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, dirPerm); err != nil { + return err + } + case tar.TypeReg, tar.TypeRegA: + if hdr.Size < 0 || hdr.Size > maxTarEntryBytes { + return fmt.Errorf("tar entry %s too large", hdr.Name) + } + written += hdr.Size + if written > maxExtractedBytes { + return fmt.Errorf("extracted archive exceeds size limit") + } + if err := os.MkdirAll(filepath.Dir(target), dirPerm); err != nil { + return err + } + f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)&0o777) + if err != nil { + return err + } + n, copyErr := io.CopyN(f, tr, hdr.Size) + _ = f.Close() + if copyErr != nil { + return copyErr + } + if n != hdr.Size { + return fmt.Errorf("short write for %s", hdr.Name) + } + case tar.TypeSymlink: + if err := os.MkdirAll(filepath.Dir(target), dirPerm); err != nil { + return err + } + if filepath.IsAbs(hdr.Linkname) || strings.Contains(hdr.Linkname, "..") { + return fmt.Errorf("refusing symlink %s -> %s", hdr.Name, hdr.Linkname) + } + _ = os.Remove(target) + if err := os.Symlink(hdr.Linkname, target); err != nil { + return err + } + default: + // skip other types (hard links, devices) + } + } +} + +func safeJoin(root, name string) (string, error) { + name = strings.TrimPrefix(filepath.ToSlash(name), "/") + if name == "" || name == "." { + return root, nil + } + if strings.Contains(name, "..") { + return "", fmt.Errorf("refusing path %q", name) + } + cleaned := filepath.Join(root, filepath.FromSlash(name)) + rel, err := filepath.Rel(root, cleaned) + if err != nil || strings.HasPrefix(rel, "..") { + return "", fmt.Errorf("refusing path %q", name) + } + return cleaned, nil +} + +func copyRegularFile(src, dst string) error { + if err := os.MkdirAll(filepath.Dir(dst), dirPerm); err != nil { + return err + } + st, err := os.Stat(src) + if err != nil { + return err + } + mode := st.Mode() & 0o777 + if mode == 0 { + mode = filePerm + } + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) + if err != nil { + return err + } + _, copyErr := io.Copy(out, in) + if copyErr != nil { + _ = out.Close() + return copyErr + } + if err := out.Sync(); err != nil { + _ = out.Close() + return err + } + if err := out.Close(); err != nil { + return err + } + // OpenFile is umask-filtered; keep the source executable bit for shim/runtime. + return os.Chmod(dst, mode) +} + +func relocateTree(src, dst string) error { + if err := os.MkdirAll(filepath.Dir(dst), dirPerm); err != nil { + return err + } + if err := os.Rename(src, dst); err == nil { + return nil + } + return copyTree(src, dst) +} + +func copyTree(src, dst string) error { + return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + target := filepath.Join(dst, rel) + switch { + case info.IsDir(): + return os.MkdirAll(target, dirPerm) + case info.Mode()&os.ModeSymlink != 0: + link, err := os.Readlink(path) + if err != nil { + return err + } + if filepath.IsAbs(link) || strings.Contains(link, "..") { + return fmt.Errorf("refusing symlink %s -> %s", path, link) + } + _ = os.Remove(target) + return os.Symlink(link, target) + case info.Mode().IsRegular(): + return copyRegularFile(path, target) + default: + return nil + } + }) +} diff --git a/CubeOps/internal/warehouse/fetch.go b/CubeOps/internal/warehouse/fetch.go new file mode 100644 index 000000000..f075ec5d6 --- /dev/null +++ b/CubeOps/internal/warehouse/fetch.go @@ -0,0 +1,177 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package warehouse + +import ( + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "time" +) + +const ( + SourceGitHub = "github" + SourceCNB = "cnb" + SourceUpload = "upload" + + oneClickAssetFmt = "cube-sandbox-one-click-%s-%s.tar.gz" +) + +// FetchConfig is remote-import policy (whitelist + optional tokens). +type FetchConfig struct { + GitHubRepos []string + CNBRepos []string + GitHubToken string + CNBToken string + Timeout time.Duration +} + +func (c FetchConfig) timeout() time.Duration { + if c.Timeout > 0 { + return c.Timeout + } + return 30 * time.Minute +} + +func allowedRepo(list []string, repo string) bool { + repo = strings.TrimSpace(repo) + if repo == "" || strings.Contains(repo, "..") || strings.ContainsAny(repo, " \t\n\r") { + return false + } + // owner/name, optionally deeper for CNB (group/sub/name) + if strings.Count(repo, "/") < 1 { + return false + } + for _, allowed := range list { + if strings.EqualFold(strings.TrimSpace(allowed), repo) { + return true + } + } + return false +} + +func oneClickAssetName(tag, arch string) string { + return fmt.Sprintf(oneClickAssetFmt, tag, arch) +} + +func githubReleaseURL(repo, tag, arch string) string { + asset := oneClickAssetName(tag, arch) + return "https://github.com/" + repo + "/releases/download/" + url.PathEscape(tag) + "/" + asset +} + +func cnbReleaseURL(repo, tag, arch string) string { + asset := oneClickAssetName(tag, arch) + return "https://cnb.cool/" + repo + "/-/releases/download/" + tag + "/" + asset +} + +func allowedDownloadHost(host string) bool { + host = strings.ToLower(host) + switch { + case host == "github.com": + return true + case strings.HasSuffix(host, ".githubusercontent.com"): + return true + case host == "cnb.cool": + return true + case strings.HasSuffix(host, ".cnb.cool"): + return true + default: + return false + } +} + +func (c FetchConfig) httpClient() *http.Client { + return &http.Client{ + Timeout: c.timeout(), + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return fmt.Errorf("too many redirects") + } + if !allowedDownloadHost(req.URL.Hostname()) { + return fmt.Errorf("redirect to disallowed host %s", req.URL.Hostname()) + } + return nil + }, + } +} + +// DownloadRelease fetches a one-click asset into destPath. +func (c FetchConfig) DownloadRelease(source, repo, tag, arch, destPath string) error { + tag = strings.TrimSpace(tag) + if tag == "" || strings.Contains(tag, "..") || strings.ContainsAny(tag, `/\`) { + return fmt.Errorf("invalid tag") + } + arch, err := NormalizeArch(arch) + if err != nil { + return err + } + + var rawURL, token, tokenHeader string + switch source { + case SourceGitHub: + if !allowedRepo(c.GitHubRepos, repo) { + return fmt.Errorf("github repo %q is not in the warehouse whitelist", repo) + } + rawURL = githubReleaseURL(repo, tag, arch) + token = c.GitHubToken + tokenHeader = "Bearer " + token + case SourceCNB: + if !allowedRepo(c.CNBRepos, repo) { + return fmt.Errorf("cnb repo %q is not in the warehouse whitelist", repo) + } + rawURL = cnbReleaseURL(repo, tag, arch) + token = c.CNBToken + tokenHeader = "Bearer " + token + default: + return fmt.Errorf("unsupported import source %q", source) + } + + u, err := url.Parse(rawURL) + if err != nil || !allowedDownloadHost(u.Hostname()) { + return fmt.Errorf("refusing download URL") + } + + req, err := http.NewRequest(http.MethodGet, rawURL, nil) + if err != nil { + return err + } + req.Header.Set("Accept", "application/octet-stream") + if token != "" { + req.Header.Set("Authorization", tokenHeader) + } + + resp, err := c.httpClient().Do(req) + if err != nil { + return fmt.Errorf("download %s: %w", path.Base(rawURL), err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download %s: HTTP %d", path.Base(rawURL), resp.StatusCode) + } + + if err := os.MkdirAll(filepath.Dir(destPath), dirPerm); err != nil { + return err + } + tmp := destPath + ".partial" + out, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + return err + } + _, copyErr := io.Copy(out, resp.Body) + closeErr := out.Close() + if copyErr != nil { + _ = os.Remove(tmp) + return copyErr + } + if closeErr != nil { + _ = os.Remove(tmp) + return closeErr + } + return os.Rename(tmp, destPath) +} diff --git a/CubeOps/internal/warehouse/ids.go b/CubeOps/internal/warehouse/ids.go new file mode 100644 index 000000000..ad9b9d560 --- /dev/null +++ b/CubeOps/internal/warehouse/ids.go @@ -0,0 +1,86 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package warehouse + +import ( + "fmt" + "path" + "regexp" + "strings" +) + +const ( + ArchAMD64 = "amd64" + ArchARM64 = "arm64" + + ComponentShim = "cube-shim" + ComponentKernel = "cube-kernel-scf" + ComponentImage = "cube-image" + ComponentAgent = "cube-agent" + + CodeNotFound = "warehouse_not_found" + CodeInvalidRequest = "warehouse_invalid_request" + CodeUnauthorizedJob = "warehouse_node_mismatch" +) + +var versionKeyRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._+-]{0,127}$`) + +func NormalizeArch(arch string) (string, error) { + switch strings.ToLower(strings.TrimSpace(arch)) { + case ArchAMD64, "x86_64": + return ArchAMD64, nil + case ArchARM64, "aarch64": + return ArchARM64, nil + default: + return "", fmt.Errorf("unsupported arch %q (want amd64 or arm64)", arch) + } +} + +// ComponentInfo is one entry in the closed warehouse catalog. +type ComponentInfo struct { + Name string +} + +// Catalog returns the four inventory components in display order. +func Catalog() []ComponentInfo { + return []ComponentInfo{ + {Name: ComponentShim}, + {Name: ComponentImage}, + {Name: ComponentAgent}, + {Name: ComponentKernel}, + } +} + +func KnownComponent(name string) bool { + _, err := NormalizeComponent(name) + return err == nil +} + +func NormalizeComponent(name string) (string, error) { + name = strings.TrimSpace(name) + switch name { + case ComponentShim, ComponentKernel, ComponentImage, ComponentAgent: + return name, nil + default: + return "", fmt.Errorf("unsupported component %q", name) + } +} + +func NormalizeVersion(version string) (string, error) { + version = path.Base(strings.TrimSpace(version)) + if version == "." || version == "/" || version == "" { + return "", fmt.Errorf("empty version") + } + if strings.ContainsAny(version, `/\`) || strings.Contains(version, "..") { + return "", fmt.Errorf("invalid version %q", version) + } + if !versionKeyRe.MatchString(version) { + return "", fmt.Errorf("invalid version %q", version) + } + return version, nil +} + +func RelPath(arch, component, version string) string { + return path.Join(arch, component, version) +} diff --git a/CubeOps/internal/warehouse/importer.go b/CubeOps/internal/warehouse/importer.go new file mode 100644 index 000000000..d6e7ab05f --- /dev/null +++ b/CubeOps/internal/warehouse/importer.go @@ -0,0 +1,306 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package warehouse + +import ( + "context" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + "sync/atomic" + "time" + + "github.com/tencentcloud/CubeSandbox/CubeOps/internal/store" +) + +// DefaultImportClaimStaleAfter matches warehouse.write_timeout's default: a +// running import idle this long may be reclaimed by another CubeOps replica. +const DefaultImportClaimStaleAfter = 30 * time.Minute + +// DefaultUploadTTL is how long an unused _uploads/*.tar.gz may sit before sweep +// deletes it. The console starts import jobs immediately after upload; 2h covers +// an abandoned file without parking multi-GB archives on the warehouse PVC. +const DefaultUploadTTL = 2 * time.Hour + +type importStore interface { + ListImportWork(ctx context.Context, staleAfter time.Duration) ([]store.ImportJob, error) + ClaimImportJob(ctx context.Context, id string, staleAfter time.Duration) (bool, error) + UpdateImportJob(ctx context.Context, id, status, errMsg string, bytesTotal int64) error + GetImportJob(ctx context.Context, id string) (*store.ImportJob, error) + InsertWarehouseItem(ctx context.Context, item store.WarehouseItem) (inserted bool, err error) + GetWarehouseItem(ctx context.Context, arch, component, version string) (*store.WarehouseItem, error) + CountLiveImportJobsBySourceRef(ctx context.Context, sourceRef string) (int, error) +} + +// Importer runs one-click imports in the background. +type Importer struct { + store importStore + layout *Layout + fetch FetchConfig + + wake chan struct{} + claimStaleAfter time.Duration + uploadTTL time.Duration + executeCount atomic.Int64 +} + +func NewImporter(s *store.Store, layout *Layout, fetch FetchConfig) *Importer { + return newImporter(s, layout, fetch) +} + +func newImporter(s importStore, layout *Layout, fetch FetchConfig) *Importer { + return &Importer{ + store: s, + layout: layout, + fetch: fetch, + wake: make(chan struct{}, 1), + claimStaleAfter: DefaultImportClaimStaleAfter, + uploadTTL: DefaultUploadTTL, + } +} + +func (im *Importer) Kick() { + select { + case im.wake <- struct{}{}: + default: + } +} + +func (im *Importer) Run(ctx context.Context) { + ticker := time.NewTicker(15 * time.Second) + defer ticker.Stop() + for { + im.drain(ctx) + select { + case <-ctx.Done(): + return + case <-im.wake: + case <-ticker.C: + } + } +} + +func (im *Importer) staleAfter() time.Duration { + if im.claimStaleAfter > 0 { + return im.claimStaleAfter + } + return DefaultImportClaimStaleAfter +} + +func (im *Importer) drain(ctx context.Context) { + jobs, err := im.store.ListImportWork(ctx, im.staleAfter()) + if err != nil { + slog.Error("list pending import jobs", "error", err) + return + } + for _, job := range jobs { + if ctx.Err() != nil { + return + } + claimed, err := im.store.ClaimImportJob(ctx, job.ID, im.staleAfter()) + if err != nil { + slog.Error("claim import job", "id", job.ID, "error", err) + continue + } + if !claimed { + continue + } + im.runJob(ctx, job) + } + im.sweepUploads(ctx) +} + +func (im *Importer) uploadMaxAge() time.Duration { + if im.uploadTTL > 0 { + return im.uploadTTL + } + return DefaultUploadTTL +} + +func isLiveImportStatus(status string) bool { + return status == store.ImportPending || status == store.ImportRunning +} + +// sweepUploads removes leftover import-* directories for finished jobs and +// expired _uploads/*.tar.gz files that no pending/running job still references. +func (im *Importer) sweepUploads(ctx context.Context) { + entries, err := os.ReadDir(im.layout.UploadsDir()) + if err != nil { + return + } + ttl := im.uploadMaxAge() + now := time.Now() + for _, ent := range entries { + path := filepath.Join(im.layout.UploadsDir(), ent.Name()) + if ent.IsDir() { + if !strings.HasPrefix(ent.Name(), "import-") { + continue + } + id := strings.TrimPrefix(ent.Name(), "import-") + if id == "" { + continue + } + job, err := im.store.GetImportJob(ctx, id) + if err != nil { + continue + } + if job != nil && isLiveImportStatus(job.Status) { + continue + } + _ = os.RemoveAll(path) + continue + } + lower := strings.ToLower(ent.Name()) + if !strings.HasSuffix(lower, ".tar.gz") && !strings.HasSuffix(lower, ".tgz") { + continue + } + info, err := ent.Info() + if err != nil { + continue + } + if now.Sub(info.ModTime()) < ttl { + continue + } + im.removeUploadIfIdle(ctx, path) + } +} + +func (im *Importer) removeUploadIfIdle(ctx context.Context, archive string) { + if archive == "" || im.store == nil { + return + } + n, err := im.store.CountLiveImportJobsBySourceRef(ctx, archive) + if err != nil || n > 0 { + return + } + _ = os.Remove(archive) +} + +func (im *Importer) runJob(ctx context.Context, job store.ImportJob) { + bytesTotal, err := im.execute(ctx, job) + if err != nil { + slog.Error("import job failed", "id", job.ID, "error", err) + _ = im.store.UpdateImportJob(ctx, job.ID, store.ImportFailed, err.Error(), bytesTotal) + im.releaseUpload(ctx, job) + return + } + _ = im.store.UpdateImportJob(ctx, job.ID, store.ImportSucceeded, "", bytesTotal) + im.releaseUpload(ctx, job) +} + +func (im *Importer) releaseUpload(ctx context.Context, job store.ImportJob) { + if job.Source != SourceUpload { + return + } + im.removeUploadIfIdle(ctx, job.SourceRef) +} + +func (im *Importer) execute(ctx context.Context, job store.ImportJob) (int64, error) { + im.executeCount.Add(1) + if err := im.layout.EnsureRoot(); err != nil { + return 0, err + } + work := filepath.Join(im.layout.UploadsDir(), "import-"+job.ID) + defer os.RemoveAll(work) + if err := os.RemoveAll(work); err != nil { + return 0, err + } + if err := os.MkdirAll(work, dirPerm); err != nil { + return 0, err + } + + archive := job.SourceRef + switch job.Source { + case SourceUpload: + if archive == "" || !fileExists(archive) { + return 0, fmt.Errorf("uploaded archive missing") + } + case SourceGitHub, SourceCNB: + archive = filepath.Join(work, oneClickAssetName(job.Tag, job.Arch)) + if err := im.fetch.DownloadRelease(job.Source, job.SourceRef, job.Tag, job.Arch, archive); err != nil { + return 0, err + } + default: + return 0, fmt.Errorf("unsupported source %q", job.Source) + } + + st, err := os.Stat(archive) + if err != nil { + return 0, err + } + bytesTotal := st.Size() + + unpackRoot := filepath.Join(work, "unpack") + extracted, err := UnpackOneClick(archive, unpackRoot) + if err != nil { + return bytesTotal, err + } + + for _, item := range extracted { + if ctx.Err() != nil { + return bytesTotal, ctx.Err() + } + if err := im.installExtracted(ctx, job, item); err != nil { + return bytesTotal, err + } + } + return bytesTotal, nil +} + +func (im *Importer) installExtracted(ctx context.Context, job store.ImportJob, item ExtractedComponent) error { + dst := im.layout.Abs(job.Arch, item.Component, item.Version) + if st, err := os.Stat(dst); err == nil && st.IsDir() { + return im.skipOrInsertDest(ctx, job, item, dst) + } + + size, checksum, err := DirSizeAndChecksum(item.Dir, item.Component) + if err != nil { + return err + } + exists, err := im.layout.InstallDir(job.Arch, item.Component, item.Version, item.Dir) + if err != nil { + return err + } + if exists { + return im.skipOrInsertDest(ctx, job, item, dst) + } + return im.insertCatalog(ctx, job, item, size, checksum) +} + +func (im *Importer) skipOrInsertDest(ctx context.Context, job store.ImportJob, item ExtractedComponent, dst string) error { + existing, err := im.store.GetWarehouseItem(ctx, job.Arch, item.Component, item.Version) + if err != nil { + return err + } + if existing != nil { + slog.Info("warehouse skip existing version", + "arch", job.Arch, "component", item.Component, "version", item.Version) + return nil + } + size, checksum, err := DirSizeAndChecksum(dst, item.Component) + if err != nil { + return err + } + return im.insertCatalog(ctx, job, item, size, checksum) +} + +func (im *Importer) insertCatalog(ctx context.Context, job store.ImportJob, item ExtractedComponent, size int64, checksum string) error { + _, err := im.store.InsertWarehouseItem(ctx, store.WarehouseItem{ + Arch: job.Arch, + Component: item.Component, + Version: item.Version, + Source: job.Source, + SourceRef: job.Tag, + RelPath: RelPath(job.Arch, item.Component, item.Version), + SizeBytes: size, + Checksum: checksum, + }) + if err != nil { + _ = im.layout.RemoveDir(job.Arch, item.Component, item.Version) + return err + } + return nil +} diff --git a/CubeOps/internal/warehouse/importer_test.go b/CubeOps/internal/warehouse/importer_test.go new file mode 100644 index 000000000..69ce6ee70 --- /dev/null +++ b/CubeOps/internal/warehouse/importer_test.go @@ -0,0 +1,368 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package warehouse + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/tencentcloud/CubeSandbox/CubeOps/internal/store" +) + +func TestExecuteCleansWorkDirOnFailure(t *testing.T) { + root := t.TempDir() + layout := NewLayout(root) + im := NewImporter(nil, layout, FetchConfig{}) + job := store.ImportJob{ + ID: "deadbeef", + Source: SourceUpload, + SourceRef: filepath.Join(root, "missing.tar.gz"), + Arch: "amd64", + } + if _, err := im.execute(context.Background(), job); err == nil { + t.Fatal("expected missing-archive error") + } + work := filepath.Join(layout.UploadsDir(), "import-deadbeef") + if _, err := os.Stat(work); !os.IsNotExist(err) { + t.Fatalf("work dir leftover after failed import: %v", err) + } +} + +func TestExecuteClearsLeftoverWorkDir(t *testing.T) { + root := t.TempDir() + layout := NewLayout(root) + if err := layout.EnsureRoot(); err != nil { + t.Fatal(err) + } + work := filepath.Join(layout.UploadsDir(), "import-staleid") + if err := os.MkdirAll(filepath.Join(work, "junk"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(work, "junk", "leftover"), []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + im := NewImporter(nil, layout, FetchConfig{}) + job := store.ImportJob{ + ID: "staleid", + Source: SourceUpload, + SourceRef: filepath.Join(root, "missing.tar.gz"), + Arch: "amd64", + } + if _, err := im.execute(context.Background(), job); err == nil { + t.Fatal("expected missing-archive error") + } + if _, err := os.Stat(work); !os.IsNotExist(err) { + t.Fatalf("leftover work dir after execute: %v", err) + } +} + +type fakeImportStore struct { + mu sync.Mutex + job store.ImportJob + siblings []store.ImportJob + claimed bool + updates []string + inserted int + insertKeys []string + items map[string]store.WarehouseItem +} + +func warehouseItemKey(arch, component, version string) string { + return arch + "|" + component + "|" + version +} + +func (f *fakeImportStore) eachJob(fn func(*store.ImportJob)) { + fn(&f.job) + for i := range f.siblings { + fn(&f.siblings[i]) + } +} + +func (f *fakeImportStore) ListImportWork(context.Context, time.Duration) ([]store.ImportJob, error) { + f.mu.Lock() + defer f.mu.Unlock() + job := f.job + return []store.ImportJob{job}, nil +} + +func (f *fakeImportStore) ClaimImportJob(context.Context, string, time.Duration) (bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.claimed { + return false, nil + } + f.claimed = true + f.job.Status = store.ImportRunning + return true, nil +} + +func (f *fakeImportStore) UpdateImportJob(_ context.Context, id, status, _ string, _ int64) error { + f.mu.Lock() + defer f.mu.Unlock() + f.updates = append(f.updates, status) + f.eachJob(func(job *store.ImportJob) { + if job.ID == id { + job.Status = status + } + }) + return nil +} + +func (f *fakeImportStore) GetImportJob(_ context.Context, id string) (*store.ImportJob, error) { + f.mu.Lock() + defer f.mu.Unlock() + var found *store.ImportJob + f.eachJob(func(job *store.ImportJob) { + if job.ID == id { + cp := *job + found = &cp + } + }) + return found, nil +} + +func (f *fakeImportStore) InsertWarehouseItem(_ context.Context, item store.WarehouseItem) (bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.items == nil { + f.items = map[string]store.WarehouseItem{} + } + key := warehouseItemKey(item.Arch, item.Component, item.Version) + f.items[key] = item + f.insertKeys = append(f.insertKeys, key) + f.inserted++ + return true, nil +} + +func (f *fakeImportStore) GetWarehouseItem(_ context.Context, arch, component, version string) (*store.WarehouseItem, error) { + f.mu.Lock() + defer f.mu.Unlock() + item, ok := f.items[warehouseItemKey(arch, component, version)] + if !ok { + return nil, nil + } + cp := item + return &cp, nil +} + +func (f *fakeImportStore) CountLiveImportJobsBySourceRef(_ context.Context, sourceRef string) (int, error) { + f.mu.Lock() + defer f.mu.Unlock() + n := 0 + f.eachJob(func(job *store.ImportJob) { + if job.ID == "" { + return + } + if job.SourceRef == sourceRef && isLiveImportStatus(job.Status) { + n++ + } + }) + return n, nil +} + +func TestDrainClaimsOnce(t *testing.T) { + root := t.TempDir() + archive := buildFakeOneClick(t, filepath.Join(root, "pkg"), "v0.6.0") + layout := NewLayout(filepath.Join(root, "wh")) + fake := &fakeImportStore{job: store.ImportJob{ + ID: "job-1", + Source: SourceUpload, + SourceRef: archive, + Tag: "v0.6.0", + Arch: "amd64", + Status: store.ImportPending, + }} + imA := newImporter(fake, layout, FetchConfig{}) + imB := newImporter(fake, layout, FetchConfig{}) + + var start, done sync.WaitGroup + start.Add(2) + done.Add(2) + run := func(im *Importer) { + defer done.Done() + start.Done() + start.Wait() + im.drain(context.Background()) + } + go run(imA) + go run(imB) + done.Wait() + + got := imA.executeCount.Load() + imB.executeCount.Load() + if got != 1 { + t.Fatalf("execute count=%d want 1", got) + } + fake.mu.Lock() + defer fake.mu.Unlock() + if !fake.claimed { + t.Fatal("job was not claimed") + } + if len(fake.updates) != 1 || fake.updates[0] != store.ImportSucceeded { + t.Fatalf("updates=%v want one succeeded", fake.updates) + } + if fake.inserted == 0 { + t.Fatal("winner did not insert warehouse items") + } +} + +func TestExecuteInsertsWhenDirExistsWithoutRow(t *testing.T) { + root := t.TempDir() + archive := buildFakeOneClick(t, filepath.Join(root, "pkg"), "v0.6.0") + layout := NewLayout(filepath.Join(root, "wh")) + if err := layout.EnsureRoot(); err != nil { + t.Fatal(err) + } + shimDir := layout.Abs(ArchAMD64, ComponentShim, "v0.6.0") + if err := os.MkdirAll(filepath.Join(shimDir, "bin"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(shimDir, "bin", "containerd-shim-cube-rs"), []byte("old"), 0o755); err != nil { + t.Fatal(err) + } + fake := &fakeImportStore{} + im := newImporter(fake, layout, FetchConfig{}) + job := store.ImportJob{ + ID: "repair-1", Source: SourceUpload, SourceRef: archive, + Tag: "v0.6.0", Arch: ArchAMD64, Status: store.ImportPending, + } + if _, err := im.execute(context.Background(), job); err != nil { + t.Fatalf("execute: %v", err) + } + fake.mu.Lock() + defer fake.mu.Unlock() + if _, ok := fake.items[warehouseItemKey(ArchAMD64, ComponentShim, "v0.6.0")]; !ok { + t.Fatal("missing warehouse row for pre-existing shim dir") + } +} + +func TestExecuteSkipsWhenDirAndRowExist(t *testing.T) { + root := t.TempDir() + archive := buildFakeOneClick(t, filepath.Join(root, "pkg"), "v0.6.0") + layout := NewLayout(filepath.Join(root, "wh")) + if err := layout.EnsureRoot(); err != nil { + t.Fatal(err) + } + shimDir := layout.Abs(ArchAMD64, ComponentShim, "v0.6.0") + if err := os.MkdirAll(filepath.Join(shimDir, "bin"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(shimDir, "bin", "containerd-shim-cube-rs"), []byte("old"), 0o755); err != nil { + t.Fatal(err) + } + shimKey := warehouseItemKey(ArchAMD64, ComponentShim, "v0.6.0") + fake := &fakeImportStore{items: map[string]store.WarehouseItem{ + shimKey: {Arch: ArchAMD64, Component: ComponentShim, Version: "v0.6.0"}, + }} + im := newImporter(fake, layout, FetchConfig{}) + job := store.ImportJob{ + ID: "skip-1", Source: SourceUpload, SourceRef: archive, + Tag: "v0.6.0", Arch: ArchAMD64, Status: store.ImportPending, + } + if _, err := im.execute(context.Background(), job); err != nil { + t.Fatalf("execute: %v", err) + } + fake.mu.Lock() + defer fake.mu.Unlock() + for _, key := range fake.insertKeys { + if key == shimKey { + t.Fatal("inserted existing shim key") + } + } +} + +func putUploadArchive(t *testing.T, layout *Layout, src, name string) string { + t.Helper() + if err := layout.EnsureRoot(); err != nil { + t.Fatal(err) + } + dest := filepath.Join(layout.UploadsDir(), name) + if err := copyRegularFile(src, dest); err != nil { + t.Fatal(err) + } + return dest +} + +func TestRunJobDeletesUploadWhenNoLiveSiblings(t *testing.T) { + root := t.TempDir() + src := buildFakeOneClick(t, filepath.Join(root, "pkg"), "v0.6.0") + layout := NewLayout(filepath.Join(root, "wh")) + archive := putUploadArchive(t, layout, src, "u.tar.gz") + fake := &fakeImportStore{job: store.ImportJob{ + ID: "job-a", Source: SourceUpload, SourceRef: archive, + Tag: "v0.6.0", Arch: ArchAMD64, Status: store.ImportRunning, + }} + im := newImporter(fake, layout, FetchConfig{}) + im.runJob(context.Background(), fake.job) + if _, err := os.Stat(archive); !os.IsNotExist(err) { + t.Fatalf("upload left after last job: %v", err) + } +} + +func TestRunJobKeepsUploadWhileSiblingLive(t *testing.T) { + root := t.TempDir() + src := buildFakeOneClick(t, filepath.Join(root, "pkg"), "v0.6.0") + layout := NewLayout(filepath.Join(root, "wh")) + archive := putUploadArchive(t, layout, src, "u.tar.gz") + jobA := store.ImportJob{ + ID: "job-a", Source: SourceUpload, SourceRef: archive, + Tag: "v0.6.0", Arch: ArchAMD64, Status: store.ImportRunning, + } + jobB := store.ImportJob{ + ID: "job-b", Source: SourceUpload, SourceRef: archive, + Tag: "v0.6.0", Arch: ArchARM64, Status: store.ImportPending, + } + fake := &fakeImportStore{job: jobA, siblings: []store.ImportJob{jobB}} + im := newImporter(fake, layout, FetchConfig{}) + im.runJob(context.Background(), jobA) + if _, err := os.Stat(archive); err != nil { + t.Fatalf("upload removed while sibling pending: %v", err) + } + im.runJob(context.Background(), jobB) + if _, err := os.Stat(archive); !os.IsNotExist(err) { + t.Fatalf("upload left after last sibling: %v", err) + } +} + +func TestSweepUploadsTTL(t *testing.T) { + root := t.TempDir() + layout := NewLayout(root) + if err := layout.EnsureRoot(); err != nil { + t.Fatal(err) + } + stale := filepath.Join(layout.UploadsDir(), "old.tar.gz") + fresh := filepath.Join(layout.UploadsDir(), "new.tar.gz") + live := filepath.Join(layout.UploadsDir(), "live.tar.gz") + for _, p := range []string{stale, fresh, live} { + if err := os.WriteFile(p, []byte("pkg"), 0o600); err != nil { + t.Fatal(err) + } + } + old := time.Now().Add(-3 * time.Hour) + if err := os.Chtimes(stale, old, old); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(live, old, old); err != nil { + t.Fatal(err) + } + fake := &fakeImportStore{job: store.ImportJob{ + ID: "live-1", Source: SourceUpload, SourceRef: live, + Status: store.ImportPending, + }} + im := newImporter(fake, layout, FetchConfig{}) + im.uploadTTL = time.Hour + im.sweepUploads(context.Background()) + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Fatal("stale unreferenced upload was not swept") + } + if _, err := os.Stat(fresh); err != nil { + t.Fatalf("fresh upload swept: %v", err) + } + if _, err := os.Stat(live); err != nil { + t.Fatalf("live-job upload swept: %v", err) + } +} diff --git a/CubeOps/internal/warehouse/unpack.go b/CubeOps/internal/warehouse/unpack.go new file mode 100644 index 000000000..c7c339ce7 --- /dev/null +++ b/CubeOps/internal/warehouse/unpack.go @@ -0,0 +1,288 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package warehouse + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" +) + +// ExtractedComponent is one inventory tree ready to install. +type ExtractedComponent struct { + Component string + Version string + Dir string +} + +// ReleaseManifest is the outer one-click release-manifest.json. +type ReleaseManifest struct { + Components map[string]manifestComponent `json:"components"` + GuestImage manifestGuestImage `json:"guest_image"` +} + +type manifestComponent struct { + Version string `json:"version"` +} + +type manifestGuestImage struct { + Version string `json:"version"` + AgentVersion string `json:"agent_version"` +} + +// UnpackOneClick extracts a one-click tar.gz into destRoot and returns the +// inventory trees (shim/image/agent plus one kernel dir per present variant). +// destRoot is a scratch directory owned by the caller. +func UnpackOneClick(archivePath, destRoot string) ([]ExtractedComponent, error) { + outer := filepath.Join(destRoot, "outer") + if err := os.MkdirAll(outer, dirPerm); err != nil { + return nil, err + } + f, err := os.Open(archivePath) + if err != nil { + return nil, err + } + err = ExtractTarGz(f, outer) + _ = f.Close() + if err != nil { + return nil, fmt.Errorf("extract outer package: %w", err) + } + + outerRoot, err := findDirContaining(outer, "release-manifest.json") + if err != nil { + return nil, err + } + manifest, err := readReleaseManifest(filepath.Join(outerRoot, "release-manifest.json")) + if err != nil { + return nil, err + } + + innerArchive := filepath.Join(outerRoot, "assets", "package", "sandbox-package.tar.gz") + if !fileExists(innerArchive) { + return nil, fmt.Errorf("missing assets/package/sandbox-package.tar.gz") + } + inner := filepath.Join(destRoot, "inner") + if err := os.MkdirAll(inner, dirPerm); err != nil { + return nil, err + } + in, err := os.Open(innerArchive) + if err != nil { + return nil, err + } + err = ExtractTarGz(in, inner) + _ = in.Close() + if err != nil { + return nil, fmt.Errorf("extract sandbox-package: %w", err) + } + + pkgRoot, err := findPackageRoot(inner) + if err != nil { + return nil, err + } + + var out []ExtractedComponent + staging := filepath.Join(destRoot, "staging") + if err := os.MkdirAll(staging, dirPerm); err != nil { + return nil, err + } + + for _, name := range []string{ComponentShim, ComponentImage, ComponentAgent} { + src := filepath.Join(pkgRoot, name) + if st, err := os.Stat(src); err != nil || !st.IsDir() { + continue + } + ver, err := resolveComponentVersion(src, name, manifest) + if err != nil { + return nil, err + } + dst := filepath.Join(staging, name+"-"+ver) + if err := relocateTree(src, dst); err != nil { + return nil, fmt.Errorf("copy %s: %w", name, err) + } + if err := ValidateInstalledTree(dst, name); err != nil { + return nil, fmt.Errorf("%s: %w", name, err) + } + out = append(out, ExtractedComponent{Component: name, Version: ver, Dir: dst}) + } + + kernelSrc := filepath.Join(pkgRoot, ComponentKernel) + if st, err := os.Stat(kernelSrc); err == nil && st.IsDir() { + kernels, err := inventoryKernelVariants(kernelSrc, staging) + if err != nil { + return nil, err + } + out = append(out, kernels...) + } + + if len(out) == 0 { + return nil, fmt.Errorf("one-click package contained no inventory components") + } + return out, nil +} + +func findDirContaining(root, filename string) (string, error) { + var found string + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if d.Name() == filename { + found = filepath.Dir(path) + return filepath.SkipAll + } + return nil + }) + if err != nil { + return "", err + } + if found == "" { + return "", fmt.Errorf("missing %s in archive", filename) + } + return found, nil +} + +func findPackageRoot(inner string) (string, error) { + if isPackageRoot(inner) { + return inner, nil + } + nested := filepath.Join(inner, "sandbox-package") + if isPackageRoot(nested) { + return nested, nil + } + var found string + _ = filepath.WalkDir(inner, func(path string, d os.DirEntry, err error) error { + if err != nil || !d.IsDir() { + return err + } + if isPackageRoot(path) { + found = path + return filepath.SkipAll + } + return nil + }) + if found == "" { + return "", fmt.Errorf("sandbox-package has no cube-shim/cube-image/cube-agent/cube-kernel-scf") + } + return found, nil +} + +func isPackageRoot(dir string) bool { + for _, name := range []string{ComponentShim, ComponentImage, ComponentAgent, ComponentKernel} { + if st, err := os.Stat(filepath.Join(dir, name)); err == nil && st.IsDir() { + return true + } + } + return false +} + +func readReleaseManifest(path string) (*ReleaseManifest, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read release-manifest.json: %w", err) + } + var m ReleaseManifest + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("parse release-manifest.json: %w", err) + } + return &m, nil +} + +func resolveComponentVersion(src, name string, manifest *ReleaseManifest) (string, error) { + ver := readVersionFile(filepath.Join(src, "version")) + if ver == "" && manifest != nil { + switch name { + case ComponentShim: + ver = manifestComponentVersion(manifest, "containerd-shim-cube-rs") + if ver == "" { + ver = manifestComponentVersion(manifest, "cube-runtime") + } + case ComponentImage: + ver = strings.TrimSpace(manifest.GuestImage.Version) + case ComponentAgent: + ver = manifestComponentVersion(manifest, "cube-agent") + if ver == "" || ver == "unknown" { + ver = strings.TrimSpace(manifest.GuestImage.AgentVersion) + } + } + } + ver = strings.TrimSpace(ver) + if ver == "" || strings.EqualFold(ver, "unknown") { + return "", fmt.Errorf("cannot resolve version for %s", name) + } + return NormalizeVersion(ver) +} + +func manifestComponentVersion(m *ReleaseManifest, key string) string { + if m == nil || m.Components == nil { + return "" + } + return strings.TrimSpace(m.Components[key].Version) +} + +func readVersionFile(path string) string { + raw, err := os.ReadFile(path) + if err != nil { + return "" + } + return strings.TrimSpace(string(raw)) +} + +func inventoryKernelVariants(src, staging string) ([]ExtractedComponent, error) { + var out []ExtractedComponent + for _, variant := range []string{"bm", "pvm"} { + file := filepath.Join(src, "vmlinux-"+variant) + if !fileExists(file) { + continue + } + digest, err := fileSHA256(file) + if err != nil { + return nil, fmt.Errorf("hash %s: %w", file, err) + } + if len(digest) < 12 { + return nil, fmt.Errorf("short sha256 for %s", file) + } + ver := "sha256-" + digest[:12] + dst := filepath.Join(staging, ComponentKernel+"-"+variant+"-"+ver) + if err := os.MkdirAll(dst, dirPerm); err != nil { + return nil, err + } + if err := copyRegularFile(file, filepath.Join(dst, "vmlinux-"+variant)); err != nil { + return nil, err + } + if err := os.Symlink("vmlinux-"+variant, filepath.Join(dst, "vmlinux")); err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(dst, "variant"), []byte(variant+"\n"), filePerm); err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(dst, "version"), []byte("sha256:"+digest+"\n"), filePerm); err != nil { + return nil, err + } + if err := ValidateInstalledTree(dst, ComponentKernel); err != nil { + return nil, err + } + out = append(out, ExtractedComponent{Component: ComponentKernel, Version: ver, Dir: dst}) + } + if len(out) == 0 { + return nil, fmt.Errorf("cube-kernel-scf has neither vmlinux-bm nor vmlinux-pvm") + } + return out, nil +} + +// ContentShortHash returns sha256-<12> of a file (kernel inventory key). +func ContentShortHash(path string) (string, error) { + sum, err := fileSHA256(path) + if err != nil { + return "", err + } + if len(sum) < 12 { + return "", fmt.Errorf("short sha256") + } + return "sha256-" + sum[:12], nil +} diff --git a/CubeOps/internal/warehouse/unpack_test.go b/CubeOps/internal/warehouse/unpack_test.go new file mode 100644 index 000000000..27273a0b5 --- /dev/null +++ b/CubeOps/internal/warehouse/unpack_test.go @@ -0,0 +1,296 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package warehouse + +import ( + "archive/tar" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "os" + "path/filepath" + "syscall" + "testing" +) + +func TestUnpackOneClick_ShimFromManifestKernelHash(t *testing.T) { + dir := t.TempDir() + archive := buildFakeOneClick(t, dir, "v0.6.0") + + out, err := UnpackOneClick(archive, filepath.Join(dir, "work")) + if err != nil { + t.Fatalf("UnpackOneClick: %v", err) + } + got := map[string]string{} + for _, item := range out { + got[item.Component+"/"+item.Version] = item.Dir + if err := ValidateInstalledTree(item.Dir, item.Component); err != nil { + t.Errorf("validate %s: %v", item.Component, err) + } + } + if _, ok := got["cube-shim/v0.6.0"]; !ok { + t.Fatalf("missing shim from manifest, got %#v", got) + } + shimBin := filepath.Join(got["cube-shim/v0.6.0"], "bin", "containerd-shim-cube-rs") + st, err := os.Stat(shimBin) + if err != nil { + t.Fatal(err) + } + if st.Mode()&0o111 == 0 { + t.Fatalf("shim binary lost executable bit: %s", st.Mode()) + } + rtBin := filepath.Join(got["cube-shim/v0.6.0"], "bin", "cube-runtime") + st, err = os.Stat(rtBin) + if err != nil { + t.Fatal(err) + } + if st.Mode()&0o111 == 0 { + t.Fatalf("cube-runtime lost executable bit: %s", st.Mode()) + } + if _, ok := got["cube-image/v0.6.0"]; !ok { + t.Fatalf("missing image, got %#v", got) + } + if _, ok := got["cube-agent/v0.6.0"]; !ok { + t.Fatalf("missing agent, got %#v", got) + } + bm := filepath.Join(dir, "pkg", "cube-kernel-scf", "vmlinux-bm") + short, err := ContentShortHash(bm) + if err != nil { + t.Fatal(err) + } + if _, ok := got["cube-kernel-scf/"+short]; !ok { + t.Fatalf("missing kernel %s, got %#v", short, got) + } +} + +func TestUnpackOneClick_SkipSameVersionInstall(t *testing.T) { + dir := t.TempDir() + archive := buildFakeOneClick(t, dir, "v0.6.0") + items, err := UnpackOneClick(archive, filepath.Join(dir, "work")) + if err != nil { + t.Fatal(err) + } + layout := NewLayout(filepath.Join(dir, "wh")) + if err := layout.EnsureRoot(); err != nil { + t.Fatal(err) + } + item := items[0] + exists, err := layout.InstallDir("amd64", item.Component, item.Version, item.Dir) + if err != nil || exists { + t.Fatalf("first install exists=%v err=%v", exists, err) + } + // Re-unpack so we have a fresh src dir. + items2, err := UnpackOneClick(archive, filepath.Join(dir, "work2")) + if err != nil { + t.Fatal(err) + } + var again ExtractedComponent + for _, it := range items2 { + if it.Component == item.Component && it.Version == item.Version { + again = it + break + } + } + exists, err = layout.InstallDir("amd64", again.Component, again.Version, again.Dir) + if err != nil { + t.Fatal(err) + } + if !exists { + t.Fatal("expected skip when version already present") + } +} + +func TestInstallDirCrossDeviceCopy(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "src") + if err := os.MkdirAll(src, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "blob"), []byte("data"), 0o644); err != nil { + t.Fatal(err) + } + old := renameDir + renameDir = func(from, to string) error { + return &os.LinkError{Op: "rename", Old: from, New: to, Err: syscall.EXDEV} + } + t.Cleanup(func() { renameDir = old }) + + layout := NewLayout(filepath.Join(dir, "wh")) + if err := layout.EnsureRoot(); err != nil { + t.Fatal(err) + } + exists, err := layout.InstallDir("amd64", "cube-shim", "v0.6.0", src) + if err != nil || exists { + t.Fatalf("exists=%v err=%v", exists, err) + } + got, err := os.ReadFile(filepath.Join(layout.Abs("amd64", "cube-shim", "v0.6.0"), "blob")) + if err != nil { + t.Fatal(err) + } + if string(got) != "data" { + t.Fatalf("copied %q", got) + } + if _, err := os.Stat(src); !os.IsNotExist(err) { + t.Fatalf("src leftover: %v", err) + } +} + +func TestSafeJoinRejectsTraversal(t *testing.T) { + root := t.TempDir() + if _, err := safeJoin(root, "../etc/passwd"); err == nil { + t.Fatal("expected traversal reject") + } +} + +func TestFetchWhitelist(t *testing.T) { + cfg := FetchConfig{GitHubRepos: []string{"TencentCloud/CubeSandbox"}} + if allowedRepo(cfg.GitHubRepos, "evil/repo") { + t.Fatal("evil repo allowed") + } + if !allowedRepo(cfg.GitHubRepos, "TencentCloud/CubeSandbox") { + t.Fatal("official repo rejected") + } + err := cfg.DownloadRelease(SourceGitHub, "evil/repo", "v0.6.0", "amd64", filepath.Join(t.TempDir(), "x")) + if err == nil { + t.Fatal("expected whitelist error") + } +} + +func buildFakeOneClick(t *testing.T, dir, tag string) string { + t.Helper() + pkg := filepath.Join(dir, "pkg") + mustMkdir(t, filepath.Join(pkg, "cube-shim", "bin")) + mustWriteMode(t, filepath.Join(pkg, "cube-shim", "bin", "containerd-shim-cube-rs"), "shim", 0o755) + mustWriteMode(t, filepath.Join(pkg, "cube-shim", "bin", "cube-runtime"), "runtime", 0o755) + mustMkdir(t, filepath.Join(pkg, "cube-image")) + mustWrite(t, filepath.Join(pkg, "cube-image", "cube-guest-image-cpu.img"), "img") + mustWrite(t, filepath.Join(pkg, "cube-image", "version"), tag+"\n") + mustMkdir(t, filepath.Join(pkg, "cube-agent")) + mustWrite(t, filepath.Join(pkg, "cube-agent", "cube-agent.ext4"), "agent") + mustWrite(t, filepath.Join(pkg, "cube-agent", "version"), tag+"\n") + mustMkdir(t, filepath.Join(pkg, "cube-kernel-scf")) + mustWrite(t, filepath.Join(pkg, "cube-kernel-scf", "vmlinux-bm"), "kernel-bm-bytes") + + inner := filepath.Join(dir, "sandbox-package.tar.gz") + tarGzDir(t, inner, pkg, "sandbox-package") + + outerDir := filepath.Join(dir, "outer") + mustMkdir(t, filepath.Join(outerDir, "assets", "package")) + if err := copyRegularFile(inner, filepath.Join(outerDir, "assets", "package", "sandbox-package.tar.gz")); err != nil { + t.Fatal(err) + } + manifest := ReleaseManifest{ + Components: map[string]manifestComponent{ + "containerd-shim-cube-rs": {Version: tag}, + }, + GuestImage: manifestGuestImage{Version: tag, AgentVersion: tag}, + } + raw, _ := json.Marshal(manifest) + mustWrite(t, filepath.Join(outerDir, "release-manifest.json"), string(raw)) + + archive := filepath.Join(dir, "one-click.tar.gz") + tarGzDir(t, archive, outerDir, "") + return archive +} + +func tarGzDir(t *testing.T, dest, src, prefix string) { + t.Helper() + f, err := os.Create(dest) + if err != nil { + t.Fatal(err) + } + defer f.Close() + gz := gzip.NewWriter(f) + defer gz.Close() + tw := tar.NewWriter(gz) + defer tw.Close() + err = filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + name := rel + if prefix != "" { + if rel == "." { + name = prefix + } else { + name = prefix + "/" + filepath.ToSlash(rel) + } + } else if rel == "." { + return nil + } + hdr, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + hdr.Name = filepath.ToSlash(name) + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + in, err := os.Open(path) + if err != nil { + return err + } + defer in.Close() + _, err = io.Copy(tw, in) + return err + }) + if err != nil { + t.Fatal(err) + } +} + +func mustMkdir(t *testing.T, dir string) { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } +} + +func mustWrite(t *testing.T, path, body string) { + t.Helper() + mustWriteMode(t, path, body, 0o644) +} + +func mustWriteMode(t *testing.T, path, body string, mode os.FileMode) { + t.Helper() + if err := os.WriteFile(path, []byte(body), mode); err != nil { + t.Fatal(err) + } +} + +func TestAllowedDownloadHost(t *testing.T) { + if allowedDownloadHost("169.254.169.254") || allowedDownloadHost("127.0.0.1") { + t.Fatal("loopback/metadata host allowed") + } + if !allowedDownloadHost("github.com") || !allowedDownloadHost("objects.githubusercontent.com") { + t.Fatal("github hosts rejected") + } + if !allowedDownloadHost("cnb.cool") || !allowedDownloadHost("cdn.cnb.cool") { + t.Fatal("cnb hosts rejected") + } +} + +func TestKernelShortHashStable(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "vmlinux-bm") + mustWrite(t, p, "kernel-bm-bytes") + sum := sha256.Sum256([]byte("kernel-bm-bytes")) + want := "sha256-" + hex.EncodeToString(sum[:])[:12] + got, err := ContentShortHash(p) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("got %s want %s", got, want) + } +} diff --git a/Cubelet/config/config.toml b/Cubelet/config/config.toml index 5d826d0aa..6099ecf1a 100644 --- a/Cubelet/config/config.toml +++ b/Cubelet/config/config.toml @@ -39,6 +39,8 @@ dynamic_config_path = "/usr/local/services/cubetoolbox/Cubelet/dynamicconf/conf. # Static-only cadence for node status and resource reporting. # Do not configure this in dynamicconf/conf.yaml. node_status_update_frequency = "1s" + cubeops_addr = "" + cubeops_timeout = "10m" [plugins."io.cubelet.internal.v1.volume"] [plugins."io.cubelet.internal.v1.shimlog"] diff --git a/Cubelet/pkg/controller/runtemplate/components/component_versioned_manager.go b/Cubelet/pkg/controller/runtemplate/components/component_versioned_manager.go index 8f4e3c578..0713a142b 100644 --- a/Cubelet/pkg/controller/runtemplate/components/component_versioned_manager.go +++ b/Cubelet/pkg/controller/runtemplate/components/component_versioned_manager.go @@ -11,12 +11,13 @@ import ( "os" "path" "strings" + "sync" "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/controller/nodedistribution/distribution" "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/controller/runtemplate/templatetypes" "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/log" "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/utils" - "github.com/tencentcloud/CubeSandbox/cubelog" + CubeLog "github.com/tencentcloud/CubeSandbox/cubelog" ) // ErrComponentVersionMissing means the requested version is not in local inventory. @@ -32,8 +33,15 @@ func DefaultConfig() *ComponentManagerConfig { } } +// MissingVersionFetcher downloads a missing inventory tree. Optional. +type MissingVersionFetcher interface { + Fetch(ctx context.Context, name, version string) error +} + type ComponentManager struct { config *ComponentManagerConfig + fetch MissingVersionFetcher + mu sync.RWMutex } func NewComponentManager(config *ComponentManagerConfig) *ComponentManager { @@ -50,10 +58,22 @@ func NewComponentManager(config *ComponentManagerConfig) *ComponentManager { return cm } +// SetFetcher installs the optional CubeOps downloader. Safe to call once at start. +func (c *ComponentManager) SetFetcher(f MissingVersionFetcher) { + c.mu.Lock() + defer c.mu.Unlock() + c.fetch = f +} + +func (c *ComponentManager) fetcher() MissingVersionFetcher { + c.mu.RLock() + defer c.mu.RUnlock() + return c.fetch +} + // Ensure resolves name/version/relativePath to an absolute LocalPath in inventory. // Missing directory or file returns ErrComponentVersionMissing. func (c *ComponentManager) Ensure(ctx context.Context, name, version, relativePath string) (string, error) { - _ = ctx name = strings.TrimSpace(name) version = strings.TrimSpace(version) relativePath = strings.TrimSpace(relativePath) @@ -80,7 +100,20 @@ func (c *ComponentManager) Ensure(ctx context.Context, name, version, relativePa return "", err } if !ok { - return "", fmt.Errorf("%w: dir %s (component=%s version=%s)", ErrComponentVersionMissing, versionedDir, name, version) + fetch := c.fetcher() + if fetch == nil { + return "", fmt.Errorf("%w: dir %s (component=%s version=%s)", ErrComponentVersionMissing, versionedDir, name, version) + } + if fetchErr := fetch.Fetch(ctx, name, version); fetchErr != nil { + return "", fetchErr + } + ok, err = utils.DenExist(versionedDir) + if err != nil { + return "", err + } + if !ok { + return "", fmt.Errorf("%w: dir %s after download (component=%s version=%s)", ErrComponentVersionMissing, versionedDir, name, version) + } } localPath := path.Join(versionedDir, relativePath) diff --git a/Cubelet/pkg/controller/runtemplate/components/component_warehouse_test.go b/Cubelet/pkg/controller/runtemplate/components/component_warehouse_test.go new file mode 100644 index 000000000..c8fe56426 --- /dev/null +++ b/Cubelet/pkg/controller/runtemplate/components/component_warehouse_test.go @@ -0,0 +1,233 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package components + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/controller/runtemplate/templatetypes" + "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/warehouse" +) + +type stubFetcher struct { + calls int32 + fn func(ctx context.Context, name, version string) error +} + +func (s *stubFetcher) Fetch(ctx context.Context, name, version string) error { + atomic.AddInt32(&s.calls, 1) + if s.fn != nil { + return s.fn(ctx, name, version) + } + return nil +} + +func TestEnsure_LocalHitDoesNotFetch(t *testing.T) { + manager, config, _ := setupTestManager(t) + shimDir := path.Join(config.VersionedBaseDir, "cube-shim", "1.0.0", "bin") + require.NoError(t, os.MkdirAll(shimDir, 0755)) + shimFile := path.Join(shimDir, "containerd-shim-cube-rs") + require.NoError(t, os.WriteFile(shimFile, []byte("shim"), 0755)) + + stub := &stubFetcher{fn: func(context.Context, string, string) error { + t.Fatal("fetch should not run on local hit") + return nil + }} + manager.SetFetcher(stub) + got, err := manager.Ensure(context.Background(), "cube-shim", "1.0.0", "") + require.NoError(t, err) + assert.Equal(t, shimFile, got) + assert.Equal(t, int32(0), atomic.LoadInt32(&stub.calls)) +} + +func TestEnsure_MissingWithoutFetcher(t *testing.T) { + manager, _, _ := setupTestManager(t) + _, err := manager.Ensure(context.Background(), "cube-shim", "9.9.9", "") + require.Error(t, err) + assert.True(t, errors.Is(err, ErrComponentVersionMissing)) +} + +func TestEnsure_DownloadSuccess(t *testing.T) { + manager, config, _ := setupTestManager(t) + stub := &stubFetcher{fn: func(_ context.Context, name, version string) error { + dir := path.Join(config.VersionedBaseDir, name, version, "bin") + require.NoError(t, os.MkdirAll(dir, 0755)) + require.NoError(t, os.WriteFile(path.Join(dir, "containerd-shim-cube-rs"), []byte("shim"), 0755)) + return nil + }} + manager.SetFetcher(stub) + got, err := manager.Ensure(context.Background(), "cube-shim", "2.0.0", "") + require.NoError(t, err) + assert.FileExists(t, got) + assert.Equal(t, int32(1), atomic.LoadInt32(&stub.calls)) +} + +func TestEnsure_WarehouseNotFound(t *testing.T) { + manager, _, _ := setupTestManager(t) + manager.SetFetcher(&stubFetcher{fn: func(context.Context, string, string) error { + return warehouse.ErrNotFound + }}) + _, err := manager.Ensure(context.Background(), "cube-shim", "missing", "") + require.Error(t, err) + assert.True(t, errors.Is(err, warehouse.ErrNotFound)) + assert.False(t, errors.Is(err, ErrComponentVersionMissing)) +} + +func TestEnsure_DownloadFailed(t *testing.T) { + manager, _, _ := setupTestManager(t) + manager.SetFetcher(&stubFetcher{fn: func(context.Context, string, string) error { + return warehouse.ErrDownloadFailed + }}) + _, err := manager.Ensure(context.Background(), "cube-shim", "bad", "") + require.Error(t, err) + assert.True(t, errors.Is(err, warehouse.ErrDownloadFailed)) +} + +func TestEnsure_ConcurrentSingleflight(t *testing.T) { + manager, config, _ := setupTestManager(t) + var blobs int32 + release := make(chan struct{}) + started := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/blob") { + if atomic.AddInt32(&blobs, 1) == 1 { + close(started) + } + <-release + writeShimTar(t, w) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + manager.SetFetcher(warehouse.NewFetcher(warehouse.NewClient(srv.URL, "node-1", "amd64", time.Minute), config.VersionedBaseDir)) + + errCh := make(chan error, 2) + for i := 0; i < 2; i++ { + go func() { + _, err := manager.Ensure(context.Background(), "cube-shim", "3.0.0", "") + errCh <- err + }() + } + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("download did not start") + } + time.Sleep(20 * time.Millisecond) + close(release) + for i := 0; i < 2; i++ { + require.NoError(t, <-errCh) + } + assert.Equal(t, int32(1), atomic.LoadInt32(&blobs)) +} + +func TestEnsure_HonorsContext(t *testing.T) { + manager, config, _ := setupTestManager(t) + started := make(chan struct{}) + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/blob") { + close(started) + <-release + writeShimTar(t, w) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + manager.SetFetcher(warehouse.NewFetcher(warehouse.NewClient(srv.URL, "node-1", "amd64", time.Minute), config.VersionedBaseDir)) + + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { + _, err := manager.Ensure(ctx, "cube-shim", "ctx", "") + errCh <- err + }() + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("download did not start") + } + cancel() + select { + case err := <-errCh: + require.Error(t, err) + assert.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("Ensure did not return after cancel") + } + close(release) + _, err := manager.Ensure(context.Background(), "cube-shim", "ctx", "") + require.NoError(t, err) +} + +func TestClient_NotFoundVsDownloadFailed(t *testing.T) { + var hits int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + assert.Equal(t, "node-1", r.Header.Get("X-Cube-Node-ID")) + switch r.URL.Query().Get("version") { + case "missing": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":"nope","code":"warehouse_not_found"}`)) + case "boom": + w.WriteHeader(http.StatusBadGateway) + _, _ = w.Write([]byte(`{"error":"upstream","code":"warehouse_download_failed"}`)) + default: + writeShimTar(t, w) + } + })) + defer srv.Close() + + c := warehouse.NewClient(srv.URL, "node-1", "amd64", time.Second) + _, err := c.DownloadBlob(context.Background(), "cube-shim", "missing") + require.Error(t, err) + assert.True(t, errors.Is(err, warehouse.ErrNotFound)) + + _, err = c.DownloadBlob(context.Background(), "cube-shim", "boom") + require.Error(t, err) + assert.True(t, errors.Is(err, warehouse.ErrDownloadFailed)) +} + +func TestInstallBlobAndScan(t *testing.T) { + base := t.TempDir() + var buf bytes.Buffer + writeShimTar(t, &buf) + require.NoError(t, warehouse.InstallBlob(context.Background(), base, templatetypes.CubeComponentCubeShim, "v1", &buf)) + assert.FileExists(t, filepath.Join(base, "cube-shim", "v1", "bin", "containerd-shim-cube-rs")) + assert.FileExists(t, filepath.Join(base, "cube-shim", "v1", "bin", "cube-runtime")) +} + +func writeShimTar(t *testing.T, w io.Writer) { + t.Helper() + gz := gzip.NewWriter(w) + defer gz.Close() + tw := tar.NewWriter(gz) + defer tw.Close() + for _, name := range []string{"bin/containerd-shim-cube-rs", "bin/cube-runtime"} { + body := []byte("x") + hdr := &tar.Header{Name: name, Mode: 0755, Size: int64(len(body))} + require.NoError(t, tw.WriteHeader(hdr)) + _, err := tw.Write(body) + require.NoError(t, err) + } +} diff --git a/Cubelet/pkg/cubelet/cubelet.go b/Cubelet/pkg/cubelet/cubelet.go index 1e2c570c1..7e078943a 100644 --- a/Cubelet/pkg/cubelet/cubelet.go +++ b/Cubelet/pkg/cubelet/cubelet.go @@ -45,6 +45,9 @@ type KubeletConfig struct { DisableCreateNode bool `toml:"disable_create_node,omitempty"` NodeStatusUpdateFrequency tomlext.Duration `toml:"node_status_update_frequency,omitempty"` + + CubeOpsAddr string `toml:"cubeops_addr,omitempty"` + CubeOpsTimeout tomlext.Duration `toml:"cubeops_timeout,omitempty"` } func DefaultCubeletConfig() *KubeletConfig { @@ -53,6 +56,7 @@ func DefaultCubeletConfig() *KubeletConfig { ResyncInterval: 10 * time.Hour, DisableCreateNode: false, NodeStatusUpdateFrequency: tomlext.FromStdTime(10 * time.Second), + CubeOpsTimeout: tomlext.FromStdTime(10 * time.Minute), } } @@ -116,6 +120,11 @@ type Cubelet struct { closeCh chan struct{} } +// StopChannel is closed when Cubelet shuts down. +func (kl *Cubelet) StopChannel() <-chan struct{} { + return kl.closeCh +} + func NewCubelet( mconfig *KubeletConfig, client *masterclient.Client, diff --git a/Cubelet/pkg/warehouse/client.go b/Cubelet/pkg/warehouse/client.go new file mode 100644 index 000000000..adac5f2de --- /dev/null +++ b/Cubelet/pkg/warehouse/client.go @@ -0,0 +1,200 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package warehouse + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +const nodeIDHeader = "X-Cube-Node-ID" + +var ( + // ErrNotFound means CubeOps has no copy of this version. + ErrNotFound = errors.New("component version not in warehouse") + // ErrDownloadFailed is a transport / 5xx / corrupt blob failure. + ErrDownloadFailed = errors.New("component version download failed") +) + +// Client talks to CubeOps /internal/warehouse with no cluster token. +type Client struct { + base string + nodeID string + arch string + http *http.Client +} + +func NewClient(base, nodeID, arch string, timeout time.Duration) *Client { + base = strings.TrimRight(strings.TrimSpace(base), "/") + if timeout <= 0 { + timeout = 10 * time.Minute + } + return &Client{ + base: base, + nodeID: nodeID, + arch: arch, + http: &http.Client{Timeout: timeout}, + } +} + +func (c *Client) enabled() bool { + return c != nil && c.base != "" +} + +// DownloadBlob GETs the version directory as tar.gz. +func (c *Client) DownloadBlob(ctx context.Context, component, version string) (io.ReadCloser, error) { + if !c.enabled() { + return nil, fmt.Errorf("%w: cubeops_addr is not configured", ErrNotFound) + } + q := url.Values{} + q.Set("arch", c.arch) + q.Set("component", component) + q.Set("version", version) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+"/internal/warehouse/blob?"+q.Encode(), nil) + if err != nil { + return nil, err + } + c.decorate(req) + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrDownloadFailed, err) + } + if resp.StatusCode == http.StatusNotFound { + defer resp.Body.Close() + code, msg := readAPIError(resp) + if code == "warehouse_not_found" { + return nil, fmt.Errorf("%w: %s", ErrNotFound, msg) + } + return nil, fmt.Errorf("%w: HTTP 404 %s", ErrDownloadFailed, msg) + } + if resp.StatusCode >= 400 { + defer resp.Body.Close() + _, msg := readAPIError(resp) + return nil, fmt.Errorf("%w: HTTP %d %s", ErrDownloadFailed, resp.StatusCode, msg) + } + return resp.Body, nil +} + +type preinstallJob struct { + ID string `json:"id"` + NodeID string `json:"nodeId"` + Arch string `json:"arch"` + Component string `json:"component"` + Version string `json:"version"` + Status string `json:"status"` +} + +func (c *Client) ListJobs(ctx context.Context) ([]preinstallJob, error) { + if !c.enabled() { + return nil, nil + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+"/internal/warehouse/jobs", nil) + if err != nil { + return nil, err + } + c.decorate(req) + resp, err := c.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + _, msg := readAPIError(resp) + return nil, fmt.Errorf("list warehouse jobs: HTTP %d %s", resp.StatusCode, msg) + } + var wrap struct { + Jobs []preinstallJob `json:"jobs"` + } + if err := json.NewDecoder(resp.Body).Decode(&wrap); err != nil { + return nil, err + } + return wrap.Jobs, nil +} + +func (c *Client) AckJob(ctx context.Context, id, status, errMsg string) error { + if !c.enabled() { + return nil + } + body, _ := json.Marshal(map[string]string{"status": status, "error": errMsg}) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+"/internal/warehouse/jobs/"+url.PathEscape(id)+"/ack", bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + c.decorate(req) + resp, err := c.http.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + _, msg := readAPIError(resp) + return fmt.Errorf("ack job: HTTP %d %s", resp.StatusCode, msg) + } + return nil +} + +// InventoryItem is one locally inventoried component version. +type InventoryItem struct { + Component string `json:"component"` + Version string `json:"version"` +} + +func (c *Client) PutInventory(ctx context.Context, items []InventoryItem) error { + if !c.enabled() { + return nil + } + if items == nil { + items = []InventoryItem{} + } + body, err := json.Marshal(map[string]any{ + "arch": c.arch, + "items": items, + }) + if err != nil { + return fmt.Errorf("report inventory: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, c.base+"/internal/warehouse/inventory", bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + c.decorate(req) + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("report inventory: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + _, msg := readAPIError(resp) + return fmt.Errorf("report inventory: HTTP %d %s", resp.StatusCode, msg) + } + return nil +} + +func (c *Client) decorate(req *http.Request) { + if c.nodeID != "" { + req.Header.Set(nodeIDHeader, c.nodeID) + } +} + +func readAPIError(resp *http.Response) (code, msg string) { + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + var wrap struct { + Error string `json:"error"` + Code string `json:"code"` + } + if json.Unmarshal(raw, &wrap) == nil { + return wrap.Code, wrap.Error + } + return "", strings.TrimSpace(string(raw)) +} diff --git a/Cubelet/pkg/warehouse/fetch_test.go b/Cubelet/pkg/warehouse/fetch_test.go new file mode 100644 index 000000000..6deda75de --- /dev/null +++ b/Cubelet/pkg/warehouse/fetch_test.go @@ -0,0 +1,150 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package warehouse + +import ( + "archive/tar" + "compress/gzip" + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/controller/runtemplate/templatetypes" +) + +func TestFetcher_ConcurrentFetchDownloadsOnce(t *testing.T) { + var blobs int32 + release := make(chan struct{}) + started := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.Contains(r.URL.Path, "/blob"): + if atomic.AddInt32(&blobs, 1) == 1 { + close(started) + } + <-release + writeShimTar(t, w) + case r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/inventory"): + w.WriteHeader(http.StatusOK) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + base := t.TempDir() + f := NewFetcher(NewClient(srv.URL, "node-1", "amd64", time.Minute), base) + + errCh := make(chan error, 2) + for i := 0; i < 2; i++ { + go func() { + errCh <- f.Fetch(context.Background(), templatetypes.CubeComponentCubeShim, "3.0.0") + }() + } + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("download did not start") + } + time.Sleep(20 * time.Millisecond) + close(release) + for i := 0; i < 2; i++ { + require.NoError(t, <-errCh) + } + assert.Equal(t, int32(1), atomic.LoadInt32(&blobs)) + assert.FileExists(t, filepath.Join(base, "cube-shim", "3.0.0", "bin", "containerd-shim-cube-rs")) +} + +func TestFetcher_SkipHTTPWhenDestDirExists(t *testing.T) { + var blobs int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/blob") { + atomic.AddInt32(&blobs, 1) + http.Error(w, "should not download", http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + base := t.TempDir() + dir := filepath.Join(base, "cube-shim", "1.0.0", "bin") + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "containerd-shim-cube-rs"), []byte("shim"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "cube-runtime"), []byte("rt"), 0o755)) + + f := NewFetcher(NewClient(srv.URL, "node-1", "amd64", time.Second), base) + require.NoError(t, f.Fetch(context.Background(), templatetypes.CubeComponentCubeShim, "1.0.0")) + assert.Equal(t, int32(0), atomic.LoadInt32(&blobs)) +} + +func TestFetcher_CallerCancelDoesNotAbortInflight(t *testing.T) { + var blobs int32 + started := make(chan struct{}) + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.Contains(r.URL.Path, "/blob"): + atomic.AddInt32(&blobs, 1) + close(started) + <-release + writeShimTar(t, w) + case r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/inventory"): + w.WriteHeader(http.StatusOK) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + base := t.TempDir() + f := NewFetcher(NewClient(srv.URL, "node-1", "amd64", time.Minute), base) + + ensureCtx, cancel := context.WithCancel(context.Background()) + ensureErr := make(chan error, 1) + preinstallErr := make(chan error, 1) + go func() { + ensureErr <- f.Fetch(ensureCtx, templatetypes.CubeComponentCubeShim, "ctx") + }() + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("download did not start") + } + go func() { + preinstallErr <- f.Fetch(context.Background(), templatetypes.CubeComponentCubeShim, "ctx") + }() + time.Sleep(20 * time.Millisecond) + cancel() + require.ErrorIs(t, <-ensureErr, context.Canceled) + + close(release) + require.NoError(t, <-preinstallErr) + assert.Equal(t, int32(1), atomic.LoadInt32(&blobs)) + assert.FileExists(t, filepath.Join(base, "cube-shim", "ctx", "bin", "containerd-shim-cube-rs")) +} + +func writeShimTar(t *testing.T, w io.Writer) { + t.Helper() + gz := gzip.NewWriter(w) + defer gz.Close() + tw := tar.NewWriter(gz) + defer tw.Close() + for _, name := range []string{"bin/containerd-shim-cube-rs", "bin/cube-runtime"} { + body := []byte("x") + hdr := &tar.Header{Name: name, Mode: 0755, Size: int64(len(body))} + require.NoError(t, tw.WriteHeader(hdr)) + _, err := tw.Write(body) + require.NoError(t, err) + } +} diff --git a/Cubelet/pkg/warehouse/install.go b/Cubelet/pkg/warehouse/install.go new file mode 100644 index 000000000..f7cdd3f66 --- /dev/null +++ b/Cubelet/pkg/warehouse/install.go @@ -0,0 +1,175 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package warehouse + +import ( + "archive/tar" + "compress/gzip" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/controller/runtemplate/templatetypes" +) + +const ( + maxEntryBytes = 8 << 30 + maxExtractedBytes = 16 << 30 +) + +func destDirExists(baseDir, name, version string) bool { + dst := templatetypes.VersionedComponentDir(baseDir, name, templatetypes.InventoryVersionKey(version)) + st, err := os.Stat(dst) + return err == nil && st.IsDir() +} + +// InstallBlob extracts a tar.gz blob into component_versions///. +func InstallBlob(ctx context.Context, baseDir, name, version string, r io.Reader) error { + version = templatetypes.InventoryVersionKey(version) + dst := templatetypes.VersionedComponentDir(baseDir, name, version) + if destDirExists(baseDir, name, version) { + return nil + } + parent := filepath.Dir(dst) + if err := os.MkdirAll(parent, 0o755); err != nil { + return err + } + tmp, err := os.MkdirTemp(parent, ".tmp-"+version+"-") + if err != nil { + return err + } + defer os.RemoveAll(tmp) + + if err := extractTarGz(ctx, r, tmp); err != nil { + return fmt.Errorf("%w: extract: %v", ErrDownloadFailed, err) + } + if err := validateTree(tmp, name); err != nil { + return fmt.Errorf("%w: %v", ErrDownloadFailed, err) + } + if err := os.Rename(tmp, dst); err != nil { + if st, statErr := os.Stat(dst); statErr == nil && st.IsDir() { + return nil + } + return fmt.Errorf("%w: install: %v", ErrDownloadFailed, err) + } + return nil +} + +func validateTree(dir, name string) error { + switch name { + case templatetypes.CubeComponentCubeShim: + for _, rel := range []string{ + templatetypes.RelativePathCubeShim, + templatetypes.RelativePathCubeRuntime, + } { + if !regularFile(filepath.Join(dir, rel)) { + return fmt.Errorf("missing %s", rel) + } + } + case templatetypes.CubeComponentCubeImage: + if !regularFile(filepath.Join(dir, templatetypes.RelativePathCubeImage)) { + return fmt.Errorf("missing %s", templatetypes.RelativePathCubeImage) + } + case templatetypes.CubeComponentCubeAgent: + if !regularFile(filepath.Join(dir, templatetypes.RelativePathCubeAgent)) { + return fmt.Errorf("missing %s", templatetypes.RelativePathCubeAgent) + } + case templatetypes.CubeComponentCubeKernel: + if !regularFile(filepath.Join(dir, "vmlinux")) && + !regularFile(filepath.Join(dir, "vmlinux-bm")) && + !regularFile(filepath.Join(dir, "vmlinux-pvm")) { + return fmt.Errorf("missing vmlinux") + } + default: + return fmt.Errorf("unsupported component %s", name) + } + return nil +} + +func regularFile(path string) bool { + st, err := os.Stat(path) + return err == nil && st.Mode().IsRegular() +} + +func extractTarGz(ctx context.Context, r io.Reader, dest string) error { + gz, err := gzip.NewReader(r) + if err != nil { + return err + } + defer gz.Close() + tr := tar.NewReader(gz) + var written int64 + for { + if err := ctx.Err(); err != nil { + return err + } + hdr, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + target, err := safeJoin(dest, hdr.Name) + if err != nil { + return err + } + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, 0o755); err != nil { + return err + } + case tar.TypeReg, tar.TypeRegA: + if hdr.Size < 0 || hdr.Size > maxEntryBytes { + return fmt.Errorf("entry %s too large", hdr.Name) + } + written += hdr.Size + if written > maxExtractedBytes { + return fmt.Errorf("archive exceeds size limit") + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)&0o777) + if err != nil { + return err + } + _, copyErr := io.CopyN(f, tr, hdr.Size) + closeErr := f.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + case tar.TypeSymlink: + if filepath.IsAbs(hdr.Linkname) || strings.Contains(hdr.Linkname, "..") { + return fmt.Errorf("refusing symlink %s", hdr.Name) + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + _ = os.Remove(target) + if err := os.Symlink(hdr.Linkname, target); err != nil { + return err + } + } + } +} + +func safeJoin(root, name string) (string, error) { + name = strings.TrimPrefix(filepath.ToSlash(name), "/") + if strings.Contains(name, "..") { + return "", fmt.Errorf("refusing path %q", name) + } + cleaned := filepath.Join(root, filepath.FromSlash(name)) + rel, err := filepath.Rel(root, cleaned) + if err != nil || strings.HasPrefix(rel, "..") { + return "", fmt.Errorf("refusing path %q", name) + } + return cleaned, nil +} diff --git a/Cubelet/pkg/warehouse/sync.go b/Cubelet/pkg/warehouse/sync.go new file mode 100644 index 000000000..75fb1a535 --- /dev/null +++ b/Cubelet/pkg/warehouse/sync.go @@ -0,0 +1,243 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package warehouse + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "time" + + "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/controller/runtemplate/templatetypes" + "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/log" + "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/utils" + "golang.org/x/sync/singleflight" +) + +const ( + inventoryPutTimeout = 15 * time.Second + jobAckTimeout = 15 * time.Second +) + +// Fetcher downloads a missing inventory version from CubeOps. +type Fetcher struct { + client *Client + base string + flight singleflight.Group + reportMu sync.Mutex + lastAck []InventoryItem + lastAckOK bool +} + +func NewFetcher(client *Client, baseDir string) *Fetcher { + if baseDir == "" { + baseDir = templatetypes.DefaultVersionedBaseDir + } + return &Fetcher{client: client, base: baseDir} +} + +func (f *Fetcher) Fetch(ctx context.Context, name, version string) error { + if f == nil || f.client == nil { + return fmt.Errorf("%w: cubeops_addr is not configured", ErrNotFound) + } + if ctx == nil { + ctx = context.Background() + } + name = strings.TrimSpace(name) + version = templatetypes.InventoryVersionKey(version) + if name == "" || version == "" { + return fmt.Errorf("component or version is empty") + } + ch := f.flight.DoChan(name+"/"+version, func() (interface{}, error) { + return nil, f.fetchOnce(context.WithoutCancel(ctx), name, version) + }) + select { + case r := <-ch: + return r.Err + case <-ctx.Done(): + select { + case r := <-ch: + return r.Err + default: + return ctx.Err() + } + } +} + +func (f *Fetcher) fetchOnce(ctx context.Context, name, version string) error { + if !destDirExists(f.base, name, version) { + body, err := f.client.DownloadBlob(ctx, name, version) + if err != nil { + return err + } + defer body.Close() + if err := InstallBlob(ctx, f.base, name, version, body); err != nil { + return err + } + } + f.reportInventoryBestEffort(ctx) + return nil +} + +func (f *Fetcher) reportInventoryBestEffort(ctx context.Context) { + if err := f.syncInventory(ctx); err != nil { + log.G(ctx).WithField("mod", "warehouse").Warnf("report inventory failed: %v", err) + } +} + +// ScanAndReport walks local inventory and replaces the CubeOps snapshot for this node. +func (f *Fetcher) ScanAndReport(ctx context.Context) { + if f == nil || f.client == nil { + return + } + f.reportInventoryBestEffort(ctx) +} + +func (f *Fetcher) syncInventory(ctx context.Context) error { + if f == nil || f.client == nil { + return nil + } + if ctx == nil { + ctx = context.Background() + } + f.reportMu.Lock() + defer f.reportMu.Unlock() + items := collectValidDirs(f.base) + if f.lastAckOK && sameInventorySet(items, f.lastAck) { + return nil + } + putCtx, cancel := context.WithTimeout(ctx, inventoryPutTimeout) + defer cancel() + if err := f.client.PutInventory(putCtx, items); err != nil { + return err + } + f.lastAck = cloneInventory(items) + f.lastAckOK = true + return nil +} + +func collectValidDirs(baseDir string) []InventoryItem { + var items []InventoryItem + for _, name := range []string{ + templatetypes.CubeComponentCubeShim, + templatetypes.CubeComponentCubeKernel, + templatetypes.CubeComponentCubeImage, + templatetypes.CubeComponentCubeAgent, + } { + compDir := filepath.Join(baseDir, name) + entries, err := os.ReadDir(compDir) + if err != nil { + continue + } + for _, e := range entries { + if !e.IsDir() || e.Name() == "" || e.Name()[0] == '.' { + continue + } + ver := templatetypes.InventoryVersionKey(e.Name()) + if ver == "" { + continue + } + if err := validateTree(filepath.Join(compDir, e.Name()), name); err != nil { + continue + } + items = append(items, InventoryItem{Component: name, Version: ver}) + } + } + return items +} + +func sameInventorySet(a, b []InventoryItem) bool { + if len(a) != len(b) { + return false + } + seen := make(map[string]struct{}, len(a)) + for _, it := range a { + seen[it.Component+"\x00"+it.Version] = struct{}{} + } + if len(seen) != len(a) { + return false + } + for _, it := range b { + if _, ok := seen[it.Component+"\x00"+it.Version]; !ok { + return false + } + } + return true +} + +func cloneInventory(items []InventoryItem) []InventoryItem { + if items == nil { + return nil + } + out := make([]InventoryItem, len(items)) + copy(out, items) + return out +} + +// RunPreinstallLoop polls CubeOps for pending jobs and installs them. +func RunPreinstallLoop(stopCh <-chan struct{}, client *Client, fetcher *Fetcher, interval time.Duration) { + if client == nil || fetcher == nil { + return + } + if interval <= 0 { + interval = 30 * time.Second + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + scanCtx, scanCancel := context.WithTimeout(context.Background(), 30*time.Second) + fetcher.ScanAndReport(scanCtx) + scanCancel() + pollOnce(client, fetcher) + select { + case <-stopCh: + return + case <-ticker.C: + } + } +} + +func pollOnce(client *Client, fetcher *Fetcher) { + listCtx, listCancel := context.WithTimeout(context.Background(), 30*time.Second) + jobs, err := client.ListJobs(listCtx) + listCancel() + if err != nil { + log.G(context.Background()).WithField("mod", "warehouse").Warnf("list preinstall jobs: %v", err) + return + } + for _, job := range jobs { + jobCtx, jobCancel := context.WithTimeout(context.Background(), 12*time.Minute) + ackJob(client, job.ID, "running", "") + if err := fetcher.Fetch(jobCtx, job.Component, job.Version); err != nil { + ackJob(client, job.ID, "failed", err.Error()) + } else { + ackJob(client, job.ID, "succeeded", "") + } + jobCancel() + } +} + +func ackJob(client *Client, id, status, errMsg string) { + ctx, cancel := context.WithTimeout(context.Background(), jobAckTimeout) + defer cancel() + if ackErr := client.AckJob(ctx, id, status, errMsg); ackErr != nil { + log.G(ctx).WithField("mod", "warehouse").Warnf("ack job %s %s: %v", id, status, ackErr) + } +} + +func NodeArch() string { + return runtime.GOARCH +} + +func NodeID() string { + id, err := utils.GetInstanceID() + if err != nil { + return "" + } + return id +} diff --git a/Cubelet/pkg/warehouse/sync_test.go b/Cubelet/pkg/warehouse/sync_test.go new file mode 100644 index 000000000..89695878d --- /dev/null +++ b/Cubelet/pkg/warehouse/sync_test.go @@ -0,0 +1,186 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package warehouse + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/controller/runtemplate/templatetypes" +) + +func TestScanAndReport_SkipUnchangedAndDropDeleted(t *testing.T) { + var puts int32 + var last atomic.Value + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut || !strings.Contains(r.URL.Path, "/inventory") { + http.NotFound(w, r) + return + } + atomic.AddInt32(&puts, 1) + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + last.Store(append([]byte(nil), body...)) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + base := t.TempDir() + writeValidShim(t, base, "vA") + writeValidShim(t, base, "vB") + + f := NewFetcher(NewClient(srv.URL, "node-1", "amd64", time.Minute), base) + f.ScanAndReport(context.Background()) + require.Equal(t, int32(1), atomic.LoadInt32(&puts)) + assert.ElementsMatch(t, []string{"vA", "vB"}, inventoryVersions(t, last.Load().([]byte), "cube-shim")) + + f.ScanAndReport(context.Background()) + assert.Equal(t, int32(1), atomic.LoadInt32(&puts), "unchanged snapshot should skip PUT") + + require.NoError(t, os.RemoveAll(filepath.Join(base, "cube-shim", "vB"))) + f.ScanAndReport(context.Background()) + require.Equal(t, int32(2), atomic.LoadInt32(&puts)) + assert.ElementsMatch(t, []string{"vA"}, inventoryVersions(t, last.Load().([]byte), "cube-shim")) +} + +func TestScanAndReport_FetchWaitsAndKeepsNewVersion(t *testing.T) { + putStarted := make(chan struct{}) + releasePut := make(chan struct{}) + var puts int32 + var last atomic.Value + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.Contains(r.URL.Path, "/blob"): + writeShimTar(t, w) + case r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/inventory"): + n := atomic.AddInt32(&puts, 1) + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + last.Store(append([]byte(nil), body...)) + if n == 1 { + close(putStarted) + <-releasePut + } + w.WriteHeader(http.StatusOK) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + base := t.TempDir() + writeValidShim(t, base, "vA") + f := NewFetcher(NewClient(srv.URL, "node-1", "amd64", time.Minute), base) + + scanDone := make(chan struct{}) + go func() { + defer close(scanDone) + f.ScanAndReport(context.Background()) + }() + select { + case <-putStarted: + case <-time.After(2 * time.Second): + t.Fatal("scan PUT did not start") + } + + fetchErr := make(chan error, 1) + go func() { + fetchErr <- f.Fetch(context.Background(), templatetypes.CubeComponentCubeShim, "vB") + }() + time.Sleep(50 * time.Millisecond) + close(releasePut) + <-scanDone + require.NoError(t, <-fetchErr) + require.GreaterOrEqual(t, atomic.LoadInt32(&puts), int32(2)) + assert.ElementsMatch(t, []string{"vA", "vB"}, inventoryVersions(t, last.Load().([]byte), "cube-shim")) +} + +func TestFetcher_InventoryPutFailureStillSucceeds(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.Contains(r.URL.Path, "/blob"): + writeShimTar(t, w) + case r.Method == http.MethodPut: + http.Error(w, "nope", http.StatusInternalServerError) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + base := t.TempDir() + f := NewFetcher(NewClient(srv.URL, "node-1", "amd64", time.Minute), base) + err := f.Fetch(context.Background(), templatetypes.CubeComponentCubeShim, "v1") + require.NoError(t, err) + assert.FileExists(t, filepath.Join(base, "cube-shim", "v1", "bin", "containerd-shim-cube-rs")) +} + +func TestAckJob_ExpiredParentContextStillSends(t *testing.T) { + var gotStatus atomic.Value + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || !strings.Contains(r.URL.Path, "/ack") { + http.NotFound(w, r) + return + } + var body struct { + Status string `json:"status"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + gotStatus.Store(body.Status) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + client := NewClient(srv.URL, "node-1", "amd64", time.Minute) + expired, cancel := context.WithCancel(context.Background()) + cancel() + require.Error(t, client.AckJob(expired, "job-1", "failed", "deadline")) + require.Nil(t, gotStatus.Load()) + + ackJob(client, "job-1", "failed", "deadline") + require.Equal(t, "failed", gotStatus.Load()) +} + +func writeValidShim(t *testing.T, base, version string) { + t.Helper() + dir := filepath.Join(base, "cube-shim", version, "bin") + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "containerd-shim-cube-rs"), []byte("shim"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "cube-runtime"), []byte("rt"), 0o755)) +} + +func inventoryVersions(t *testing.T, raw []byte, component string) []string { + t.Helper() + var wrap struct { + Items []InventoryItem `json:"items"` + } + require.NoError(t, json.Unmarshal(raw, &wrap)) + var out []string + for _, it := range wrap.Items { + if it.Component == component { + out = append(out, it.Version) + } + } + return out +} diff --git a/Cubelet/plugins/controller/cubelet_plugin.go b/Cubelet/plugins/controller/cubelet_plugin.go index 0b9bfc196..f1da538e5 100644 --- a/Cubelet/plugins/controller/cubelet_plugin.go +++ b/Cubelet/plugins/controller/cubelet_plugin.go @@ -20,6 +20,7 @@ import ( "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/log" "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/masterclient" "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/version" + "github.com/tencentcloud/CubeSandbox/Cubelet/services/cubebox" ) func init() { @@ -100,6 +101,7 @@ func registerCubelet() { if err != nil { return nil, fmt.Errorf("failed to create cubelet: %w", err) } + cubebox.StartWarehouseSync(cl.StopChannel(), cfg.CubeOpsAddr, tomlext.ToStdTime(cfg.CubeOpsTimeout)) readyHook := ic.RegisterReadiness() go func() { diff --git a/Cubelet/services/cubebox/warehouse.go b/Cubelet/services/cubebox/warehouse.go new file mode 100644 index 000000000..fdd9a8a3c --- /dev/null +++ b/Cubelet/services/cubebox/warehouse.go @@ -0,0 +1,45 @@ +// Copyright (c) 2026 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cubebox + +import ( + "context" + "strings" + "time" + + "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/log" + "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/warehouse" + CubeLog "github.com/tencentcloud/CubeSandbox/cubelog" +) + +// StartWarehouseSync wires CubeOps download into Ensure and starts inventory +// writeback + preinstall polling. No-op when cubeops_addr is unset. +func StartWarehouseSync(stopCh <-chan struct{}, addr string, timeout time.Duration) { + addr = strings.TrimSpace(addr) + if addr == "" { + return + } + if !strings.Contains(addr, "://") { + addr = "http://" + addr + } + if timeout <= 0 { + timeout = 10 * time.Minute + } + cm := getComponentManager() + client := warehouse.NewClient(addr, warehouse.NodeID(), warehouse.NodeArch(), timeout) + fetcher := warehouse.NewFetcher(client, cm.Config().VersionedBaseDir) + cm.SetFetcher(fetcher) + + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + fetcher.ScanAndReport(ctx) + }() + + log.G(context.Background()).WithFields(CubeLog.Fields{ + "mod": "warehouse", "cubeops_addr": addr, + }).Info("component warehouse download enabled") + + go warehouse.RunPreinstallLoop(stopCh, client, fetcher, 30*time.Second) +} diff --git a/configs/single-node/cubemaster.yaml b/configs/single-node/cubemaster.yaml index 76c7f971a..7b3b313f9 100644 --- a/configs/single-node/cubemaster.yaml +++ b/configs/single-node/cubemaster.yaml @@ -28,7 +28,7 @@ cubelet_conf: # Shipped default: no cluster-wide idle timeout. Set e.g. 300 in production if needed. default_timeout_insec: -1 # Create/scheduling RPC deadline only (not sandbox idle TTL). - create_timeout_insec: 300 + create_timeout_insec: 600 create_concurrent_limit: 100 destroy_concurent_limit: 100 enable_exposed_port: true diff --git a/deploy/kubernetes/chart/README.md b/deploy/kubernetes/chart/README.md index d15cb775e..1f60c78d5 100644 --- a/deploy/kubernetes/chart/README.md +++ b/deploy/kubernetes/chart/README.md @@ -242,7 +242,8 @@ The chart uses PVC-backed persistence by default so state can survive rescheduling across dedicated control nodes: ```yaml -# Optional: pin all three control-plane PVCs at once +# Optional: pin master / mysql / redis / warehouse PVCs at once. +# Default warehouse access mode is ReadWriteOnce (CBS / local-path). persistence: storageClassName: "" # empty → cluster default SC @@ -262,6 +263,10 @@ redis: enabled: true hostPath: "" storageClassName: "" +cubeOps: + persistence: + enabled: true + storageClassName: "" # ReadWriteOnce by default; RWX (NFS / CFS) for replicas > 1 ``` Set `persistence.storageClassName` (or a component-level @@ -272,7 +277,9 @@ cluster's default StorageClass, which works out of the box for most self-hosted / EKS / GKE / AKS clusters. Use `hostPath` only for single-node throwaway environments; multi-control-node deployments must use PVCs or external MySQL / Redis. `existingClaim` overrides both -`storageClassName` and `hostPath`. +`storageClassName` and `hostPath`. The warehouse volume is a +ReadWriteOnce PVC by default. Helm rejects `cubeOps.replicas > 1` unless +`accessModes` includes ReadWriteMany (NFS/CFS). Do not confuse `storageClass.*` (whether the chart **creates** a StorageClass) with `persistence.storageClassName` (which SC **name** PVCs @@ -502,6 +509,26 @@ cubeNode: - `/sandbox/` proxies to CubeProxy; static assets are unchanged; - the Service listens on port `12088`, matching one-click `WEB_UI_HOST_PORT`. +Warehouse import allow-lists and tokens are the same `CUBE_OPS_WAREHOUSE_*` env vars as one-click. Set them via `cubeOps.warehouse` (empty lists omit the env so CubeOps defaults apply): + +```yaml +cubeOps: + warehouse: + githubRepos: ["TencentCloud/CubeSandbox"] + cnbRepos: ["CubeSandbox/CubeSandbox"] + # Prefer a Secret for private-repo tokens: + githubTokenSecret: + name: my-warehouse-tokens + key: github-token + cnbTokenSecret: + name: my-warehouse-tokens + key: cnb-token +``` + +These render as `CUBE_OPS_WAREHOUSE_GITHUB_REPOS`, `CUBE_OPS_WAREHOUSE_CNB_REPOS`, `CUBE_OPS_WAREHOUSE_GITHUB_TOKEN`, and `CUBE_OPS_WAREHOUSE_CNB_TOKEN`. Disk path stays `/data/cubeops/warehouse` (`CUBE_OPS_WAREHOUSE_DIR`). + +The warehouse PVC is **ReadWriteOnce** by default (CBS / `local-path`). `cubeOps.replicas` defaults to `1` with a Recreate strategy so the volume can remount on upgrade. Set `2`+ only with `accessModes: [ReadWriteMany]` on an RWX StorageClass (NFS/CFS) — Helm rejects `replicas > 1` otherwise. + CubeAPI serves external E2B-compatible SDK clients. Expose the WebUI externally by changing `webui.service.type` or by adding your platform's ingress/load balancer configuration. diff --git a/deploy/kubernetes/chart/files/cube-master/conf.yaml b/deploy/kubernetes/chart/files/cube-master/conf.yaml index 4e43823cb..eb6a61c81 100644 --- a/deploy/kubernetes/chart/files/cube-master/conf.yaml +++ b/deploy/kubernetes/chart/files/cube-master/conf.yaml @@ -21,6 +21,7 @@ cubelet_conf: grpc_port: 9999 common_timeout_insec: 30 create_image_timeout_insec: 300 + create_timeout_insec: 600 create_concurrent_limit: 100 destroy_concurent_limit: 100 enable_exposed_port: true diff --git a/deploy/kubernetes/chart/scripts/test-ops-rwx-replicas-guard.sh b/deploy/kubernetes/chart/scripts/test-ops-rwx-replicas-guard.sh new file mode 100755 index 000000000..a569e9287 --- /dev/null +++ b/deploy/kubernetes/chart/scripts/test-ops-rwx-replicas-guard.sh @@ -0,0 +1,107 @@ +#!/bin/sh +# Guard: default warehouse PVC is ReadWriteOnce with Recreate (cannot +# multi-attach). replicas > 1 without RWX must fail render. replicas > 1 +# with RWX renders a ReadWriteMany PVC. +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)" +CHART_DIR="$(dirname "$SCRIPT_DIR")" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +COMMON_SETS="--set-string mysql.password=test --set-string mysql.rootPassword=test --set-string redis.password=test" + +expect_fail() { + name="$1" + err_file="$2" + needle="$3" + shift 3 + if helm template "$name" "$CHART_DIR" $COMMON_SETS "$@" >/dev/null 2>"$err_file"; then + echo "expected fail: $name" >&2 + exit 1 + fi + grep -qi "$needle" "$err_file" || { + echo "unexpected error for $name (wanted /$needle/):" >&2 + cat "$err_file" >&2 + exit 1 + } +} + +extract_component_doc() { + component="$1" + input="$2" + output="$3" + awk -v component="$component" ' + BEGIN { RS="\n---\n"; ORS="\n---\n" } + index($0, "app.kubernetes.io/component: " component) { print; found=1 } + END { exit found ? 0 : 1 } + ' "$input" > "$output" +} + +expect_fail ops-replicas-rwo "$TMP_DIR/rwo.err" \ + 'cubeOps.replicas > 1 requires cubeOps.persistence.accessModes to include ReadWriteMany' \ + --set cubeOps.replicas=2 + +helm template ops-rwx-replicas "$CHART_DIR" $COMMON_SETS \ + --set cubeOps.replicas=2 \ + --set cubeOps.persistence.accessModes={ReadWriteMany} \ + > "$TMP_DIR/rwx.yaml" +extract_component_doc ops "$TMP_DIR/rwx.yaml" "$TMP_DIR/ops-rwx.yaml" + +helm template ops-rwo-default "$CHART_DIR" $COMMON_SETS \ + > "$TMP_DIR/default.yaml" +extract_component_doc ops "$TMP_DIR/default.yaml" "$TMP_DIR/ops-default.yaml" + +python3 - "$TMP_DIR/ops-default.yaml" "$TMP_DIR/ops-rwx.yaml" <<'PY' +import pathlib +import re +import sys + + +def split_ops(path): + text = pathlib.Path(path).read_text() + docs = [d for d in text.split("\n---\n") if d.strip()] + deploy = None + pvc = None + for doc in docs: + body = f"\n{doc}\n" + if "\nkind: Deployment\n" in body: + deploy = doc + if "\nkind: PersistentVolumeClaim\n" in body: + pvc = doc + if deploy is None: + raise SystemExit(f"{path}: missing cube-ops Deployment") + if pvc is None: + raise SystemExit(f"{path}: missing cube-ops warehouse PVC") + return deploy, pvc + + +def strategy_type(deploy): + m = re.search(r"(?m)^ strategy:\n(?: .*\n)*? type:\s*(\S+)\s*$", deploy) + if not m: + m = re.search(r"(?m)^ strategy:\n type:\s*(\S+)\s*$", deploy) + if not m: + raise SystemExit("missing strategy.type on cube-ops") + return m.group(1) + + +default_deploy, default_pvc = split_ops(sys.argv[1]) +if strategy_type(default_deploy) != "Recreate": + raise SystemExit( + f"default cube-ops strategy.type must be Recreate, got {strategy_type(default_deploy)!r}" + ) +if "maxSurge:" in default_deploy: + raise SystemExit("default cube-ops must not render maxSurge (Recreate omits rollingUpdate)") +if "ReadWriteOnce" not in default_pvc: + raise SystemExit("default warehouse PVC must include ReadWriteOnce") +if "ReadWriteMany" in default_pvc: + raise SystemExit("default warehouse PVC must not include ReadWriteMany") +print("ok: default replicas=1 Recreate + ReadWriteOnce") + +_, rwx_pvc = split_ops(sys.argv[2]) +if "ReadWriteMany" not in rwx_pvc: + raise SystemExit("replicas=2 + RWX warehouse PVC must include ReadWriteMany") +print("ok: replicas=2 + RWX renders ReadWriteMany PVC") +PY + +echo "All cube-ops RWX replica guard tests passed" diff --git a/deploy/kubernetes/chart/templates/_helpers.tpl b/deploy/kubernetes/chart/templates/_helpers.tpl index fe552ad0b..c6eed1384 100644 --- a/deploy/kubernetes/chart/templates/_helpers.tpl +++ b/deploy/kubernetes/chart/templates/_helpers.tpl @@ -197,6 +197,21 @@ tolerations: {{- if and .Values.controlPlane.enabled (dig "enabled" true $ops) -}}true{{- else -}}false{{- end -}} {{- end -}} +{{/* +cubeOps.replicas > 1 needs a ReadWriteMany warehouse volume. Default PVC is +RWO (CBS / local-path) and cannot multi-attach; fail at render rather than +hang on volume attach. +*/}} +{{- define "cube.opsReplicasStorageGuard" -}} +{{- $replicas := .Values.cubeOps.replicas | int -}} +{{- if gt $replicas 1 }} +{{- $modes := default list (.Values.cubeOps.persistence).accessModes }} +{{- if not (has "ReadWriteMany" $modes) }} +{{- fail "cubeOps.replicas > 1 requires cubeOps.persistence.accessModes to include ReadWriteMany (NFS/CFS). Default warehouse PVC is ReadWriteOnce." }} +{{- end }} +{{- end }} +{{- end -}} + {{- define "cube.opsFQDN" -}} {{- printf "%s.%s.svc.%s" (include "cube.opsName" .) .Release.Namespace (include "cube.clusterDomain" .) -}} {{- end -}} @@ -284,8 +299,9 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 300s; - proxy_send_timeout 300s; + client_max_body_size 8g; + proxy_read_timeout 1800s; + proxy_send_timeout 1800s; proxy_pass {{ $opsUpstream }}/api/; } @@ -296,8 +312,9 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 300s; - proxy_send_timeout 300s; + client_max_body_size 8g; + proxy_read_timeout 1800s; + proxy_send_timeout 1800s; rewrite ^/cubeapi/v1/(.*)$ /api/v1/sdk/$1 break; proxy_pass {{ $opsUpstream }}; @@ -318,8 +335,9 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 300s; - proxy_send_timeout 300s; + client_max_body_size 8g; + proxy_read_timeout 1800s; + proxy_send_timeout 1800s; rewrite ^/(.*)$ /api/v1/sdk/$1 break; proxy_pass {{ $opsUpstream }}; @@ -424,6 +442,23 @@ http { {{- end -}} {{- end -}} +{{- define "cube.opsWarehousePVCName" -}} +{{- if .Values.cubeOps.persistence.existingClaim -}} +{{- .Values.cubeOps.persistence.existingClaim -}} +{{- else -}} +{{- printf "%s-ops-warehouse" (include "cube.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* Join a string or list into a comma-separated CubeOps warehouse env value. */}} +{{- define "cube.csvOrString" -}} +{{- if kindIs "string" . -}} +{{- . -}} +{{- else -}} +{{- join "," . -}} +{{- end -}} +{{- end -}} + {{- define "cube.mysqlPVCName" -}} {{- if .Values.mysql.persistence.existingClaim -}} {{- .Values.mysql.persistence.existingClaim -}} @@ -715,7 +750,7 @@ Render a Deployment strategy block. Call with the root context to use controlPlane.deploymentStrategy, or with (dict "root" $ "strategy" .Values.controlPlane.master.deploymentStrategy) for a component override. type Recreate omits rollingUpdate (required for single- -replica RWO PVC workloads such as cube-master). +replica RWO PVC workloads such as cube-master and cube-ops). */}} {{- define "cube.deploymentStrategy" -}} {{- $strategy := dict -}} @@ -936,6 +971,10 @@ Bootstrap: host mutation mounts for pvm / node-init. value: {{ .Values.hostPaths.bootstrapState | quote }} - name: CUBE_MASTER_ENDPOINT value: {{ include "cube.masterEndpoint" . | quote }} +{{- if eq (include "cube.opsEnabled" .) "true" }} +- name: CUBE_OPS_ADDR + value: {{ printf "http://%s:%v" (include "cube.opsFQDN" .) .Values.cubeOps.service.port | quote }} +{{- end }} - name: CUBE_SANDBOX_NODE_ID valueFrom: fieldRef: diff --git a/deploy/kubernetes/chart/templates/ops-pvc.yaml b/deploy/kubernetes/chart/templates/ops-pvc.yaml new file mode 100644 index 000000000..5341ff974 --- /dev/null +++ b/deploy/kubernetes/chart/templates/ops-pvc.yaml @@ -0,0 +1,19 @@ +{{- if and (eq (include "cube.opsEnabled" .) "true") .Values.cubeOps.persistence.enabled (not .Values.cubeOps.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "cube.opsWarehousePVCName" . }} + labels: + {{- include "cube.labels" . | nindent 4 }} + app.kubernetes.io/component: ops +spec: + accessModes: + {{- toYaml .Values.cubeOps.persistence.accessModes | nindent 4 }} + resources: + requests: + storage: {{ .Values.cubeOps.persistence.size | quote }} + {{- $sc := include "cube.persistenceStorageClassName" (dict "root" . "component" .Values.cubeOps.persistence) -}} + {{- if $sc }} + storageClassName: {{ $sc | quote }} + {{- end }} +{{- end }} diff --git a/deploy/kubernetes/chart/templates/ops.yaml b/deploy/kubernetes/chart/templates/ops.yaml index 95b832bf7..08d76df25 100644 --- a/deploy/kubernetes/chart/templates/ops.yaml +++ b/deploy/kubernetes/chart/templates/ops.yaml @@ -1,4 +1,5 @@ {{- if eq (include "cube.opsEnabled" .) "true" }} +{{- include "cube.opsReplicasStorageGuard" . }} {{- $opsPort := .Values.cubeOps.service.port -}} apiVersion: apps/v1 kind: Deployment @@ -9,7 +10,7 @@ metadata: app.kubernetes.io/component: ops spec: replicas: {{ .Values.cubeOps.replicas }} - {{- include "cube.deploymentStrategy" . | nindent 2 }} + {{- include "cube.deploymentStrategy" (dict "root" . "strategy" .Values.cubeOps.deploymentStrategy) | nindent 2 }} selector: matchLabels: {{- include "cube.selectorLabels" . | nindent 6 }} @@ -37,6 +38,39 @@ spec: {{- include "cube.timezoneEnv" . | nindent 12 }} - name: CUBE_OPS_BIND value: {{ .Values.cubeOps.bind | quote }} + - name: CUBE_OPS_WAREHOUSE_DIR + value: /data/cubeops/warehouse + {{- $wh := .Values.cubeOps.warehouse | default dict }} + {{- with $wh.githubRepos }} + - name: CUBE_OPS_WAREHOUSE_GITHUB_REPOS + value: {{ include "cube.csvOrString" . | quote }} + {{- end }} + {{- with $wh.cnbRepos }} + - name: CUBE_OPS_WAREHOUSE_CNB_REPOS + value: {{ include "cube.csvOrString" . | quote }} + {{- end }} + {{- $ghSecret := $wh.githubTokenSecret | default dict }} + {{- if $ghSecret.name }} + - name: CUBE_OPS_WAREHOUSE_GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: {{ $ghSecret.name | quote }} + key: {{ $ghSecret.key | default "github-token" | quote }} + {{- else if $wh.githubToken }} + - name: CUBE_OPS_WAREHOUSE_GITHUB_TOKEN + value: {{ $wh.githubToken | quote }} + {{- end }} + {{- $cnbSecret := $wh.cnbTokenSecret | default dict }} + {{- if $cnbSecret.name }} + - name: CUBE_OPS_WAREHOUSE_CNB_TOKEN + valueFrom: + secretKeyRef: + name: {{ $cnbSecret.name | quote }} + key: {{ $cnbSecret.key | default "cnb-token" | quote }} + {{- else if $wh.cnbToken }} + - name: CUBE_OPS_WAREHOUSE_CNB_TOKEN + value: {{ $wh.cnbToken | quote }} + {{- end }} - name: CUBE_MASTER_ADDR value: {{ printf "http://%s" (include "cube.masterEndpoint" .) | quote }} {{- if eq (include "cube.dbDriver" .) "postgres" }} @@ -96,6 +130,17 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} + volumeMounts: + - name: cube-ops-warehouse + mountPath: /data/cubeops/warehouse + volumes: + - name: cube-ops-warehouse + {{- if .Values.cubeOps.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ include "cube.opsWarehousePVCName" . }} + {{- else }} + emptyDir: {} + {{- end }} --- apiVersion: v1 kind: Service diff --git a/deploy/kubernetes/chart/values-tke.yaml b/deploy/kubernetes/chart/values-tke.yaml index c3c227b1c..46d3f31d0 100644 --- a/deploy/kubernetes/chart/values-tke.yaml +++ b/deploy/kubernetes/chart/values-tke.yaml @@ -7,7 +7,8 @@ # # What this file does compared to values.yaml defaults: # 1. Provisions a chart-owned CBS StorageClass named "cube-cbs-wffc" and -# pins every CubeMaster / MySQL / Redis PVC to it. +# pins CubeMaster / MySQL / Redis / CubeOps warehouse PVCs to it. +# cubeOps.replicas > 1 still needs a CFS/NFS class (ReadWriteMany). # 2. Uses WaitForFirstConsumer volume binding so multi-AZ TKE clusters # create CBS disks in the same AZ as the scheduled Pod. # 3. Exposes CubeProxy via a TKE CLB (LoadBalancer Service) and disables @@ -45,6 +46,22 @@ redis: persistence: storageClassName: cube-cbs-wffc +cubeOps: + persistence: + storageClassName: cube-cbs-wffc +# replicas > 1: switch warehouse to CFS/NFS (ReadWriteMany) and RollingUpdate: +# cubeOps: +# replicas: 2 +# deploymentStrategy: +# type: RollingUpdate +# rollingUpdate: +# maxUnavailable: 0 +# maxSurge: 1 +# persistence: +# storageClassName: +# accessModes: +# - ReadWriteMany + cubeProxy: service: type: LoadBalancer diff --git a/deploy/kubernetes/chart/values.yaml b/deploy/kubernetes/chart/values.yaml index 9c6159f3c..3e78e405a 100644 --- a/deploy/kubernetes/chart/values.yaml +++ b/deploy/kubernetes/chart/values.yaml @@ -205,8 +205,8 @@ controlPlane: enabled: true namespaceScoped: true # Native Deployment rolling strategy for stateless control-plane workloads - # (api, webui, proxy, ops, lifecycle-manager, cubemastercli). Do not apply - # maxSurge > 0 to cube-master when it mounts a ReadWriteOnce PVC. + # (api, webui, proxy, lifecycle-manager, cubemastercli). Do not apply + # maxSurge > 0 to cube-master or cube-ops when they mount a ReadWriteOnce PVC. deploymentStrategy: type: RollingUpdate rollingUpdate: @@ -337,6 +337,11 @@ cubeOps: # Required when webui.enabled=true. enabled: true replicas: 1 + # Default warehouse PVC is ReadWriteOnce (CBS / local-path). Recreate deletes + # the old Pod first so the volume can remount. replicas > 1 needs RWX + # (NFS/CFS) — Helm rejects it otherwise — and RollingUpdate for HA. + deploymentStrategy: + type: Recreate podAnnotations: {} bind: "0.0.0.0:3010" # Empty uses cubeProxy.domain for CUBE_API_SANDBOX_DOMAIN. @@ -345,6 +350,29 @@ cubeOps: type: ClusterIP port: 3010 env: [] + warehouse: + # Empty lists omit the env vars so CubeOps built-in allow-lists apply. + # A string is also accepted (comma-separated owner/repo). + githubRepos: [] + cnbRepos: [] + githubToken: "" + cnbToken: "" + githubTokenSecret: + name: "" + key: github-token + cnbTokenSecret: + name: "" + key: cnb-token + persistence: + # Warehouse blobs survive CubeOps restarts on a ReadWriteOnce PVC. + # Disable only for throwaway clusters (emptyDir). replicas > 1 must switch + # accessModes to ReadWriteMany (NFS/CFS). + enabled: true + existingClaim: "" + storageClassName: "" + size: 50Gi + accessModes: + - ReadWriteOnce probes: readiness: enabled: true diff --git a/deploy/kubernetes/images/scripts/component-entrypoint.sh b/deploy/kubernetes/images/scripts/component-entrypoint.sh index a0b776300..77acee3d4 100755 --- a/deploy/kubernetes/images/scripts/component-entrypoint.sh +++ b/deploy/kubernetes/images/scripts/component-entrypoint.sh @@ -681,6 +681,15 @@ run_cubelet() { local ep_esc ep_esc="$(sed_escape_replacement "${CUBE_MASTER_ENDPOINT}")" sed -i -e "s#^\([[:space:]]*meta_server_endpoint:[[:space:]]*\).*#\1\"${ep_esc}\"#" "${dyn}" + if [[ -n "${CUBE_OPS_ADDR:-}" ]]; then + local ops_esc + ops_esc="$(sed_escape_replacement "${CUBE_OPS_ADDR}")" + if grep -Eq '^[[:space:]]*cubeops_addr[[:space:]]*=' "${cfg}"; then + sed -i -e "s#^[[:space:]]*cubeops_addr[[:space:]]*=.*# cubeops_addr = \"${ops_esc}\"#" "${cfg}" + else + sed -i -e "/node_status_update_frequency/a\\ cubeops_addr = \"${ops_esc}\"" "${cfg}" + fi + fi configure_sandbox_dns if [[ -z "${CUBE_SANDBOX_ETH_NAME:-}" && "${CUBE_SANDBOX_AUTO_DETECT_ETH:-true}" == "true" ]]; then diff --git a/deploy/kubernetes/images/scripts/cube-node-entrypoint.sh b/deploy/kubernetes/images/scripts/cube-node-entrypoint.sh index 4f961e298..b4bbe20fb 100755 --- a/deploy/kubernetes/images/scripts/cube-node-entrypoint.sh +++ b/deploy/kubernetes/images/scripts/cube-node-entrypoint.sh @@ -204,6 +204,14 @@ sed_escape_replacement() { CUBE_MASTER_ENDPOINT_ESC="$(sed_escape_replacement "${CUBE_MASTER_ENDPOINT}")" sed -i -e "s#^\([[:space:]]*meta_server_endpoint:[[:space:]]*\).*#\1\"${CUBE_MASTER_ENDPOINT_ESC}\"#" "${CUBELET_DYNAMICCONF}" +if [[ -n "${CUBE_OPS_ADDR:-}" ]]; then + CUBE_OPS_ADDR_ESC="$(sed_escape_replacement "${CUBE_OPS_ADDR}")" + if grep -Eq '^[[:space:]]*cubeops_addr[[:space:]]*=' "${CUBELET_CONFIG}"; then + sed -i -e "s#^[[:space:]]*cubeops_addr[[:space:]]*=.*# cubeops_addr = \"${CUBE_OPS_ADDR_ESC}\"#" "${CUBELET_CONFIG}" + else + sed -i -e "/node_status_update_frequency/a\\ cubeops_addr = \"${CUBE_OPS_ADDR_ESC}\"" "${CUBELET_CONFIG}" + fi +fi configure_sandbox_dns if [[ -z "${CUBE_SANDBOX_ETH_NAME:-}" && "${CUBE_SANDBOX_AUTO_DETECT_ETH}" == "true" ]]; then diff --git a/deploy/one-click/README.md b/deploy/one-click/README.md index 0c1b9b6de..299a2c259 100644 --- a/deploy/one-click/README.md +++ b/deploy/one-click/README.md @@ -175,7 +175,7 @@ One-click does not create an extra global `configs/` layer on the target machine - `cubeproxy/` → `/usr/local/services/cubetoolbox/cubeproxy/` - `webui/` → `/usr/local/services/cubetoolbox/webui/` -`Cubelet` uses the existing `dynamicconf/conf.yaml` from the repository as-is, and its embedded network runtime reads the network plugin configuration from `Cubelet/config/config.toml` directly. `cube-api` reads environment variables directly from `.one-click.env` on startup, listening on `0.0.0.0:3000` by default and forwarding to the local `cubemaster`. MySQL/Redis are always deployed to `/usr/local/services/cubetoolbox/support` and run in Docker containers managed by dedicated systemd services on the target machine. `cube proxy` is always deployed to `/usr/local/services/cubetoolbox/cubeproxy`, built locally from the bundled build context, and managed by systemd. WebUI is deployed to `/usr/local/services/cubetoolbox/webui`, listens on `12088` by default, serves the packaged `webui/dist` directory through a standard nginx container, and proxies `/cubeapi` to CubeAPI through Docker `host-gateway` under systemd management. +`Cubelet` uses the existing `dynamicconf/conf.yaml` from the repository as-is, and its embedded network runtime reads the network plugin configuration from `Cubelet/config/config.toml` directly. `cube-api` and `cubeops` read environment variables from `.one-click.env` on startup. CubeOps warehouse knobs are `CUBE_OPS_WAREHOUSE_*` (directory, timeouts, GitHub/CNB allow-lists and tokens); there is no CubeOps YAML file in the one-click layout. `cube-api` listens on `0.0.0.0:3000` by default and forwards to the local `cubemaster`. MySQL/Redis are always deployed to `/usr/local/services/cubetoolbox/support` and run in Docker containers managed by dedicated systemd services on the target machine. `cube proxy` is always deployed to `/usr/local/services/cubetoolbox/cubeproxy`, built locally from the bundled build context, and managed by systemd. WebUI is deployed to `/usr/local/services/cubetoolbox/webui`, listens on `12088` by default, serves the packaged `webui/dist` directory through a standard nginx container, and proxies `/cubeapi` to CubeAPI through Docker `host-gateway` under systemd management. ## Target Machine Installation diff --git a/deploy/one-click/README_zh.md b/deploy/one-click/README_zh.md index 0def14894..9c12b5ca9 100644 --- a/deploy/one-click/README_zh.md +++ b/deploy/one-click/README_zh.md @@ -164,7 +164,7 @@ one-click 不会在目标机额外创建一层全局 `configs/`，而是直接 - `cubeproxy/` -> `/usr/local/services/cubetoolbox/cubeproxy/` - `webui/` -> `/usr/local/services/cubetoolbox/webui/` -其中 `Cubelet` 直接使用仓库内现成的 `dynamicconf/conf.yaml`，其内置 network runtime 直接读取 `Cubelet/config/config.toml` 中的网络插件配置；`cube-api` 则直接读取 `.one-click.env` 中的环境变量启动，默认监听 `0.0.0.0:3000` 并转发到本机 `cubemaster`。MySQL/Redis 固定部署到 `/usr/local/services/cubetoolbox/support`，以 Docker 容器运行并由专用 systemd service 管理；`cube proxy` 固定部署到 `/usr/local/services/cubetoolbox/cubeproxy`，从发布包内 build context 本地构建镜像，并由 systemd 管理。WebUI 固定部署到 `/usr/local/services/cubetoolbox/webui`，默认监听 `12088`，通过标准 nginx 容器托管发布包里的 `webui/dist`，并通过 Docker `host-gateway` 把 `/cubeapi` 反代到宿主机 CubeAPI；其生命周期同样由 systemd 托管。 +其中 `Cubelet` 直接使用仓库内现成的 `dynamicconf/conf.yaml`，其内置 network runtime 直接读取 `Cubelet/config/config.toml` 中的网络插件配置；`cube-api` 和 `cubeops` 都从 `.one-click.env` 读环境变量。CubeOps 仓库相关项是 `CUBE_OPS_WAREHOUSE_*`（目录、超时、GitHub/CNB 白名单和 token），一键安装不写 CubeOps YAML。`cube-api` 默认监听 `0.0.0.0:3000` 并转发到本机 `cubemaster`。MySQL/Redis 固定部署到 `/usr/local/services/cubetoolbox/support`，以 Docker 容器运行并由专用 systemd service 管理；`cube proxy` 固定部署到 `/usr/local/services/cubetoolbox/cubeproxy`，从发布包内 build context 本地构建镜像，并由 systemd 管理。WebUI 固定部署到 `/usr/local/services/cubetoolbox/webui`，默认监听 `12088`，通过标准 nginx 容器托管发布包里的 `webui/dist`，并通过 Docker `host-gateway` 把 `/cubeapi` 反代到宿主机 CubeAPI；其生命周期同样由 systemd 托管。 ## 目标机安装 diff --git a/deploy/one-click/env.example b/deploy/one-click/env.example index 7b5e3f391..31ac217db 100644 --- a/deploy/one-click/env.example +++ b/deploy/one-click/env.example @@ -254,6 +254,14 @@ CUBE_OPS_BIND=0.0.0.0:3010 CUBE_OPS_LOG_LEVEL=info CUBE_OPS_LOG_DIR=/data/log/CubeOps CUBE_OPS_UPSTREAM=http://host.docker.internal:3010 +# Warehouse (CUBE_OPS_WAREHOUSE_*). Uncomment to override CubeOps defaults. +# CUBE_OPS_WAREHOUSE_DIR=/data/cubeops/warehouse +# CUBE_OPS_WAREHOUSE_WRITE_TIMEOUT=30m +# CUBE_OPS_WAREHOUSE_FETCH_TIMEOUT=30m +# CUBE_OPS_WAREHOUSE_GITHUB_REPOS=TencentCloud/CubeSandbox +# CUBE_OPS_WAREHOUSE_CNB_REPOS=CubeSandbox/CubeSandbox +# CUBE_OPS_WAREHOUSE_GITHUB_TOKEN= +# CUBE_OPS_WAREHOUSE_CNB_TOKEN= # JWT_SECRET left unset → auto-generated and persisted to t_system_setting. # JWT_SECRET= JWT_ACCESS_TTL=15m diff --git a/deploy/one-click/scripts/one-click/up-compute.sh b/deploy/one-click/scripts/one-click/up-compute.sh index b80e992e8..87f1c4f93 100644 --- a/deploy/one-click/scripts/one-click/up-compute.sh +++ b/deploy/one-click/scripts/one-click/up-compute.sh @@ -30,6 +30,15 @@ sed -i \ -e "s#^\([[:space:]]*meta_server_endpoint:[[:space:]]*\).*#\1\"${CONTROL_PLANE_ADDR}\"#" \ "${CUBELET_DYNAMICCONF}" +# CubeOps warehouse: same host as CubeMaster, port 3010, unless CUBE_OPS_ADDR is set. +cp_host="${CONTROL_PLANE_ADDR%:*}" +CUBE_OPS_ADDR="${CUBE_OPS_ADDR:-http://${cp_host}:3010}" +if grep -Eq '^[[:space:]]*cubeops_addr[[:space:]]*=' "${CUBELET_CONFIG}"; then + sed -i -e "s#^[[:space:]]*cubeops_addr[[:space:]]*=.*# cubeops_addr = \"${CUBE_OPS_ADDR}\"#" "${CUBELET_CONFIG}" +else + sed -i -e "/node_status_update_frequency/a\\ cubeops_addr = \"${CUBE_OPS_ADDR}\"" "${CUBELET_CONFIG}" +fi + mkdir -p \ "${TOOLBOX_ROOT}/cube-vs/network" \ "${TOOLBOX_ROOT}/cube-snapshot" \ diff --git a/deploy/one-click/scripts/systemd/cubeops-start.sh b/deploy/one-click/scripts/systemd/cubeops-start.sh index a8f6bcffe..20e575bc1 100644 --- a/deploy/one-click/scripts/systemd/cubeops-start.sh +++ b/deploy/one-click/scripts/systemd/cubeops-start.sh @@ -29,6 +29,14 @@ export CUBE_MASTER_ADDR="${CUBE_MASTER_ADDR:-http://127.0.0.1:8089}" export JWT_ACCESS_TTL="${JWT_ACCESS_TTL:-15m}" export JWT_REFRESH_TTL="${JWT_REFRESH_TTL:-168h}" +mkdir -p "${CUBE_OPS_WAREHOUSE_DIR:-/data/cubeops/warehouse}" +export CUBE_OPS_WAREHOUSE_DIR="${CUBE_OPS_WAREHOUSE_DIR:-/data/cubeops/warehouse}" +# Optional warehouse knobs from .one-click.env (defaults live in CubeOps): +# CUBE_OPS_WAREHOUSE_WRITE_TIMEOUT +# CUBE_OPS_WAREHOUSE_FETCH_TIMEOUT +# CUBE_OPS_WAREHOUSE_GITHUB_REPOS / CUBE_OPS_WAREHOUSE_CNB_REPOS +# CUBE_OPS_WAREHOUSE_GITHUB_TOKEN / CUBE_OPS_WAREHOUSE_CNB_TOKEN + # Shared MySQL (same instance as CubeMaster, database cube_mvp). if [[ -n "${DATABASE_URL:-}" ]]; then export DATABASE_URL diff --git a/deploy/one-click/scripts/systemd/prepare-compute-role.sh b/deploy/one-click/scripts/systemd/prepare-compute-role.sh index 25f6eab91..92ff5a855 100755 --- a/deploy/one-click/scripts/systemd/prepare-compute-role.sh +++ b/deploy/one-click/scripts/systemd/prepare-compute-role.sh @@ -14,19 +14,36 @@ fi require_cmd sed +CUBELET_CONFIG="${TOOLBOX_ROOT}/Cubelet/config/config.toml" CUBELET_DYNAMICCONF="${TOOLBOX_ROOT}/Cubelet/dynamicconf/conf.yaml" +ensure_file "${CUBELET_CONFIG}" ensure_file "${CUBELET_DYNAMICCONF}" [[ -n "${CUBE_SANDBOX_NODE_IP:-}" ]] || die "CUBE_SANDBOX_NODE_IP is required for compute role" CONTROL_PLANE_ADDR="$(resolve_control_plane_cubemaster_addr)" grep -Eq "meta_server_endpoint:" "${CUBELET_DYNAMICCONF}" || die "meta_server_endpoint missing in ${CUBELET_DYNAMICCONF}" +write_cubeops_addr() { + local cp_host="${CONTROL_PLANE_ADDR%:*}" + local addr="${CUBE_OPS_ADDR:-http://${cp_host}:3010}" + if grep -Eq '^[[:space:]]*cubeops_addr[[:space:]]*=' "${CUBELET_CONFIG}"; then + sed -i -e "s#^[[:space:]]*cubeops_addr[[:space:]]*=.*# cubeops_addr = \"${addr}\"#" "${CUBELET_CONFIG}" + else + sed -i -e "/node_status_update_frequency/a\\ cubeops_addr = \"${addr}\"" "${CUBELET_CONFIG}" + fi + log "updated cubelet cubeops_addr=${addr}" +} + current_endpoint="$(sed -nE '/^[[:space:]]*meta_server_endpoint:[[:space:]]*"/{s/^[[:space:]]*meta_server_endpoint:[[:space:]]*"([^"]+)".*/\1/p;q;}' "${CUBELET_DYNAMICCONF}" 2>/dev/null || true)" if [[ "${current_endpoint}" == "${CONTROL_PLANE_ADDR}" ]]; then + # Existing nodes already have the Master endpoint; still write cubeops_addr + # so warehouse downloads work after an in-place upgrade. + write_cubeops_addr exit 0 fi sed -i \ -e "s#^\([[:space:]]*meta_server_endpoint:[[:space:]]*\).*#\1\"${CONTROL_PLANE_ADDR}\"#" \ "${CUBELET_DYNAMICCONF}" +write_cubeops_addr log "updated cubelet dynamic meta_server_endpoint=${CONTROL_PLANE_ADDR}" diff --git a/deploy/one-click/webui/nginx.conf b/deploy/one-click/webui/nginx.conf index 8b5faac0d..a39cb3b66 100644 --- a/deploy/one-click/webui/nginx.conf +++ b/deploy/one-click/webui/nginx.conf @@ -63,8 +63,9 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 300s; - proxy_send_timeout 300s; + client_max_body_size 8g; + proxy_read_timeout 1800s; + proxy_send_timeout 1800s; proxy_pass __CUBE_OPS_UPSTREAM__/api/; } @@ -78,8 +79,8 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 300s; - proxy_send_timeout 300s; + proxy_read_timeout 1800s; + proxy_send_timeout 1800s; rewrite ^/cubeapi/v1/(.*)$ /api/v1/sdk/$1 break; proxy_pass __CUBE_OPS_UPSTREAM__; @@ -106,8 +107,8 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 300s; - proxy_send_timeout 300s; + proxy_read_timeout 1800s; + proxy_send_timeout 1800s; rewrite ^/(.*)$ /api/v1/sdk/$1 break; proxy_pass __CUBE_OPS_UPSTREAM__; diff --git a/docs/.vitepress/config.mjs b/docs/.vitepress/config.mjs index 055f79b5c..67de4124c 100644 --- a/docs/.vitepress/config.mjs +++ b/docs/.vitepress/config.mjs @@ -177,7 +177,8 @@ export default withMermaid(defineConfig({ { text: 'Template Inspection & Request Preview', link: '/guide/template-inspection-and-preview' }, { text: 'HTTPS & Domain Resolution', link: '/guide/https-and-domain' }, { text: 'Network Hardening', link: '/guide/network-hardening' }, - { text: 'Authentication', link: '/guide/authentication' } + { text: 'Authentication', link: '/guide/authentication' }, + { text: 'Component multi-version', link: '/guide/component-multiversion' }, ] }, { @@ -329,7 +330,8 @@ export default withMermaid(defineConfig({ { text: '模板检查与请求预览', link: '/zh/guide/template-inspection-and-preview' }, { text: 'HTTPS 证书与域名解析', link: '/zh/guide/https-and-domain' }, { text: '网络加固', link: '/zh/guide/network-hardening' }, - { text: '鉴权', link: '/zh/guide/authentication' } + { text: '鉴权', link: '/zh/guide/authentication' }, + { text: '组件多版本', link: '/zh/guide/component-multiversion' }, ] }, { diff --git a/docs/guide/component-multiversion.md b/docs/guide/component-multiversion.md new file mode 100644 index 000000000..722cd795e --- /dev/null +++ b/docs/guide/component-multiversion.md @@ -0,0 +1,104 @@ +--- +title: Component multi-version +--- + +# Component multi-version + +A node keeps several versions of each component on disk at the same time. When creating or restoring a sandbox, the version bound to the template is used; if it is not on the node, a copy is pulled from the CubeOps warehouse. This way, after a node upgrades its components to a new version, sandboxes built with the old version still restore and old templates still schedule — the upgrade does not affect them. + +The four components: + +| Component | What it is | +| --- | --- | +| `cube-shim` | Container runtime shim and cube-runtime | +| `cube-image` | Guest root filesystem image | +| `cube-agent` | The agent inside the guest | +| `cube-kernel-scf` | Guest kernel | + +## Background: why multi-version + +Previously a node had only one copy of each component (the current toolbox). When the upgrader bumped it, the version changed. The trouble is that a template is built against a specific version. Say the node bumps `cube-image` from v1 to v2: + +- A sandbox built with v1, on restore, can only pick up v2 — the version does not match, restore fails outright. +- An old template replica built on v1 is flagged incompatible (STALE) and cannot be scheduled. +- To recover, the old template has to be rebuilt and the node re-provisioned with components. New and old templates could not coexist on one node. + +With multi-version: every version is kept. v1 is still on the node, so an old sandbox restores against v1 and an old template still schedules — upgrading to v2 does not affect it at all. New and old templates coexist on one node; upgrades are no longer disruptive. + +## How it runs + +When creating or restoring a sandbox, the flow is: + +```mermaid +flowchart TD + A["Create/restore sandbox
needs component v1.2"] --> B{"Node inventory
has v1.2?"} + B -- yes --> C["Use it"] + B -- no --> D{"cubeops_addr
configured?"} + D -- no --> E["❌ Fail: component version missing"] + D -- yes --> F["Pull from CubeOps warehouse"] + F --> G{"Warehouse has v1.2?"} + G -- no --> H["❌ Fail: not in warehouse
→ import a one-click package"] + G -- yes --> I["Download · extract · validate · write to disk"] + I --> C + C --> J["✅ Continue create"] +``` + +There are two places on the node disk: + +``` +/usr/local/services/cubetoolbox ← current toolbox: updated in place by the upgrader, what the node uses now +/data/cubelet/root/component_versions/ ← versioned inventory: versions side by side, read at create/restore +├── cube-image/ +│ ├── v1.0/ +│ └── v1.2/ +└── cube-kernel-scf/ + └── v1.2/ +``` + +In one line: **a missing version is never papered over with the current toolbox — it is fetched from the warehouse.** So the current toolbox can be upgraded freely, and replicas already bound to an old version are unaffected — this is stable restore. New templates bind all four components; only history replicas migrated from an older version may have bound only two, and they do not get stable restore. + +## What operators do + +**1. Import versions into the warehouse** (so nodes have something to download) + +On the warehouse home page, click “Import one-click package.” Only `cube-sandbox-one-click--{amd64,arm64}.tar.gz` is accepted, imported per package — one package writes several components at once; there is no “import only one.” Three sources: GitHub Release (defaults to allowing only `TencentCloud/CubeSandbox`), CNB Release (`CubeSandbox/CubeSandbox`), local tar.gz upload (max 8 GB). After submitting you do not have to watch it — go to the “Jobs” page for progress. + +**2. Stage large versions onto nodes ahead of time** (don’t let the first create stall on download) + +On a component’s detail page, click “Preinstall”, tick the nodes that don’t have it, and it downloads in the background. **Preinstall does not create a sandbox** — it only stages the version onto the node. `cube-image` and the kernel are large (GB-scale); downloading on first create will most likely exceed the 10-minute timeout — stage them first and it is fine. + +**3. Read coverage, clean up old versions** (manage disk) + +Each card on the warehouse home has a “coverage” line: green = everything covered, yellow = N nodes missing, grey “coverage unavailable” = CubeOps cannot reach CubeMaster. **The inventory is not cleaned up automatically**; old versions accumulate. Before deleting, check the “bound version” column of the compat matrix and confirm no replica still binds that version. Deleting a version in the console removes only the central copy in the warehouse — the node-local copy is not removed. + +## Troubleshooting + +| Error | What it means | What to do | +| --- | --- | --- | +| `component version missing on node` | `cubeops_addr` is not set and the node lacks the version | Set `cubeops_addr` on the node and open access to CubeOps `:3010` | +| `component version not in warehouse` | Address is set but the warehouse has no such version | Import the matching one-click package in the console | +| Download failed (network error / 5xx / validation failed) | Warehouse has it but download or extract failed | Check the Cubelet logs `mod=warehouse`; retry or re-import | +| Coverage shows “unavailable” | CubeOps cannot reach CubeMaster | Check `CUBE_MASTER_ADDR` and the network | +| Import job failed | The jobs page shows the reason: out of allow-list, missing token, source unreachable, bad package format | Fix per the message | + +## Deployment + +The “fetch on miss” part of multi-version requires the node to reach CubeOps. If it cannot, it falls back to “only versions already on the node,” and a miss fails outright. + +| Deployment | Warehouse disk | Node address | Notes | +| --- | --- | --- | --- | +| **Helm** | Auto-provisioned PVC (default 50Gi, **ReadWriteOnce**) | `CUBE_OPS_ADDR` auto-injected into compute pods | Works out of the box. Keep `replicas: 1` unless the StorageClass supports RWX (NFS/CFS). | +| **One-click install** | `/data/cubeops/warehouse` on the CubeOps host | The script writes “CubeMaster host + 3010” into it; override with `CUBE_OPS_ADDR` | The directory must be persistent and the network reachable. | +| **Terraform TKE** | ❌ No warehouse volume | ❌ No address injected | **Not supported.** If you must use Terraform, you have to attach a persistent volume, inject the address, and wire the network yourself. | + +## Configuration + +| Setting | Default | Notes | +| --- | --- | --- | +| `cubeops_addr` (Cubelet) | empty | CubeOps address, e.g. `http://:3010`. **Empty = no download**; a missing version fails outright. | +| `cubeops_timeout` (Cubelet) | `10m` | Timeout for downloading one version on the node. | +| `CUBE_OPS_WAREHOUSE_DIR` (CubeOps) | `/data/cubeops/warehouse` | Warehouse root; keep it persistent. | +| `CUBE_OPS_WAREHOUSE_GITHUB_REPOS` / `CNB_REPOS` | see allow-lists above | Import allow-lists, comma-separated, overridable. | +| `CUBE_OPS_WAREHOUSE_*_TOKEN` | empty | Only needed for private releases. | + +> `/internal/warehouse/*` called by nodes carries no JWT (identified only by `X-Cube-Node-ID`, same treatment as `/internal/meta`) — **expose it only to the compute-node network, never to the public.** Admin APIs go through `/opsapi` and carry JWT. diff --git a/docs/guide/resource-metrics.md b/docs/guide/resource-metrics.md index baca0efeb..b6406f0f6 100644 --- a/docs/guide/resource-metrics.md +++ b/docs/guide/resource-metrics.md @@ -370,16 +370,16 @@ An old `cube-agent` may lack the required cgroup v2 accounting semantics and res Run template `redo` for an image-built template. -`redo` uses the node's current guest image and `cube-agent` to rebuild a replica for the same template ID. After the task completes and compatibility returns to `OK`, newly created sandboxes can export `guest_workload` metrics. +`redo` uses the node's current guest image and `cube-agent` to rebuild a replica for the same template ID. After the task completes, newly created sandboxes can export `guest_workload` metrics. -`STALE` only means the node's current guest/agent versions differ from the versions recorded on the template. It does not block creation. You can still create sandboxes from a `READY` template; CubeSandbox looks up the recorded component versions on the node. If those versions are missing locally, creation fails until you place the matching component versions under the multi-version directory. Run template `redo` when you need a newer guest image / `cube-agent` so new sandboxes can export `guest_workload` metrics. +After a node upgrade, templates marked **Needs rebuild** in the Dashboard must be rebuilt (click **Rebuild Template**) before you create sandboxes from them. Other templates can still be used directly — creation looks up the recorded versions on the node (or downloads them from the component warehouse). Click **Rebuild Template** on an already-usable template only when you want the node's current guest image / `cube-agent` so new sandboxes can export `guest_workload` metrics. ### User snapshots and existing sandboxes - For a user snapshot created from a running sandbox, create a sandbox from a compatible new template and then create a new snapshot. - A running or paused old sandbox retains its in-memory `cube-agent` and must be deleted and recreated to complete the upgrade. -The one-click bundle writes the reviewed `cube-agent` into the guest image and records component versions and checksums. A `STALE` entry in the compat matrix is for visibility only; as long as the template is `READY`, you can still create sandboxes using the component versions recorded on that template. +The one-click bundle writes the reviewed `cube-agent` into the guest image. As long as the template is `READY` and is not marked **Needs rebuild**, you can still create sandboxes after an upgrade. At runtime, CubeShim also validates the resource metrics capability version returned by `StatsContainer`, preventing all-zero or incomplete `guest_workload` data from being accepted as valid metrics. diff --git a/docs/guide/service-management.md b/docs/guide/service-management.md index 9b8ba0354..2ba9e7900 100644 --- a/docs/guide/service-management.md +++ b/docs/guide/service-management.md @@ -135,7 +135,7 @@ Under `cubelet_conf`: | Key | Purpose | |-----|---------| | `default_timeout_insec` | Server default **sandbox idle TTL** (seconds) when the client omits `timeout`. **Unset or `<= 0` means no cluster-wide idle timeout** (sandboxes never time out from idle unless the client sets `timeout`). The repository ships `-1` for this “no default” behavior. Set a positive value (e.g. `300`) in production if you want automatic reclamation of sandboxes created without an explicit TTL. | -| `create_timeout_insec` | Create/scheduling RPC deadline only — **not** sandbox idle TTL. Defaults to `300` when unset. | +| `create_timeout_insec` | Create/scheduling RPC deadline only — **not** sandbox idle TTL. Defaults to `600` when unset. | | `common_timeout_insec` | Generic CubeMaster→Cubelet RPC timeout for non-create paths. | After changing `default_timeout_insec`, restart CubeMaster and read [Sandbox lifecycle — Operational Notes](lifecycle.md#cluster-default-idle-timeout-default_timeout_insec) for client-visible behavior. For node selection, quota, labels, scheduler scoring, or template redo after adding compute nodes, see [CubeMaster Scheduler Configuration](./cubemaster-scheduler-config.md). diff --git a/docs/guide/webui.md b/docs/guide/webui.md index b8fbcb60d..40d07c29c 100644 --- a/docs/guide/webui.md +++ b/docs/guide/webui.md @@ -39,12 +39,13 @@ Everything lives behind the 11 icons in the left rail. Hover any icon to see its | 3 | 🧩 | **Templates** | Catalog of reusable sandbox snapshots; create new ones from OCI images | | 4 | 🖥️ | **Nodes** | Fleet health: per-host CPU, memory, slot capacity | | 5 | 🧬 | **Versions** | Component version matrix across nodes (kernel, agent, guest image) | -| 6 | 🌐 | **Network** | API gateway config and per-node rate limits | -| 7 | 📈 | **Observability** | Runtime status, sandbox health, template build overview | -| 8 | 🔑 | **API Keys** | SDK API key management (JWT-based since v0.6.0) | -| 9 | 🏪 | **Template Store** | Install official preset images to bootstrap templates | -| 10 | 🤖 | **AgentHub** | Recruit and manage AI agent instances running on Cube Sandbox | -| 11 | ⚙️ | **Settings** | Theme, language, cluster info, keyboard shortcuts | +| 6 | 📦 | **Warehouse** | Import one-click packages; nodes download a missing bound version on create/restore | +| 7 | 🌐 | **Network** | API gateway config and per-node rate limits | +| 8 | 📈 | **Observability** | Runtime status, sandbox health, template build overview | +| 9 | 🔑 | **API Keys** | SDK API key management (JWT-based since v0.6.0) | +| 10 | 🏪 | **Template Store** | Install official preset images to bootstrap templates | +| 11 | 🤖 | **AgentHub** | Recruit and manage AI agent instances running on Cube Sandbox | +| 12 | ⚙️ | **Settings** | Theme, language, cluster info, keyboard shortcuts | ::: tip New user? Start with **Overview**. It shows everything important in one screen and refreshes automatically. @@ -65,7 +66,7 @@ If any number is red, click into **Nodes** to see which host is unhappy. ### 3.2 Create a sandbox 1. Click **Sandboxes** in the left rail, then **+ New sandbox** (top-right). -2. Pick a template from the grid. Prefer a `READY` template. A `STALE` badge only means the node's current component versions differ from those recorded on the template — creation is still allowed; make sure the node already has the component versions the template needs. +2. Pick a template from the grid. Prefer a `READY` template. If it says **Needs rebuild**, open the template and click **Rebuild Template** before creating a sandbox. Other `READY` templates keep working after a node upgrade. 3. (Optional) Add a few `meta` key/value pairs as labels. 4. Click **Create**. Within a couple of seconds you'll be redirected to the sandbox's detail page, where you can watch its logs stream in real time. @@ -127,6 +128,7 @@ Yes — it lives in `web/` of the repo, built with Vite + React + TypeScript + T - [Quick Start](./quickstart.md) — if you haven't installed yet, get to a running Dashboard in minutes - [Service Management](./service-management.md) — how to start/stop/restart the `cube-sandbox-webui.service` container +- [Component multi-version](./component-multiversion.md) — nodes keep multiple component versions; upgrading components does not break old templates or sandboxes - [Authentication](./authentication.md) — turn on API keys if you haven't - [HTTPS & Domain Resolution](./https-and-domain.md) — put the Dashboard behind TLS - [Architecture Overview](../architecture/overview.md) — understand how CubeAPI, CubeMaster, Cubelet fit together behind the scenes diff --git a/docs/zh/guide/component-multiversion.md b/docs/zh/guide/component-multiversion.md new file mode 100644 index 000000000..1fd95f4c4 --- /dev/null +++ b/docs/zh/guide/component-multiversion.md @@ -0,0 +1,104 @@ +--- +title: 组件多版本 +--- + +# 组件多版本 + +节点上同时留着好几个版本的组件。建沙箱、恢复沙箱都按模板绑定的版本来取；本地没有，就去 CubeOps 仓库拉一份下来。这样节点把组件升到新版后，用老版本建的沙箱照样能恢复、老模板照样能调度，不会被升级连累。 + +四个组件： + +| 组件 | 是什么 | +| --- | --- | +| `cube-shim` | 容器运行时 shim 和 cube-runtime | +| `cube-image` | 客户机根文件系统镜像 | +| `cube-agent` | 客户机里的 agent | +| `cube-kernel-scf` | 客户机内核 | + +## 背景：为什么要多版本 + +以前节点上只有一份组件（当前工具箱），升级器一升，版本就跟着变。麻烦在于：模板是按特定版本建的。举例说，节点把 `cube-image` 从 v1 升到 v2： + +- 用 v1 建好的沙箱，恢复时只能取到 v2，版本对不上，恢复直接失败。 +- 用 v1 的老模板副本被判成不兼容（STALE），没法调度。 +- 想救回来，只能把老模板重做、重新给节点装组件。新老模板没法在一台节点上并存。 + +有了多版本：每个版本各留一份。v1 还在节点上存着，老沙箱恢复照样取 v1，老模板照样调度，节点升到 v2 一点都不影响它。新老模板能在一台节点上并存，升级不再伤筋动骨。 + +## 它怎么跑 + +建沙箱、恢复沙箱的时候，走这么一条路： + +```mermaid +flowchart TD + A["建/恢复沙箱
要用组件 v1.2"] --> B{"节点版本库里
有 v1.2?"} + B -- 有 --> C["直接用"] + B -- 没有 --> D{"节点配了
cubeops_addr?"} + D -- 没配 --> E["❌ 失败:component version missing"] + D -- 配了 --> F["向 CubeOps 仓库拉"] + F --> G{"仓库里有 v1.2?"} + G -- 没有 --> H["❌ 失败:not in warehouse
→ 去导入一键包"] + G -- 有 --> I["下载 · 解压 · 校验 · 写入磁盘"] + I --> C + C --> J["✅ 继续建"] +``` + +节点磁盘上有两块地方： + +``` +/usr/local/services/cubetoolbox ← 当前工具箱:升级器就地更新,节点现在用的就是它 +/data/cubelet/root/component_versions/ ← 版本库:多版本并排,建/恢复从这里取 +├── cube-image/ +│ ├── v1.0/ +│ └── v1.2/ +└── cube-kernel-scf/ + └── v1.2/ +``` + +一句话：**缺版本不会拿当前工具箱凑合，而是去仓库下**。所以当前工具箱随便升，都不影响已经绑定老版本的副本——这就是稳定恢复（stable restore）。新模板会把四个组件的版本都绑上；只有从老版本迁过来的历史副本可能只绑了两件，不享受稳定恢复。 + +## 运维该干啥 + +**1. 把版本导进仓库**（节点才有东西可下） + +仓库首页点「导入一键包」。只认 `cube-sandbox-one-click--{amd64,arm64}.tar.gz`，按包导——一个包里几个组件一次写进去，没有「只导某一个」的说法。来源三种：GitHub Release（默认只让下 `TencentCloud/CubeSandbox`）、CNB Release（`CubeSandbox/CubeSandbox`）、本地传 tar.gz（最大 8 GB）。提交完不用盯着，去「任务」页看进度。 + +**2. 把大版本提前装到节点**（别让第一次建沙箱卡在下载） + +进组件详情页，点「预装」，勾上没装的节点，后台就会下。**预装不建沙箱**，纯粹是把版本先装过去。`cube-image`、内核这些体积大（GB 级），第一次建的时候才去下，十有八九会超过 10 分钟超时——提前装好就没事。 + +**3. 看对照、清旧版本**（管磁盘） + +仓库首页每张卡片有个「对照」：绿的 = 都齐了，黄的 = 有 N 个节点缺，灰的「对照不可用」= CubeOps 连不上 CubeMaster。**版本库不会自动清**，老版本越攒越多；删之前先看兼容矩阵的「已绑版本」列，确认没有副本还绑着这个版本。在控制台删某版本，只删仓库里的中心副本，节点上那份不删。 + +## 出问题怎么查 + +| 报错 | 啥意思 | 怎么办 | +| --- | --- | --- | +| `component version missing on node` | 没配 `cubeops_addr`，节点上又没这版本 | 给节点配上 `cubeops_addr`，打通到 CubeOps `:3010` | +| `component version not in warehouse` | 地址配了，但仓库里压根没这版本 | 去控制台导入对应的一键包 | +| 下载失败（网络错 / 5xx / 校验不过） | 仓库有，但下载或解压出了错 | 翻 Cubelet 日志 `mod=warehouse`，重试或重新导 | +| 对照栏「对照不可用」 | CubeOps 连不上 CubeMaster | 查 `CUBE_MASTER_ADDR` 和网络 | +| 导入任务失败 | 任务页会写原因：白名单外、没 token、源连不上、包格式不对 | 照着改 | + +## 部署 + +多版本里「缺了自动拉」这一部分，前提是节点连得上 CubeOps。连不上，就退回「只能用节点已有的」，缺了直接失败。 + +| 部署方式 | 仓库盘 | 节点地址 | 要点 | +| --- | --- | --- | --- | +| **Helm** | 自动建 PVC（默认 50Gi、**ReadWriteOnce**） | 自动给计算 Pod 注入 `CUBE_OPS_ADDR` | 开箱即用。除非 StorageClass 支持 RWX（NFS/CFS），否则保持 `replicas: 1`。 | +| **一键安装** | CubeOps 机器上的 `/data/cubeops/warehouse` | 脚本按「CubeMaster 所在机器 + 3010」写进去，也能用 `CUBE_OPS_ADDR` 覆盖 | 目录得持久，网络得通。 | +| **Terraform TKE** | ❌ 没挂仓库盘 | ❌ 没注入地址 | **暂不支持**。非要用 Terraform，就得自己挂持久卷、注入地址、打通网络。 | + +## 配置 + +| 配置 | 默认 | 说明 | +| --- | --- | --- | +| `cubeops_addr`（Cubelet） | 空 | CubeOps 地址，比如 `http://:3010`。**留空 = 不下**，缺版本直接失败。 | +| `cubeops_timeout`（Cubelet） | `10m` | 节点下一个版本的超时。 | +| `CUBE_OPS_WAREHOUSE_DIR`（CubeOps） | `/data/cubeops/warehouse` | 仓库根目录，务必持久化。 | +| `CUBE_OPS_WAREHOUSE_GITHUB_REPOS` / `CNB_REPOS` | 见上面白名单 | 导入白名单，逗号分隔，可覆盖。 | +| `CUBE_OPS_WAREHOUSE_*_TOKEN` | 空 | 私有 release 才用得上。 | + +> 节点调的 `/internal/warehouse/*` 不带 JWT（就靠 `X-Cube-Node-ID` 认，跟 `/internal/meta` 一个待遇），**只开给计算节点网络，别挂公网**。管理 API 走 `/opsapi`，带 JWT。 diff --git a/docs/zh/guide/resource-metrics.md b/docs/zh/guide/resource-metrics.md index 8e7e158c6..f2a72a4da 100644 --- a/docs/zh/guide/resource-metrics.md +++ b/docs/zh/guide/resource-metrics.md @@ -381,16 +381,16 @@ Prometheus 不理解 Cubelet 的统计周期语义。只有累计值实际下降 对于通过镜像构建的模板，需要执行模板 `redo`。 -`redo` 会使用节点当前的沙箱虚拟机镜像和 `cube-agent`，为同一个模板 ID 重新制作副本。任务完成并且兼容状态恢复为 `OK` 后，新创建的沙箱即可导出 `guest_workload` 指标。 +`redo` 会使用节点当前的沙箱虚拟机镜像和 `cube-agent`，为同一个模板 ID 重新制作副本。任务完成后，新创建的沙箱即可导出 `guest_workload` 指标。 -`STALE` 只表示节点当前 guest/agent 版本与模板记录的版本不一致，不阻止创建。`READY` 模板仍可用来创建沙箱；创建时会按模板记录的版本在节点本地查找对应组件。如果节点上没有这些版本，创建会失败，需要先把对应组件版本放到多版本目录。 +节点升级后，Dashboard 里标了「需重建」的模板要先点「重建模板」，再建沙箱。其它模板可以继续直接创建（会按模板记录的版本在节点本地查找，或从组件仓库下载）。只有当你需要当前节点上的 guest image / `cube-agent`、以便新沙箱导出 `guest_workload` 指标时，才需要对已可用的模板再点一次「重建模板」。 ### 用户快照和已有沙箱 - 对于从运行中沙箱创建的用户快照，应先使用兼容的新模板创建沙箱，再重新创建快照。 - 已经运行或暂停的旧沙箱仍保留内存中的旧 `cube-agent`，需要删除并重新创建才能完成升级。 -一键安装包会将经过审核的 `cube-agent` 写入沙箱虚拟机镜像，并记录组件版本和校验值。兼容矩阵里的 `STALE` 只供查看；只要模板是 `READY`，仍可按模板记录的组件版本创建沙箱。 +一键安装包会将经过审核的 `cube-agent` 写入沙箱虚拟机镜像。只要模板是 `READY` 且没有标「需重建」，升级后仍可直接创建沙箱。 运行时，CubeShim 还会校验 `StatsContainer` 返回的资源指标能力版本，避免将旧版本的 `guest_workload` 数据识别为有效指标。 diff --git a/docs/zh/guide/service-management.md b/docs/zh/guide/service-management.md index 9fe66c3f2..2d8b67d61 100644 --- a/docs/zh/guide/service-management.md +++ b/docs/zh/guide/service-management.md @@ -135,7 +135,7 @@ sudo systemctl restart cube-sandbox-.service | 配置项 | 说明 | |--------|------| | `default_timeout_insec` | 客户端**不传** `timeout` 时，集群默认的**沙箱空闲 TTL**（秒）。**未配置或 `<= 0` 表示不设集群级空闲超时**（沙箱不会因空闲被自动回收，除非客户端显式传 `timeout`）。仓库默认为 `-1`，即“无集群默认”。生产环境若需自动回收未带 TTL 的沙箱，可改为正数（如 `300`）。 | -| `create_timeout_insec` | 仅限制创建/调度 RPC 的截止时间，**不是**沙箱空闲 TTL。未配置时默认 `300`。 | +| `create_timeout_insec` | 仅限制创建/调度 RPC 的截止时间，**不是**沙箱空闲 TTL。未配置时默认 `600`。 | | `common_timeout_insec` | CubeMaster 访问 Cubelet 的通用 RPC 超时（非 create 专用）。 | 修改 `default_timeout_insec` 后需重启 CubeMaster；客户端可见语义见[沙箱生命周期 — 设计与运维要点](lifecycle.md#集群默认空闲超时default_timeout_insec)。如果要调整节点选择、quota、label、调度评分或新增计算节点后的 template redo，请参阅[CubeMaster 调度器配置参考](./cubemaster-scheduler-config.md)。 diff --git a/docs/zh/guide/webui.md b/docs/zh/guide/webui.md index d09c129fa..03999ba3d 100644 --- a/docs/zh/guide/webui.md +++ b/docs/zh/guide/webui.md @@ -35,12 +35,13 @@ Dashboard 是一个静态前端，由 **控制节点** 上的 nginx 容器托管 | 3 | 🧩 | **Templates（模板）** | 可复用的沙箱快照目录，支持从 OCI 镜像创建新模板 | | 4 | 🖥️ | **Nodes（节点）** | 集群健康：每台宿主机的 CPU、内存、可用槽位 | | 5 | 🧬 | **Versions（版本矩阵）** | 跨节点的组件版本分布（内核、agent、guest 镜像） | -| 6 | 🌐 | **Network（网络）** | API 网关配置与每节点速率限制 | -| 7 | 📈 | **Observability（可观测性）** | 运行时状态、沙箱健康、模板构建总览 | -| 8 | 🔑 | **API Keys（API 密钥）** | 存储 Dashboard 请求使用的 `X-API-Key` | -| 9 | 🏪 | **Template Store（模板商店）** | 安装官方预置镜像，一键生成模板 | -| 10 | 🤖 | **AgentHub（智能体中心）** | 在 Cube Sandbox 上招募并管理 AI 智能体实例 | -| 11 | ⚙️ | **Settings（设置）** | 主题、语言、集群信息、键盘快捷键 | +| 6 | 📦 | **Warehouse（组件仓库）** | 导入一键包；创建/恢复时节点可按需下载缺失的绑定版本 | +| 7 | 🌐 | **Network（网络）** | API 网关配置与每节点速率限制 | +| 8 | 📈 | **Observability（可观测性）** | 运行时状态、沙箱健康、模板构建总览 | +| 9 | 🔑 | **API Keys（API 密钥）** | 存储 Dashboard 请求使用的 `X-API-Key` | +| 10 | 🏪 | **Template Store（模板商店）** | 安装官方预置镜像，一键生成模板 | +| 11 | 🤖 | **AgentHub（智能体中心）** | 在 Cube Sandbox 上招募并管理 AI 智能体实例 | +| 12 | ⚙️ | **Settings（设置）** | 主题、语言、集群信息、键盘快捷键 | ::: tip 新用户？从 **Overview** 开始。 它把最重要的信息聚在同一屏上，并且会自动刷新。 @@ -61,7 +62,7 @@ Dashboard 是一个静态前端，由 **控制节点** 上的 nginx 容器托管 ### 3.2 创建一个沙箱 1. 点左侧栏的 **Sandboxes**，再点右上角 **+ New sandbox**。 -2. 在网格里挑一个模板。优先选 `READY`。出现 `STALE` 徽章只表示节点当前组件版本和模板记录的不一致——仍可创建；请先确认节点本地已有模板所需的组件版本。 +2. 在网格里挑一个模板。优先选 `READY`。如果模板标了「需重建」，先打开详情点「重建模板」，再建沙箱。其它 `READY` 模板在节点升级后仍可直接创建。 3. （可选）填几对 `meta` 键值对作为标签。 4. 点 **Create**。几秒内你就会被跳转到该沙箱的详情页，能看到日志在实时滚动。 @@ -121,6 +122,7 @@ Dashboard 对键盘很友好。最常用的三个： - [快速开始](./quickstart.md) — 如果你还没安装，几分钟到能跑的 Dashboard - [服务管理与日志](./service-management.md) — 如何启停 / 重启 `cube-sandbox-webui.service` 容器 +- [组件多版本](./component-multiversion.md) — 节点同时存多个组件版本，升级组件不影响老模板和老沙箱 - [鉴权](./authentication.md) — 还没开启 API Key？这里有完整步骤 - [HTTPS 证书与域名解析](./https-and-domain.md) — 给 Dashboard 加 TLS - [架构概览](../architecture/overview.md) — 了解 Dashboard 背后的 CubeAPI / CubeMaster / Cubelet 怎么协作 diff --git a/web/src/api/client.ts b/web/src/api/client.ts index f0b2f5ee5..d9992812e 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright (C) 2026 Tencent. All rights reserved. -import { api, ops, setTokens, clearTokens } from '@/lib/api'; +import { api, ops, setTokens, clearTokens, ApiError } from '@/lib/api'; import type { components } from './generated/schema'; export type ClusterOverviewDto = components['schemas']['ClusterOverview']; @@ -275,6 +275,112 @@ export const versionApi = { matrix: () => ops('/cluster/versions'), }; +export interface WarehouseComponentSummary { + name: string; + versionCount: number; + arches: string[]; + sizeBytes: number; + nodesMissing?: number; +} + +export interface WarehouseArtifact { + arch: string; + sizeBytes: number; + source: string; + sourceRef: string; + checksum: string; + createdAt: string; + nodesInstalled?: string[]; + nodesMissing?: string[]; +} + +export interface WarehouseVersionGroup { + version: string; + artifacts: WarehouseArtifact[]; +} + +export interface WarehouseComponentDetail { + name: string; + versions: WarehouseVersionGroup[]; +} + +export interface WarehouseImportJob { + id: string; + source: string; + sourceRef: string; + tag: string; + arch: string; + status: string; + error?: string; + bytesTotal: number; +} + +export interface WarehousePreinstallJob { + id: string; + nodeId: string; + arch: string; + component: string; + version: string; + status: string; + error?: string; +} + +export const warehouseApi = { + listComponents: () => ops<{ components: WarehouseComponentSummary[] }>('/warehouse/components'), + getComponent: (name: string) => + ops(`/warehouse/components/${encodeURIComponent(name)}`), + preinstallJobs: (params?: { + node_id?: string; + status?: string; + limit?: number; + offset?: number; + }) => ops<{ jobs: WarehousePreinstallJob[]; total: number }>('/warehouse/preinstall', { params }), + importStatus: (id: string) => ops(`/warehouse/imports/${id}`), + listImports: (params?: { limit?: number; offset?: number }) => + ops<{ jobs: WarehouseImportJob[]; total: number }>('/warehouse/imports', { params }), + createImport: (body: { + source: 'github' | 'cnb' | 'upload'; + repo?: string; + tag?: string; + uploadId?: string; + arch: string[]; + }) => + ops<{ jobs: WarehouseImportJob[] }>('/warehouse/imports', { + method: 'POST', + body: JSON.stringify(body), + }), + preinstall: (body: { nodeIds: string[]; arch: string; component: string; version: string }) => + ops<{ jobs: WarehousePreinstallJob[] }>('/warehouse/preinstall', { + method: 'POST', + body: JSON.stringify(body), + }), + deleteVersion: (component: string, version: string, arch: string) => + ops( + `/warehouse/components/${encodeURIComponent(component)}/versions/${encodeURIComponent(version)}`, + { method: 'DELETE', params: { arch } }, + ), + upload: async (file: File): Promise<{ uploadId: string; filename: string }> => { + const fd = new FormData(); + fd.append('file', file); + const token = localStorage.getItem('cube.accessToken') ?? ''; + const resp = await fetch('/opsapi/v1/warehouse/uploads', { + method: 'POST', + headers: token ? { Authorization: `Bearer ${token}` } : {}, + body: fd, + }); + const text = await resp.text(); + const body = text ? JSON.parse(text) : undefined; + if (!resp.ok) { + const msg = + body && typeof body === 'object' && 'error' in body + ? String((body as { error: string }).error) + : resp.statusText; + throw new ApiError(resp.status, msg, body); + } + return body as { uploadId: string; filename: string }; + }, +}; + export const clusterApi = { overview: () => ops('/cluster/overview'), nodes: () => ops('/nodes').then((items) => items.map(mapNode)), diff --git a/web/src/components/CommandPalette.tsx b/web/src/components/CommandPalette.tsx index da3ce8f32..af515c1fe 100644 --- a/web/src/components/CommandPalette.tsx +++ b/web/src/components/CommandPalette.tsx @@ -4,7 +4,16 @@ import { Command } from 'cmdk'; import { useNavigate } from 'react-router-dom'; import { useTranslation } from 'react-i18next'; -import { Boxes, Package, Server, LayoutDashboard, Activity, Settings, Plus } from 'lucide-react'; +import { + Boxes, + Package, + Server, + LayoutDashboard, + Activity, + Settings, + Plus, + Archive, +} from 'lucide-react'; import { useCommandPaletteStore } from '@/store/ui'; export function CommandPalette() { @@ -65,6 +74,11 @@ export function CommandPalette() { label={tNav('nodes')} onSelect={() => go('/nodes')} /> + } + label={tNav('warehouse')} + onSelect={() => go('/warehouse')} + /> } label={tNav('observability')} diff --git a/web/src/components/Rail.tsx b/web/src/components/Rail.tsx index 1d4ee43b5..56899668c 100644 --- a/web/src/components/Rail.tsx +++ b/web/src/components/Rail.tsx @@ -14,6 +14,7 @@ import { Settings, Store, Layers, + Archive, Github, } from 'lucide-react'; import { cn } from '@/lib/utils'; @@ -25,6 +26,7 @@ const NAV_ITEMS = [ { to: '/templates', icon: Package, key: 'templates' }, { to: '/nodes', icon: Server, key: 'nodes' }, { to: '/versions', icon: Layers, key: 'versions' }, + { to: '/warehouse', icon: Archive, key: 'warehouse' }, { to: '/network', icon: Network, key: 'network' }, { to: '/observability', icon: Activity, key: 'observability' }, { to: '/store', icon: Store, key: 'store' }, diff --git a/web/src/components/ui/pagination.tsx b/web/src/components/ui/pagination.tsx new file mode 100644 index 000000000..6739beba0 --- /dev/null +++ b/web/src/components/ui/pagination.tsx @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Tencent. All rights reserved. + +import { useEffect, useRef, useState, type ReactNode, type RefObject } from 'react'; +import { ChevronLeft, ChevronRight } from 'lucide-react'; +import { cn } from '@/lib/utils'; + +export type PaginationItem = number | 'ellipsis'; + +export function pageWindow(page: number, last: number): PaginationItem[] { + if (last <= 7) { + return Array.from({ length: last }, (_, i) => i + 1); + } + if (page <= 4) { + return [1, 2, 3, 4, 5, 'ellipsis', last]; + } + if (page >= last - 3) { + return [1, 'ellipsis', last - 4, last - 3, last - 2, last - 1, last]; + } + return [1, 'ellipsis', page - 1, page, page + 1, 'ellipsis', last]; +} + +type PaginationProps = { + page: number; + pageSize: number; + total: number; + onPage: (page: number) => void; + totalLabel: string; + prevLabel: string; + nextLabel: string; + jumpLabel: string; + jumpUnit?: string; +}; + +export function Pagination({ + page, + pageSize, + total, + onPage, + totalLabel, + prevLabel, + nextLabel, + jumpLabel, + jumpUnit = '', +}: PaginationProps) { + const jumpRef = useRef(null); + + if (total <= 0) { + return null; + } + + const last = Math.max(1, Math.ceil(total / pageSize)); + const current = Math.min(Math.max(1, page), last); + + return ( +
+

{totalLabel}

+ {last > 1 ? ( +
+ + +
+ ) : null} +
+ ); +} + +function parseJump(raw: string, last: number): number | null { + const n = Number.parseInt(raw, 10); + if (!Number.isInteger(n)) { + return null; + } + return Math.min(Math.max(1, n), last); +} + +function JumpToPage({ + inputRef, + current, + last, + onPage, + jumpLabel, + jumpUnit, +}: { + inputRef: RefObject; + current: number; + last: number; + onPage: (page: number) => void; + jumpLabel: string; + jumpUnit: string; +}) { + const [draft, setDraft] = useState(String(current)); + + useEffect(() => { + setDraft(String(current)); + }, [current]); + + const commit = () => { + const next = parseJump(draft, last); + if (next == null) { + setDraft(String(current)); + return; + } + setDraft(String(next)); + if (next !== current) { + onPage(next); + } + }; + + return ( + + ); +} + +function PageItem({ + item, + current, + onPage, + jumpRef, + jumpLabel, +}: { + item: PaginationItem; + current: number; + onPage: (page: number) => void; + jumpRef: RefObject; + jumpLabel: string; +}) { + if (item === 'ellipsis') { + return ( + + ); + } + + const active = item === current; + return ( + + ); +} + +function PagerIconButton({ + label, + disabled, + onClick, + children, +}: { + label: string; + disabled: boolean; + onClick: () => void; + children: ReactNode; +}) { + return ( + + ); +} diff --git a/web/src/components/warehouse/ImportTab.tsx b/web/src/components/warehouse/ImportTab.tsx new file mode 100644 index 000000000..3acb733b5 --- /dev/null +++ b/web/src/components/warehouse/ImportTab.tsx @@ -0,0 +1,217 @@ +import { useState } from 'react'; +import { useNavigate } from 'react-router-dom'; +import { useTranslation } from 'react-i18next'; +import { Upload, CheckCircle2, RefreshCw, Github, Cloud, HardDriveUpload } from 'lucide-react'; +import { warehouseApi } from '@/api/client'; +import { ApiError } from '@/lib/api'; +import { Button } from '@/components/ui/button'; +import { Card } from '@/components/ui/card'; +import { Input } from '@/components/ui/input'; +import { showToast } from '@/components/ui/ToastProvider'; +import { cn } from '@/lib/utils'; + +type ImportSource = 'github' | 'cnb' | 'upload'; + +export function ImportTab({ onDone, onCancel }: { onDone: () => void; onCancel: () => void }) { + const { t } = useTranslation('warehouse'); + const navigate = useNavigate(); + const [source, setSource] = useState('github'); + const [repo, setRepo] = useState('TencentCloud/CubeSandbox'); + const [tag, setTag] = useState(''); + const [arch, setArch] = useState(['amd64']); + const [file, setFile] = useState(null); + const [busy, setBusy] = useState(false); + + const toggleArch = (a: string) => { + setArch((cur) => (cur.includes(a) ? cur.filter((x) => x !== a) : [...cur, a])); + }; + + const submit = async () => { + setBusy(true); + try { + let uploadId: string | undefined; + if (source === 'upload') { + if (!file) throw new Error(t('uploadFile')); + showToast(t('uploading')); + const up = await warehouseApi.upload(file); + uploadId = up.uploadId; + } + await warehouseApi.createImport({ + source, + repo: source === 'upload' ? undefined : repo, + tag: source === 'upload' ? undefined : tag, + uploadId, + arch, + }); + showToast(t('importSubmitted'), 'success'); + onDone(); + navigate('/warehouse/jobs?tab=import'); + } catch (err) { + showToast(err instanceof ApiError ? err.message : String(err), 'warn'); + } finally { + setBusy(false); + } + }; + + const sources: { id: ImportSource; icon: React.ElementType; label: string; desc: string }[] = [ + { id: 'github', icon: Github, label: 'GitHub Release', desc: 'Pull from GitHub' }, + { id: 'cnb', icon: Cloud, label: 'CNB Release', desc: 'Pull from CNB' }, + { id: 'upload', icon: HardDriveUpload, label: 'Local File', desc: 'Upload .tar.gz' }, + ]; + + return ( + +
+

{t('import')}

+

+ Import a one-click component package into the warehouse +

+
+ +
+
+ +
+ {sources.map((s) => { + const Icon = s.icon; + const isActive = source === s.id; + return ( + + ); + })} +
+
+ +
+ {source !== 'upload' && ( +
+ + +
+ )} + + {source === 'upload' && ( + + )} + +
+ +
+ {['amd64', 'arm64'].map((a) => ( + + ))} +
+ {arch.length === 0 && ( +

+ Please select at least one architecture. +

+ )} +
+
+
+ +
+ + +
+ + ); +} diff --git a/web/src/i18n/resources.ts b/web/src/i18n/resources.ts index b45c89f7c..001de070d 100644 --- a/web/src/i18n/resources.ts +++ b/web/src/i18n/resources.ts @@ -22,6 +22,7 @@ import enObservability from '@/locales/en/observability.json'; import enStore from '@/locales/en/store.json'; import enAgentHub from '@/locales/en/agentHub.json'; import enAuth from '@/locales/en/auth.json'; +import enWarehouse from '@/locales/en/warehouse.json'; import zhCommon from '@/locales/zh/common.json'; import zhNav from '@/locales/zh/nav.json'; @@ -44,6 +45,7 @@ import zhObservability from '@/locales/zh/observability.json'; import zhStore from '@/locales/zh/store.json'; import zhAgentHub from '@/locales/zh/agentHub.json'; import zhAuth from '@/locales/zh/auth.json'; +import zhWarehouse from '@/locales/zh/warehouse.json'; export const resources = { en: { @@ -68,6 +70,7 @@ export const resources = { store: enStore, agentHub: enAgentHub, auth: enAuth, + warehouse: enWarehouse, }, zh: { common: zhCommon, @@ -91,6 +94,7 @@ export const resources = { store: zhStore, agentHub: zhAgentHub, auth: zhAuth, + warehouse: zhWarehouse, }, } as const; diff --git a/web/src/lib/utils.ts b/web/src/lib/utils.tsx similarity index 57% rename from web/src/lib/utils.ts rename to web/src/lib/utils.tsx index b87cde9f7..6823ca0dd 100644 --- a/web/src/lib/utils.ts +++ b/web/src/lib/utils.tsx @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright (C) 2026 Tencent. All rights reserved. +import React from 'react'; import { clsx, type ClassValue } from 'clsx'; import { twMerge } from 'tailwind-merge'; @@ -8,12 +9,72 @@ export function cn(...inputs: ClassValue[]) { return twMerge(clsx(inputs)); } +export function getStatusBadge(status: string, t: any, error?: string) { + switch (status) { + case 'pending': + return ( + + + {t('jobPending')} + + ); + case 'running': + return ( + + + {t('jobRunning')} + + ); + case 'succeeded': + return ( + + + {t('jobSucceeded')} + + ); + case 'failed': + return ( +
+ + + {t('jobFailed')} + + {error && ( + {error} + )} +
+ ); + case 'cancelled': + return ( + + + {t('jobCancelled')} + + ); + default: + return ( + + + {status} + + ); + } +} + export function formatBytes(mib: number | undefined | null): string { if (mib == null) return '—'; if (mib < 1024) return `${mib} MiB`; return `${(mib / 1024).toFixed(1)} GiB`; } +export function formatArtifactBytes(n: number): string { + if (!n) return '—'; + if (n < 1024) return `${n} B`; + if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KiB`; + if (n < 1024 * 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)} MiB`; + return `${(n / 1024 / 1024 / 1024).toFixed(2)} GiB`; +} + export function formatRelative(ts?: string | number | null, locale?: string): string { if (!ts) return '—'; const d = new Date(ts); diff --git a/web/src/locales/en/nav.json b/web/src/locales/en/nav.json index 11628d2ca..599004984 100644 --- a/web/src/locales/en/nav.json +++ b/web/src/locales/en/nav.json @@ -8,5 +8,6 @@ "observability": "Observability", "settings": "Settings", "store": "Template Store", - "agentHub": "AgentHub" + "agentHub": "AgentHub", + "warehouse": "Warehouse" } diff --git a/web/src/locales/en/sandboxNew.json b/web/src/locales/en/sandboxNew.json index a4d240244..890f4235f 100644 --- a/web/src/locales/en/sandboxNew.json +++ b/web/src/locales/en/sandboxNew.json @@ -23,8 +23,9 @@ "addMeta": "Add entry" }, "compat": { - "stale": "Version drift", - "staleHelp": "The node's current component versions differ from those recorded on this template. You can still create a sandbox; make sure the node already has the versions this template needs." + "unpinned": "Needs rebuild", + "unpinnedHelp": "After the node upgrade, creating a sandbox from this template may fail. Open the template and click Rebuild Template first.", + "openTemplate": "Open template" }, "actions": { "cancel": "Cancel", diff --git a/web/src/locales/en/templateDetail.json b/web/src/locales/en/templateDetail.json index 57ca577b7..7108b617a 100644 --- a/web/src/locales/en/templateDetail.json +++ b/web/src/locales/en/templateDetail.json @@ -10,7 +10,7 @@ "replicas": "Replicas", "replicasDesc": "Snapshot replicas of this template across nodes.", "compat": "Compatibility", - "compatDesc": "Component versions bound to template replicas compared with the current node environment.", + "compatDesc": "Whether this template can still create sandboxes after a node upgrade. If it needs a rebuild, click Rebuild Template.", "danger": "Delete Template", "dangerDesc": "This action cannot be undone. All replicas of this template will also be deleted." }, @@ -72,15 +72,17 @@ "blocked": "Blocked" }, "compat": { - "staleTitle": "This template needs rebuilding for the current node environment", - "staleDesc": "{{count}} replica has guest-image or cube-agent versions that differ from the current node. The template build status can still be ready, but these replicas cannot create new sandboxes.", + "staleTitle": "Rebuild this template", + "staleDesc": "{{count}} copies may fail to create sandboxes after the node upgrade. Click Rebuild Template to remake them on the current node.", + "driftTitle": "The node environment has been updated", + "driftDesc": "You can still create sandboxes. Click Rebuild Template only if you want the node's current environment.", "loading": "Loading compatibility information…", "empty": "No compatibility information available.", "status": { - "OK": "OK", - "STALE": "Needs rebuild", - "UNKNOWN": "Unknown", - "MISSING": "Missing" + "OK": "Ready", + "STALE": "Ready", + "UNKNOWN": "Needs rebuild", + "MISSING": "Not on this node" }, "components": { "guestImage": "guest-image", diff --git a/web/src/locales/en/templates.json b/web/src/locales/en/templates.json index c1bfecfab..f23b81b39 100644 --- a/web/src/locales/en/templates.json +++ b/web/src/locales/en/templates.json @@ -7,22 +7,23 @@ "compat": "Compatibility" }, "compat": { - "banner": "{{templates}} templates have {{replicas}} stale replicas after node environment changes and need rebuild.", - "view": "View compatibility", - "adoptBaseline": "Adopt current baseline", - "adoptConfirm": "Use the current node versions as the new baseline for UNKNOWN replicas of this template? Only continue if you have verified the template works in the current environment.", + "banner": "{{count}} templates need to be rebuilt. After the node upgrade, creating sandboxes from them may fail. Open a template and click Rebuild Template.", + "view": "Rebuild now", + "openRebuild": "Open and rebuild", + "driftHint": "You can still create sandboxes. Click Rebuild Template only if you want the node's current environment.", "status": { - "OK": "OK", - "STALE": "Needs rebuild", - "UNKNOWN": "Unknown", - "MISSING": "Missing" + "OK": "Ready", + "STALE": "Ready", + "UNKNOWN": "Needs rebuild", + "MISSING": "Not on this node" }, "kpi": { - "staleTemplates": "Stale templates", - "staleReplicas": "Stale replicas", - "affectedNodes": "Affected nodes", - "missingReplicas": "Missing replicas", - "unknownReplicas": "Unknown replicas" + "staleTemplates": "Needs rebuild", + "staleReplicas": "Copies to rebuild", + "affectedNodes": "Nodes involved", + "missingReplicas": "Missing copies", + "unknownReplicas": "Copies to rebuild", + "driftReplicas": "Newer node environment (rebuild optional)" } }, "col": { diff --git a/web/src/locales/en/warehouse.json b/web/src/locales/en/warehouse.json new file mode 100644 index 000000000..cc41bbce2 --- /dev/null +++ b/web/src/locales/en/warehouse.json @@ -0,0 +1,78 @@ +{ + "title": "Component warehouse", + "subtitle": "Manage global version inventory for sandbox components. Missing versions are automatically downloaded during sandbox creation or restoration, or you can manually dispatch preinstall tasks here.", + "empty": "Warehouse is empty.", + "emptyHint": "Import a one-click package from GitHub, CNB, or a local upload.", + "error": "Failed to load the warehouse.", + "retry": "Retry", + "list": "Inventory", + "coverage": "Coverage", + "jobs": "Jobs", + "jobsSubtitle": "Track one-click package imports and node preinstall progress.", + "jobsShow": "Show jobs", + "jobsHide": "Hide jobs", + "jobsEmpty": "No jobs.", + "importJobs": "Import jobs", + "preinstallJobs": "Preinstall jobs", + "noJobs": "No preinstall jobs", + "noJobsDesc": "There are currently no preinstall tasks running or in history.", + "noImportJobs": "No import jobs", + "noImportJobsDesc": "Import progress will appear here after you start an import.", + "pagePrev": "Previous", + "pageNext": "Next", + "pageTotal": "{{total}} items", + "pageJump": "Go to", + "pageJumpUnit": "", + "import": "Import package", + "delete": "Delete warehouse copy", + "deleteConfirm": "Delete {{component}} {{version}} ({{arch}}) from the warehouse? Existing local copies on nodes will not be deleted.", + "preinstall": "Preinstall", + "preinstallHint": "After dispatching, the target node will download this component version in the background. This action does not create a sandbox.", + "source.github": "GitHub Release", + "source.cnb": "CNB Release", + "source.upload": "Upload tar.gz", + "repo": "Repository", + "tag": "Release tag", + "arch": "Architecture", + "uploadFile": "One-click package", + "submit": "Start import", + "submitting": "Submitting…", + "uploading": "Uploading…", + "imported": "Import finished", + "importing": "Importing package…", + "importSubmitted": "Import started. Track progress on the jobs page.", + "importFailed": "Import failed", + "importTimeout": "Import timed out. Check the warehouse list later.", + "component": "Component", + "version": "Version", + "versionCount": "Versions", + "size": "Size", + "source": "Source", + "installed": "Installed", + "missing": "Not on node", + "status": "Status", + "node": "Node", + "noMissing": "All known nodes already have this version.", + "selectNodes": "Select nodes", + "cancel": "Cancel", + "refresh": "Refresh", + "jobPending": "Pending", + "jobRunning": "Running", + "jobSucceeded": "Succeeded", + "jobFailed": "Failed", + "jobCancelled": "Cancelled", + "timeoutNote": "Sandbox creation/restoration will wait up to 10 minutes for component downloads. If cubeops_addr is unset on the node, missing components will cause an immediate failure.", + "coverageUnavailable": "Coverage unavailable", + "noArtifacts": "Not imported yet", + "allCovered": "All known nodes covered", + "nodesMissingCount": "{{count}} nodes missing", + "componentEmpty": "This component has no artifacts yet.", + "componentEmptyHint": "You can return to the warehouse overview to import a one-click package.", + "backToWarehouse": "Back to warehouse", + "about": { + "cube-shim": "Container runtime shim and cube-runtime.", + "cube-image": "Guest root filesystem image.", + "cube-agent": "In-guest agent.", + "cube-kernel-scf": "Guest kernel." + } +} diff --git a/web/src/locales/zh/nav.json b/web/src/locales/zh/nav.json index d0ca893ad..df5f9d2f5 100644 --- a/web/src/locales/zh/nav.json +++ b/web/src/locales/zh/nav.json @@ -8,5 +8,6 @@ "observability": "可观测性", "settings": "设置", "store": "模板市场", - "agentHub": "数字助手" + "agentHub": "数字助手", + "warehouse": "组件仓库" } diff --git a/web/src/locales/zh/sandboxNew.json b/web/src/locales/zh/sandboxNew.json index 013f2d5da..637913c01 100644 --- a/web/src/locales/zh/sandboxNew.json +++ b/web/src/locales/zh/sandboxNew.json @@ -23,8 +23,9 @@ "addMeta": "添加键值对" }, "compat": { - "stale": "版本漂移", - "staleHelp": "节点当前组件版本与该模板记录的版本不一致。仍可创建；请确认节点本地已有模板所需的组件版本。" + "unpinned": "需重建", + "unpinnedHelp": "节点升级后，用这个模板新建沙箱可能失败。请先打开模板，点击「重建模板」。", + "openTemplate": "打开模板" }, "actions": { "cancel": "取消", diff --git a/web/src/locales/zh/templateDetail.json b/web/src/locales/zh/templateDetail.json index 33d457608..3266ccd25 100644 --- a/web/src/locales/zh/templateDetail.json +++ b/web/src/locales/zh/templateDetail.json @@ -10,7 +10,7 @@ "replicas": "Replicas", "replicasDesc": "该模板在各节点上的快照副本状态。", "compat": "兼容性", - "compatDesc": "模板副本绑定的组件版本与当前节点环境的对比。", + "compatDesc": "节点升级后，这个模板还能不能直接用来建沙箱。需要重建时，点「重建模板」。", "danger": "删除模板", "dangerDesc": "删除后无法恢复，该模板下所有 replica 将一并删除。" }, @@ -72,15 +72,17 @@ "blocked": "禁止" }, "compat": { - "staleTitle": "该模板在当前节点环境下需要重建", - "staleDesc": "{{count}} 个副本绑定的 guest-image 或 cube-agent 版本与当前节点不一致。模板构建状态仍可为就绪，但这些副本不可用于创建新沙箱。", + "staleTitle": "需要重建此模板", + "staleDesc": "{{count}} 个副本在节点升级后可能无法正常创建沙箱。请点击「重建模板」，按当前节点重新制作。", + "driftTitle": "节点环境已更新", + "driftDesc": "仍可直接创建沙箱。如果想用节点上的新环境，再点「重建模板」。", "loading": "正在读取兼容性信息…", "empty": "暂无兼容性信息。", "status": { - "OK": "正常", - "STALE": "需重建", - "UNKNOWN": "未知", - "MISSING": "缺失" + "OK": "可用", + "STALE": "可用", + "UNKNOWN": "需重建", + "MISSING": "此节点没有副本" }, "components": { "guestImage": "guest-image", diff --git a/web/src/locales/zh/templates.json b/web/src/locales/zh/templates.json index d9d9befca..2fbc3c767 100644 --- a/web/src/locales/zh/templates.json +++ b/web/src/locales/zh/templates.json @@ -7,22 +7,23 @@ "compat": "兼容性" }, "compat": { - "banner": "{{templates}} 个模板的 {{replicas}} 个副本已因节点环境变化失效，需要重建。", - "view": "查看兼容性", - "adoptBaseline": "采用当前为基线", - "adoptConfirm": "确认将当前节点版本作为该模板 UNKNOWN 副本的新基线？这只适用于你已确认模板可在当前环境运行的场景。", + "banner": "{{count}} 个模板需要重建。节点升级后，用它们新建沙箱可能失败。请打开模板，点击「重建模板」。", + "view": "去重建", + "openRebuild": "打开并重建", + "driftHint": "仍可直接创建沙箱。如果想用节点上的新环境，再点「重建模板」。", "status": { - "OK": "正常", - "STALE": "需重建", - "UNKNOWN": "未知", - "MISSING": "缺失" + "OK": "可用", + "STALE": "可用", + "UNKNOWN": "需重建", + "MISSING": "此节点没有副本" }, "kpi": { - "staleTemplates": "失效模板", - "staleReplicas": "失效副本", - "affectedNodes": "受影响节点", - "missingReplicas": "缺失副本", - "unknownReplicas": "未知副本" + "staleTemplates": "需重建的模板", + "staleReplicas": "需重建的副本", + "affectedNodes": "涉及节点", + "missingReplicas": "缺少副本", + "unknownReplicas": "需重建的副本", + "driftReplicas": "环境已更新（可选重建）" } }, "col": { diff --git a/web/src/locales/zh/warehouse.json b/web/src/locales/zh/warehouse.json new file mode 100644 index 000000000..5940e0225 --- /dev/null +++ b/web/src/locales/zh/warehouse.json @@ -0,0 +1,78 @@ +{ + "title": "组件仓库", + "subtitle": "管理沙箱组件的全局版本库存。创建或恢复沙箱时，若节点缺失所需版本会自动下载；您也可以在此处手动下发预装任务。", + "empty": "仓库是空的。", + "emptyHint": "从 GitHub、CNB 或本地上传导入一份一键包。", + "error": "加载组件仓库失败。", + "retry": "重试", + "list": "清单", + "coverage": "对照", + "jobs": "任务", + "jobsSubtitle": "查看一键包导入和节点预装的进度。", + "jobsShow": "显示任务", + "jobsHide": "隐藏任务", + "jobsEmpty": "没有任务。", + "importJobs": "导入任务", + "preinstallJobs": "预装任务", + "noJobs": "暂无预装任务", + "noJobsDesc": "当前没有正在进行或历史预装任务记录。", + "noImportJobs": "暂无导入任务", + "noImportJobsDesc": "导入一键包后，进度会显示在这里。", + "pagePrev": "上一页", + "pageNext": "下一页", + "pageTotal": "共 {{total}} 条", + "pageJump": "跳至", + "pageJumpUnit": "页", + "import": "导入一键包", + "delete": "删除仓库副本", + "deleteConfirm": "确定从仓库中删除 {{component}} {{version}} ({{arch}}) 吗？节点上已存在的本地副本不会被删除。", + "preinstall": "预装", + "preinstallHint": "下发任务后，目标节点将在后台自动下载该版本组件。此操作不会创建沙箱实例。", + "source.github": "GitHub Release", + "source.cnb": "CNB Release", + "source.upload": "上传 tar.gz", + "repo": "仓库", + "tag": "Release tag", + "arch": "架构", + "uploadFile": "一键包", + "submit": "开始导入", + "submitting": "提交中…", + "uploading": "上传中…", + "imported": "导入完成", + "importing": "正在导入…", + "importSubmitted": "已开始导入，可在任务页查看进度。", + "importFailed": "导入失败", + "importTimeout": "导入超时。稍后刷新清单查看结果。", + "component": "组件", + "version": "版本", + "versionCount": "版本数", + "size": "大小", + "source": "来源", + "installed": "已装", + "missing": "未装", + "status": "状态", + "node": "节点", + "noMissing": "已知节点都已有该版本。", + "selectNodes": "选择节点", + "cancel": "取消", + "refresh": "刷新", + "jobPending": "等待中", + "jobRunning": "进行中", + "jobSucceeded": "成功", + "jobFailed": "失败", + "jobCancelled": "已取消", + "timeoutNote": "沙箱创建或恢复时，最多会等待 10 分钟来下载缺失组件。如果节点未配置 cubeops_addr，缺失组件将直接导致创建失败。", + "coverageUnavailable": "对照不可用", + "noArtifacts": "尚未导入", + "allCovered": "所有已知节点均已覆盖", + "nodesMissingCount": "{{count}} 个节点缺失", + "componentEmpty": "该组件尚未导入任何版本制品。", + "componentEmptyHint": "您可以返回仓库首页，通过导入一键包来添加组件制品。", + "backToWarehouse": "返回组件仓库", + "about": { + "cube-shim": "容器运行时 shim 与 cube-runtime。", + "cube-image": "客户机根文件系统镜像。", + "cube-agent": "客户机内 agent。", + "cube-kernel-scf": "客户机内核。" + } +} diff --git a/web/src/main.tsx b/web/src/main.tsx index 15113fe81..c018a7d28 100644 --- a/web/src/main.tsx +++ b/web/src/main.tsx @@ -17,6 +17,9 @@ import SandboxNewPage from '@/pages/SandboxNew'; import TemplatesPage from '@/pages/Templates'; import NodesPage from '@/pages/Nodes'; import VersionsPage from '@/pages/Versions'; +import WarehousePage from '@/pages/Warehouse'; +import WarehouseComponentPage from '@/pages/WarehouseComponent'; +import WarehouseJobsPage from '@/pages/WarehouseJobs'; import SettingsPage from '@/pages/Settings'; import TemplateDetailPage from '@/pages/TemplateDetail'; import NodeDetailPage from '@/pages/NodeDetail'; @@ -57,6 +60,9 @@ const App = () => ( } /> } /> } /> + } /> + } /> + } /> } /> } /> } /> diff --git a/web/src/mocks/fixtures/index.ts b/web/src/mocks/fixtures/index.ts index f930cdd7d..3068ab70e 100644 --- a/web/src/mocks/fixtures/index.ts +++ b/web/src/mocks/fixtures/index.ts @@ -3,6 +3,7 @@ import type { components } from '@/api/generated/schema'; import type { TemplateCompatMatrix } from '@/api/client'; +import { resetWarehouseState } from './warehouse'; type ClusterOverviewDto = components['schemas']['ClusterOverview']; type ListedSandboxDto = components['schemas']['ListedSandbox']; @@ -233,6 +234,7 @@ export function resetMockState() { sandboxes = buildSandboxes(); templates = buildTemplates(); nodes = buildNodes(); + resetWarehouseState(); } export async function mockDelay() { @@ -386,7 +388,7 @@ export function getTemplate(templateID: string): TemplateDetailDto | undefined { spec: 'cpu=2000m,mem=4096Mi', artifact_id: 'rfs-mock-edge-01', last_job_id: 'job-mock-edge-01', - compat_status: base.templateID === 'python-3.11-ai' ? 'STALE' : 'OK', + compat_status: 'OK', guest_image_version: base.templateID === 'python-3.11-ai' ? 'guest-image@2024.11.02' @@ -416,9 +418,9 @@ export function getTemplate(templateID: string): TemplateDetailDto | undefined { export function getTemplateCompat(): TemplateCompatMatrix { return { summary: { - staleTemplates: 1, - staleReplicas: 1, - affectedNodes: 1, + staleTemplates: 0, + staleReplicas: 0, + affectedNodes: 0, missingReplicas: 1, unknownReplicas: 1, }, @@ -426,12 +428,12 @@ export function getTemplateCompat(): TemplateCompatMatrix { { templateID: 'python-3.11-ai', instanceType: 'standard', - overall: 'STALE', + overall: 'OK', nodes: [ { nodeID: 'cube-edge-01', nodeIP: '10.0.2.11', - compatStatus: 'STALE', + compatStatus: 'OK', boundGuestImageVersion: 'guest-image@2024.11.02', currentGuestImageVersion: 'guest-image@2024.12.01', boundAgentVersion: 'cube-agent@0.1.7', diff --git a/web/src/mocks/fixtures/warehouse.ts b/web/src/mocks/fixtures/warehouse.ts new file mode 100644 index 000000000..41f026cdb --- /dev/null +++ b/web/src/mocks/fixtures/warehouse.ts @@ -0,0 +1,552 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Tencent. All rights reserved. + +import type { + WarehouseComponentDetail, + WarehouseComponentSummary, + WarehouseImportJob, + WarehousePreinstallJob, + WarehouseVersionGroup, +} from '@/api/client'; + +const ago = (secs: number) => new Date(Date.now() - secs * 1000).toISOString(); +const clone = (value: T): T => JSON.parse(JSON.stringify(value)) as T; + +function nid(prefix: string): string { + if (typeof crypto !== 'undefined' && crypto.randomUUID) { + return crypto.randomUUID(); + } + return `${prefix}-${Date.now()}-${Math.random().toString(16).slice(2)}`; +} + +type NodeInstall = { nodeId: string; arch: string; component: string; version: string }; + +type WarehouseItem = { + arch: string; + component: string; + version: string; + source: string; + sourceRef: string; + relPath: string; + sizeBytes: number; + checksum: string; + createdAt: string; + updatedAt: string; +}; + +const CATALOG = ['cube-shim', 'cube-image', 'cube-agent', 'cube-kernel-scf'] as const; + +function item( + arch: string, + component: string, + version: string, + source: string, + sizeBytes: number, + ageSec: number, +): WarehouseItem { + const createdAt = ago(ageSec); + return { + arch, + component, + version, + source, + sourceRef: source === 'github' ? 'TencentCloud/CubeSandbox' : 'local-upload', + relPath: `${arch}/${component}/${version}`, + sizeBytes, + checksum: `sha256:${component.slice(0, 8)}${version.replace(/[^a-z0-9]/gi, '').slice(0, 8)}deadbeef`, + createdAt, + updatedAt: createdAt, + }; +} + +function buildItems(): WarehouseItem[] { + return [ + item('amd64', 'cube-shim', 'v0.6.0', 'github', 82_345_216, 86_400 * 2), + item('amd64', 'cube-image', 'v0.6.0', 'github', 412_221_440, 86_400 * 2), + item('amd64', 'cube-agent', 'v0.6.0', 'github', 8_388_608, 86_400 * 2), + item('amd64', 'cube-kernel-scf', 'sha256-b7c91a3e4f12', 'github', 61_472_768, 86_400 * 2), + item('amd64', 'cube-shim', 'v0.5.0', 'github', 80_117_760, 86_400 * 18), + item('arm64', 'cube-shim', 'v0.6.0', 'github', 79_691_776, 86_400), + item('arm64', 'cube-image', 'v0.6.0', 'github', 398_458_880, 86_400), + ]; +} + +function buildInstalls(): NodeInstall[] { + const v06 = [ + ['cube-shim', 'v0.6.0'], + ['cube-image', 'v0.6.0'], + ['cube-agent', 'v0.6.0'], + ['cube-kernel-scf', 'sha256-b7c91a3e4f12'], + ] as const; + const out: NodeInstall[] = []; + for (const [component, version] of v06) { + out.push({ nodeId: 'cube-edge-01', arch: 'amd64', component, version }); + } + out.push({ nodeId: 'cube-edge-02', arch: 'amd64', component: 'cube-shim', version: 'v0.6.0' }); + out.push({ nodeId: 'cube-edge-02', arch: 'amd64', component: 'cube-image', version: 'v0.6.0' }); + return out; +} + +const NODES = ['cube-edge-01', 'cube-edge-02', 'cube-edge-03'] as const; +const VERSIONS = ['v0.6.0', 'v0.5.0', 'v0.4.1'] as const; +/** Enough rows for 9 pages at PAGE_SIZE=20 so the pager shows numbers and ellipsis. */ +const MOCK_JOB_PAGES = 9; +const MOCK_JOB_PAGE_SIZE = 20; +const MOCK_JOB_COUNT = MOCK_JOB_PAGES * MOCK_JOB_PAGE_SIZE; + +function buildPreinstall(): WarehousePreinstallJob[] { + const showcase: WarehousePreinstallJob[] = [ + { + id: 'pre-mock-running', + nodeId: 'cube-edge-02', + arch: 'amd64', + component: 'cube-agent', + version: 'v0.6.0', + status: 'running', + }, + { + id: 'pre-mock-failed', + nodeId: 'cube-edge-03', + arch: 'amd64', + component: 'cube-shim', + version: 'v0.5.0', + status: 'failed', + error: 'download timed out contacting CubeOps', + }, + ]; + const extra: WarehousePreinstallJob[] = []; + for (let i = 0; i < MOCK_JOB_COUNT - showcase.length; i++) { + const failed = i % 9 === 8; + const cancelled = i % 9 === 7; + extra.push({ + id: `pre-mock-hist-${i + 1}`, + nodeId: NODES[i % NODES.length], + arch: i % 7 === 0 ? 'arm64' : 'amd64', + component: CATALOG[i % CATALOG.length], + version: VERSIONS[i % VERSIONS.length], + status: failed ? 'failed' : cancelled ? 'cancelled' : 'succeeded', + ...(failed ? { error: 'download timed out contacting CubeOps' } : {}), + }); + } + return [...showcase, ...extra]; +} + +function buildImportJobs(): WarehouseImportJob[] { + const sources = ['github', 'cnb', 'upload'] as const; + const jobs: WarehouseImportJob[] = []; + for (let i = 0; i < MOCK_JOB_COUNT; i++) { + const source = sources[i % sources.length]; + const failed = i % 17 === 16; + jobs.push({ + id: `imp-mock-${i + 1}`, + source, + sourceRef: source === 'upload' ? `upl-mock-${i + 1}` : 'TencentCloud/CubeSandbox', + tag: VERSIONS[i % VERSIONS.length], + arch: i % 5 === 0 ? 'arm64' : 'amd64', + status: failed ? 'failed' : 'succeeded', + bytesTotal: failed ? 0 : 564_428_032, + ...(failed ? { error: 'release asset not found for this tag' } : {}), + }); + } + return jobs; +} + +let items = buildItems(); +let installs = buildInstalls(); +let importJobs: WarehouseImportJob[] = buildImportJobs(); +const importPolls = new Map(); +let preinstallJobs = buildPreinstall(); +const uploads = new Set(); + +export function resetWarehouseState() { + items = buildItems(); + installs = buildInstalls(); + importJobs = buildImportJobs(); + importPolls.clear(); + preinstallJobs = buildPreinstall(); + uploads.clear(); +} + +function keyOf(arch: string, component: string, version: string) { + return `${arch}|${component}|${version}`; +} + +function upsertItem(next: WarehouseItem) { + const i = items.findIndex( + (row) => + row.arch === next.arch && row.component === next.component && row.version === next.version, + ); + if (i >= 0) { + items[i] = { ...items[i], ...next, updatedAt: new Date().toISOString() }; + return; + } + items.push(next); +} + +function markInstalled(nodeId: string, arch: string, component: string, version: string) { + if ( + installs.some( + (row) => + row.nodeId === nodeId && + row.arch === arch && + row.component === component && + row.version === version, + ) + ) { + return; + } + installs.push({ nodeId, arch, component, version }); +} + +function installIndex(): Map> { + const have = new Map>(); + for (const inst of installs) { + const k = keyOf(inst.arch, inst.component, inst.version); + if (!have.has(k)) have.set(k, new Set()); + have.get(k)!.add(inst.nodeId); + } + return have; +} + +function splitCoverage( + arch: string, + component: string, + version: string, + have: Map>, + nodeIds: string[], +) { + const installed = have.get(keyOf(arch, component, version)) ?? new Set(); + const nodesInstalled: string[] = []; + const nodesMissing: string[] = []; + const seen = new Set(); + for (const n of nodeIds) { + seen.add(n); + if (installed.has(n)) nodesInstalled.push(n); + else nodesMissing.push(n); + } + for (const n of installed) { + if (!seen.has(n)) nodesInstalled.push(n); + } + return { nodesInstalled, nodesMissing }; +} + +function nodesMissingAny( + component: string, + group: WarehouseItem[], + have: Map>, + nodeIds: string[], +): number { + if (group.length === 0) return 0; + let count = 0; + for (const n of nodeIds) { + for (const item of group) { + if (item.component !== component) continue; + const set = have.get(keyOf(item.arch, item.component, item.version)); + if (!set || !set.has(n)) { + count++; + break; + } + } + } + return count; +} + +export function listWarehouseComponents(nodeIds: string[]): { + components: WarehouseComponentSummary[]; +} { + tickPreinstallJobs(); + const have = installIndex(); + const byComp = new Map(); + for (const item of items) { + const cur = byComp.get(item.component) ?? []; + cur.push(item); + byComp.set(item.component, cur); + } + const components: WarehouseComponentSummary[] = CATALOG.map((name) => { + const group = byComp.get(name) ?? []; + const versions = new Set(); + const arches = new Set(); + let sizeBytes = 0; + for (const item of group) { + versions.add(item.version); + arches.add(item.arch); + sizeBytes += item.sizeBytes; + } + return { + name, + versionCount: versions.size, + arches: [...arches].sort(), + sizeBytes, + nodesMissing: nodesMissingAny(name, group, have, nodeIds), + }; + }); + return { components }; +} + +export function getWarehouseComponent( + name: string, + nodeIds: string[], +): { ok: true; detail: WarehouseComponentDetail } | { ok: false; status: number; error: string } { + if (!CATALOG.includes(name as (typeof CATALOG)[number])) { + return { ok: false, status: 400, error: `unsupported component ${JSON.stringify(name)}` }; + } + tickPreinstallJobs(); + const have = installIndex(); + const order: string[] = []; + const seen = new Map(); + for (const item of items) { + if (item.component !== name) continue; + if (!seen.has(item.version)) { + order.push(item.version); + seen.set(item.version, []); + } + seen.get(item.version)!.push(item); + } + const versions: WarehouseVersionGroup[] = order.map((version) => { + const arts = [...(seen.get(version) ?? [])].sort((a, b) => a.arch.localeCompare(b.arch)); + return { + version, + artifacts: arts.map((item) => { + const cov = splitCoverage(item.arch, item.component, item.version, have, nodeIds); + return { + arch: item.arch, + sizeBytes: item.sizeBytes, + source: item.source, + sourceRef: item.sourceRef, + checksum: item.checksum, + createdAt: item.createdAt, + nodesInstalled: cov.nodesInstalled, + nodesMissing: cov.nodesMissing, + }; + }), + }; + }); + return { ok: true, detail: { name, versions } }; +} + +export function deleteWarehouseVersion( + component: string, + version: string, + arch: string, +): 'ok' | 'not_found' { + const before = items.length; + items = items.filter( + (row) => !(row.arch === arch && row.component === component && row.version === version), + ); + if (items.length === before) { + return 'not_found'; + } + preinstallJobs = preinstallJobs.map((job) => { + if ( + job.arch === arch && + job.component === component && + job.version === version && + (job.status === 'pending' || job.status === 'running') + ) { + return { ...job, status: 'cancelled' }; + } + return job; + }); + return 'ok'; +} + +export function createUpload( + filename: string, +): { ok: true; uploadId: string; filename: string } | { ok: false; error: string } { + const lower = filename.toLowerCase(); + if (!lower.endsWith('.tar.gz') && !lower.endsWith('.tgz')) { + return { ok: false, error: 'upload must be a .tar.gz one-click package' }; + } + const uploadId = nid('upload'); + uploads.add(uploadId); + return { ok: true, uploadId, filename }; +} + +export function createImportJobs(body: { + source?: string; + repo?: string; + tag?: string; + uploadId?: string; + arch?: string[]; +}): { ok: true; jobs: WarehouseImportJob[] } | { ok: false; status: number; error: string } { + const source = (body.source ?? '').trim().toLowerCase(); + const arches = (body.arch ?? []).map((a) => a.trim().toLowerCase()).filter(Boolean); + if (arches.length === 0) { + return { ok: false, status: 400, error: 'arch is required' }; + } + for (const a of arches) { + if (a !== 'amd64' && a !== 'arm64' && a !== 'x86_64' && a !== 'aarch64') { + return { + ok: false, + status: 400, + error: `unsupported arch ${JSON.stringify(a)} (want amd64 or arm64)`, + }; + } + } + const normArch = (a: string) => (a === 'x86_64' ? 'amd64' : a === 'aarch64' ? 'arm64' : a); + const jobs: WarehouseImportJob[] = []; + for (const raw of arches) { + const arch = normArch(raw); + const job: WarehouseImportJob = { + id: nid('imp'), + source, + sourceRef: '', + tag: (body.tag ?? '').trim(), + arch, + status: 'pending', + bytesTotal: 0, + }; + if (source === 'upload') { + const uploadId = (body.uploadId ?? '').trim(); + if (!uploadId) return { ok: false, status: 400, error: 'uploadId is required' }; + if (!uploads.has(uploadId)) return { ok: false, status: 400, error: 'upload not found' }; + job.sourceRef = uploadId; + } else if (source === 'github' || source === 'cnb') { + if (!(body.repo ?? '').trim() || !(body.tag ?? '').trim()) { + return { ok: false, status: 400, error: 'repo and tag are required' }; + } + job.sourceRef = body.repo!.trim(); + } else { + return { ok: false, status: 400, error: 'source must be github, cnb, or upload' }; + } + importJobs.push(job); + importPolls.set(job.id, 0); + jobs.push(job); + } + return { ok: true, jobs: clone(jobs) }; +} + +function extractedFor(job: WarehouseImportJob): WarehouseItem[] { + const tag = job.tag || 'v0.6.0'; + const source = job.source || 'github'; + const now = 0; + return [ + item(job.arch, 'cube-shim', tag, source, 82_345_216, now), + item(job.arch, 'cube-image', tag, source, 412_221_440, now), + item(job.arch, 'cube-agent', tag, source, 8_388_608, now), + item(job.arch, 'cube-kernel-scf', 'sha256-b7c91a3e4f12', source, 61_472_768, now), + ]; +} + +export function getImportJob(id: string): WarehouseImportJob | undefined { + const job = importJobs.find((row) => row.id === id); + if (!job) return undefined; + tickImportJob(job); + return clone(job); +} + +export function listImportJobs(opts: { limit?: string | null; offset?: string | null } = {}): { + jobs: WarehouseImportJob[]; + total: number; +} { + for (const job of importJobs) { + tickImportJob(job); + } + return paginate(clone(importJobs).reverse(), opts.limit, opts.offset); +} + +function tickImportJob(job: WarehouseImportJob) { + const polls = (importPolls.get(job.id) ?? 0) + 1; + importPolls.set(job.id, polls); + if (job.status === 'pending' || job.status === 'running') { + if (job.tag.trim().toLowerCase() === 'fail' && polls >= 2) { + job.status = 'failed'; + job.error = 'release asset not found for this tag'; + } else if (polls === 1) { + job.status = 'running'; + } else if (polls >= 2) { + job.status = 'succeeded'; + job.bytesTotal = 564_428_032; + for (const next of extractedFor(job)) { + upsertItem(next); + } + } + } +} + +export function createPreinstallJobs(body: { + nodeIds?: string[]; + arch?: string; + component?: string; + version?: string; +}): + | { ok: true; jobs: WarehousePreinstallJob[] } + | { ok: false; status: number; error: string; code?: string } { + const arch = (body.arch ?? '').trim(); + const component = (body.component ?? '').trim(); + const version = (body.version ?? '').trim(); + if (!arch) return { ok: false, status: 400, error: 'arch query is required (amd64 or arm64)' }; + if (!component || !version) + return { ok: false, status: 400, error: 'component and version are required' }; + const found = items.find( + (row) => row.arch === arch && row.component === component && row.version === version, + ); + if (!found) { + return { + ok: false, + status: 404, + error: 'warehouse version not found', + code: 'warehouse_not_found', + }; + } + const nodeIds = (body.nodeIds ?? []).map((n) => n.trim()).filter(Boolean); + if (nodeIds.length === 0) return { ok: false, status: 400, error: 'nodeIds is required' }; + const jobs: WarehousePreinstallJob[] = nodeIds.map((nodeId) => ({ + id: nid('pre'), + nodeId, + arch, + component, + version, + status: 'running', + })); + preinstallJobs.push(...jobs); + return { ok: true, jobs: clone(jobs) }; +} + +function tickPreinstallJobs() { + for (const job of preinstallJobs) { + if (job.status === 'pending') { + job.status = 'running'; + } else if (job.status === 'running') { + job.status = 'succeeded'; + markInstalled(job.nodeId, job.arch, job.component, job.version); + } + } +} + +export function listPreinstallJobs( + filters: { + node_id?: string | null; + status?: string | null; + limit?: string | null; + offset?: string | null; + } = {}, +) { + tickPreinstallJobs(); + const jobs = preinstallJobs.filter((job) => { + if (filters.node_id && job.nodeId !== filters.node_id) return false; + if (filters.status && job.status !== filters.status) return false; + return true; + }); + return paginate(clone(jobs), filters.limit, filters.offset); +} + +function paginate( + items: T[], + rawLimit?: string | null, + rawOffset?: string | null, +): { jobs: T[]; total: number } { + const total = items.length; + let offset = Number(rawOffset); + if (!Number.isFinite(offset) || offset < 0) { + offset = 0; + } + let limit = Number(rawLimit); + if (!Number.isFinite(limit) || limit <= 0) { + limit = 50; + } + if (limit > 200) { + limit = 200; + } + return { jobs: items.slice(offset, offset + limit), total }; +} diff --git a/web/src/mocks/handlers/index.ts b/web/src/mocks/handlers/index.ts index 987de73dd..c7381f22e 100644 --- a/web/src/mocks/handlers/index.ts +++ b/web/src/mocks/handlers/index.ts @@ -10,7 +10,6 @@ import { getVersionMatrix, getSandboxDetail, getSandboxLogs, - getSandboxSession, getTemplate, getTemplateCompat, listNodes, @@ -21,11 +20,26 @@ import { resetMockState, resumeSandbox, } from '../fixtures'; +import { + createImportJobs, + createPreinstallJobs, + createUpload, + deleteWarehouseVersion, + getImportJob, + listImportJobs, + getWarehouseComponent, + listPreinstallJobs, + listWarehouseComponents, +} from '../fixtures/warehouse'; function notFound(message: string) { return HttpResponse.json({ code: 404, message }, { status: 404 }); } +function opsError(status: number, error: string, code?: string) { + return HttpResponse.json(code ? { error, code } : { error }, { status }); +} + export const handlers = [ http.get('/cubeapi/v1/health', async () => { await mockDelay(); @@ -141,4 +155,151 @@ export const handlers = [ resetMockState(); return HttpResponse.json({ ok: true }); }), + + // Mock mode has no CubeOps: keep the shell open so warehouse and other + // ops-backed pages can be exercised locally. + http.get('/opsapi/v1/auth/session', async () => { + await mockDelay(); + return HttpResponse.json({ authRequired: false, authenticated: true, username: 'mock' }); + }), + http.post('/opsapi/v1/auth/login', async () => { + await mockDelay(); + return HttpResponse.json({ + accessToken: 'mock-access', + refreshToken: 'mock-refresh', + username: 'mock', + expiresInSecs: 3600, + }); + }), + http.post('/opsapi/v1/auth/logout', async () => { + await mockDelay(); + return new HttpResponse(null, { status: 204 }); + }), + http.post('/opsapi/v1/auth/refresh', async () => { + await mockDelay(); + return HttpResponse.json({ accessToken: 'mock-access', refreshToken: 'mock-refresh' }); + }), + + http.get('/opsapi/v1/warehouse/components', async () => { + await mockDelay(); + const nodeIds = listNodes() + .map((n) => n.nodeID) + .filter(Boolean); + return HttpResponse.json(listWarehouseComponents(nodeIds)); + }), + + http.get('/opsapi/v1/warehouse/components/:component', async ({ params }) => { + await mockDelay(); + const nodeIds = listNodes() + .map((n) => n.nodeID) + .filter(Boolean); + const result = getWarehouseComponent(String(params.component), nodeIds); + if (!result.ok) { + return opsError(result.status, result.error); + } + return HttpResponse.json(result.detail); + }), + + http.delete( + '/opsapi/v1/warehouse/components/:component/versions/:version', + async ({ request, params }) => { + await mockDelay(); + const url = new URL(request.url); + const arch = url.searchParams.get('arch') ?? ''; + if (!arch) { + return opsError(400, 'arch query is required (amd64 or arm64)'); + } + const result = deleteWarehouseVersion(String(params.component), String(params.version), arch); + if (result === 'not_found') { + return opsError(404, 'warehouse version not found', 'warehouse_not_found'); + } + return new HttpResponse(null, { status: 204 }); + }, + ), + + http.post('/opsapi/v1/warehouse/uploads', async ({ request }) => { + await mockDelay(); + const form = await request.formData(); + const file = form.get('file'); + if (!(file instanceof File)) { + return opsError(400, 'multipart field file is required'); + } + const result = createUpload(file.name); + if (!result.ok) { + return opsError(400, result.error); + } + return HttpResponse.json( + { uploadId: result.uploadId, filename: result.filename }, + { status: 201 }, + ); + }), + + http.post('/opsapi/v1/warehouse/imports', async ({ request }) => { + await mockDelay(); + let body: { + source?: string; + repo?: string; + tag?: string; + uploadId?: string; + arch?: string[]; + }; + try { + body = (await request.json()) as typeof body; + } catch { + return opsError(400, 'invalid JSON body'); + } + const result = createImportJobs(body); + if (!result.ok) { + return opsError(result.status, result.error); + } + return HttpResponse.json({ jobs: result.jobs }, { status: 202 }); + }), + + http.get('/opsapi/v1/warehouse/imports', async ({ request }) => { + await mockDelay(); + const url = new URL(request.url); + return HttpResponse.json( + listImportJobs({ + limit: url.searchParams.get('limit'), + offset: url.searchParams.get('offset'), + }), + ); + }), + + http.get('/opsapi/v1/warehouse/imports/:id', async ({ params }) => { + await mockDelay(); + const job = getImportJob(String(params.id)); + if (!job) { + return opsError(404, 'import job not found'); + } + return HttpResponse.json(job); + }), + + http.get('/opsapi/v1/warehouse/preinstall', async ({ request }) => { + await mockDelay(); + const url = new URL(request.url); + return HttpResponse.json( + listPreinstallJobs({ + node_id: url.searchParams.get('node_id'), + status: url.searchParams.get('status'), + limit: url.searchParams.get('limit'), + offset: url.searchParams.get('offset'), + }), + ); + }), + + http.post('/opsapi/v1/warehouse/preinstall', async ({ request }) => { + await mockDelay(); + let body: { nodeIds?: string[]; arch?: string; component?: string; version?: string }; + try { + body = (await request.json()) as typeof body; + } catch { + return opsError(400, 'invalid JSON body'); + } + const result = createPreinstallJobs(body); + if (!result.ok) { + return opsError(result.status, result.error, result.code); + } + return HttpResponse.json({ jobs: result.jobs }, { status: 202 }); + }), ]; diff --git a/web/src/pages/SandboxNew.tsx b/web/src/pages/SandboxNew.tsx index d27ce3378..5f9f2a89f 100644 --- a/web/src/pages/SandboxNew.tsx +++ b/web/src/pages/SandboxNew.tsx @@ -51,8 +51,10 @@ function TemplatePicker({ queryFn: templateApi.compat, staleTime: 15_000, }); - const staleTemplates = new Set( - (compat?.templates ?? []).filter((row) => row.overall === 'STALE').map((row) => row.templateID), + const unpinnedTemplates = new Set( + (compat?.templates ?? []) + .filter((row) => row.overall === 'UNKNOWN') + .map((row) => row.templateID), ); if (isLoading) { @@ -70,7 +72,7 @@ function TemplatePicker({ {(templates ?? []).map((tpl) => { const statusLower = tpl.status.toLowerCase(); const isReady = statusLower === 'ready'; - const isStale = staleTemplates.has(tpl.templateID); + const isUnpinned = unpinnedTemplates.has(tpl.templateID); const isSelected = tpl.templateID === selected; return (
diff --git a/web/src/pages/TemplateDetail.tsx b/web/src/pages/TemplateDetail.tsx index efafb66c4..7ddacde2e 100644 --- a/web/src/pages/TemplateDetail.tsx +++ b/web/src/pages/TemplateDetail.tsx @@ -88,17 +88,31 @@ function compatTone(status: string) { switch (status.toUpperCase()) { case 'OK': return 'bg-cube-ok/15 text-cube-ok border-cube-ok/30'; - case 'STALE': - return 'bg-destructive/10 text-destructive border-destructive/30'; case 'UNKNOWN': - return 'bg-cube-warn/15 text-cube-warn border-cube-warn/30'; + return 'bg-cube-err/15 text-cube-err border-cube-err/30'; + case 'STALE': + return 'bg-muted text-muted-foreground border-border'; default: return 'bg-muted text-muted-foreground border-border'; } } -function isStaleCompat(status: string) { - return status.toUpperCase() === 'STALE'; +function isUnknownCompat(status: string) { + return status.toUpperCase() === 'UNKNOWN'; +} + +function replicaLiveDiffers(node: TemplateNodeCompat): boolean { + const guestBound = (node.boundGuestImageVersion ?? '').trim(); + const guestCurrent = (node.currentGuestImageVersion ?? '').trim(); + const agentBound = (node.boundAgentVersion ?? '').trim(); + const agentCurrent = (node.currentAgentVersion ?? '').trim(); + const kernelBound = (node.boundKernelVersion ?? '').trim(); + const kernelCurrent = (node.currentKernelVersion ?? '').trim(); + return ( + (guestBound !== '' && guestCurrent !== '' && guestBound !== guestCurrent) || + (agentBound !== '' && agentCurrent !== '' && agentBound !== agentCurrent) || + (kernelBound !== '' && kernelCurrent !== '' && kernelBound !== kernelCurrent) + ); } function CompatBadge({ status }: { status: string }) { @@ -526,33 +540,27 @@ function CompatWarning({ disabled: boolean; }) { const { t } = useTranslation('templateDetail'); - const staleNodes = row.nodes.filter((node) => isStaleCompat(node.compatStatus)); - if (staleNodes.length === 0) return null; + const unknownNodes = row.nodes.filter((node) => isUnknownCompat(node.compatStatus)); + if (unknownNodes.length === 0) return null; return ( -
+
-
+

{t('compat.staleTitle')}

- {t('compat.staleDesc', { count: staleNodes.length })} + {t('compat.staleDesc', { count: unknownNodes.length })}

-
- {staleNodes.map((node) => ( + {unknownNodes.map((node) => ( ))}
@@ -560,6 +568,19 @@ function CompatWarning({ ); } +function CompatDriftNote({ row }: { row: TemplateCompatRow }) { + const { t } = useTranslation('templateDetail'); + const drifted = row.nodes.filter((node) => replicaLiveDiffers(node)); + if (drifted.length === 0) return null; + if (row.nodes.some((node) => isUnknownCompat(node.compatStatus))) return null; + return ( +
+

{t('compat.driftTitle')}

+

{t('compat.driftDesc')}

+
+ ); +} + function CompatSection({ row }: { row?: TemplateCompatRow }) { const { t } = useTranslation('templateDetail'); if (!row) { @@ -772,8 +793,8 @@ export default function TemplateDetailPage() { const compatRow = compat?.templates.find((row) => row.templateID === templateID); const compatStatus = compatRow?.overall ?? 'UNKNOWN'; const compatNodes = compatRow?.nodes ?? []; - const isStale = isStaleCompat(compatStatus); - const headerAccentClass = isStale ? 'border-destructive' : 'border-cube-ok'; + const isUnknown = isUnknownCompat(compatStatus); + const headerAccentClass = isUnknown ? 'border-cube-err' : 'border-cube-ok'; return (
@@ -830,7 +851,7 @@ export default function TemplateDetailPage() {
{[ @@ -870,11 +891,14 @@ export default function TemplateDetailPage() {
{compatRow && ( - setShowRebuildConfirm(true)} - /> + <> + setShowRebuildConfirm(true)} + /> + + )} {/* ── build progress ── */} diff --git a/web/src/pages/Templates.tsx b/web/src/pages/Templates.tsx index e063a7a98..761b3280b 100644 --- a/web/src/pages/Templates.tsx +++ b/web/src/pages/Templates.tsx @@ -10,6 +10,7 @@ import { templateApi, type TemplateCompatMatrix, type TemplateCompatRow, + type TemplateNodeCompat, type TemplateSummary, } from '@/api/client'; import { Card, CardHeader, CardTitle, CardDescription, CardContent } from '@/components/ui/card'; @@ -590,6 +591,9 @@ export default function TemplatesPage() { const [deletingID, setDeletingID] = useState(null); const [tab, setTab] = useState<'list' | 'compat'>('list'); const compatByTemplate = new Map((compat?.templates ?? []).map((row) => [row.templateID, row])); + const unknownTemplateCount = (compat?.templates ?? []).filter( + (row) => row.overall === 'UNKNOWN', + ).length; return (
@@ -603,15 +607,14 @@ export default function TemplatesPage() { - {(compat?.summary.staleTemplates ?? 0) > 0 && ( - + {unknownTemplateCount > 0 && ( +
-
+
{t('compat.banner', { - templates: compat?.summary.staleTemplates, - replicas: compat?.summary.staleReplicas, + count: unknownTemplateCount, })}
@@ -636,9 +639,9 @@ export default function TemplatesPage() { onClick={() => setTab('compat')} > {t('tabs.compat')} - {(compat?.summary.staleTemplates ?? 0) > 0 && ( + {unknownTemplateCount > 0 && ( - {compat?.summary.staleTemplates} + {unknownTemplateCount} )} @@ -678,8 +681,8 @@ export default function TemplatesPage() {
- {compatByTemplate.get(tpl.templateID)?.overall === 'STALE' ? ( - {t('compat.status.STALE')} + {compatByTemplate.get(tpl.templateID)?.overall === 'UNKNOWN' ? ( + {t('compat.status.UNKNOWN')} ) : ( -
+
- -
+ {matrix.summary.staleTemplates > 0 && ( +
+ + + +
+ )} {matrix.templates.length === 0 ? (
{t('noTemplates')}
@@ -839,7 +851,7 @@ function compatKpiToneClass(tone: 'err' | 'warn' | 'mute') { case 'err': return 'text-destructive'; case 'warn': - return 'text-warning'; + return 'text-cube-warn'; default: return 'text-muted-foreground'; } @@ -847,12 +859,8 @@ function compatKpiToneClass(tone: 'err' | 'warn' | 'mute') { function CompatTemplateRow({ row }: { row: TemplateCompatRow }) { const { t } = useTranslation('templates'); - const queryClient = useQueryClient(); - const adoptMutation = useMutation({ - mutationFn: () => templateApi.adoptCompatBaseline(row.templateID), - onSuccess: () => queryClient.invalidateQueries({ queryKey: ['templates', 'compat'] }), - }); const hasUnknown = row.nodes.some((node) => node.compatStatus === 'UNKNOWN'); + const hasDrift = row.nodes.some((node) => replicaLiveDiffers(node)); return (
@@ -865,17 +873,8 @@ function CompatTemplateRow({ row }: { row: TemplateCompatRow }) {
{hasUnknown && ( - )} @@ -883,6 +882,9 @@ function CompatTemplateRow({ row }: { row: TemplateCompatRow }) {
+ {hasDrift && !hasUnknown && ( +

{t('compat.driftHint')}

+ )}
{row.nodes.map((node) => (
n + row.nodes.filter((node) => replicaLiveDiffers(node)).length, + 0, + ); +} + function compatTone(status: string): 'ok' | 'err' | 'warn' | 'mute' { if (status === 'OK') return 'ok'; - if (status === 'STALE') return 'err'; + if (status === 'UNKNOWN') return 'err'; if (status === 'MISSING') return 'warn'; + if (status === 'STALE') return 'mute'; return 'mute'; } diff --git a/web/src/pages/Warehouse.tsx b/web/src/pages/Warehouse.tsx new file mode 100644 index 000000000..be31eaff0 --- /dev/null +++ b/web/src/pages/Warehouse.tsx @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Tencent. All rights reserved. + +import { useState } from 'react'; +import { createPortal } from 'react-dom'; +import { useQuery, useQueryClient } from '@tanstack/react-query'; +import { Link, useNavigate } from 'react-router-dom'; +import { useTranslation } from 'react-i18next'; +import type { TFunction } from 'i18next'; +import { + Archive, + RefreshCw, + Upload, + ChevronRight, + Box, + Disc, + Cpu, + Microchip, + FileClock, +} from 'lucide-react'; +import { warehouseApi, type WarehouseComponentSummary } from '@/api/client'; +import { Button } from '@/components/ui/button'; +import { Card } from '@/components/ui/card'; +import { Skeleton } from '@/components/ui/skeleton'; +import { formatArtifactBytes } from '@/lib/utils'; +import { ImportTab } from '@/components/warehouse/ImportTab'; + +export default function WarehousePage() { + const { t } = useTranslation('warehouse'); + const qc = useQueryClient(); + const navigate = useNavigate(); + const [showImport, setShowImport] = useState(false); + + const listQ = useQuery({ + queryKey: ['warehouse', 'components'], + queryFn: warehouseApi.listComponents, + }); + const jobsQ = useQuery({ + queryKey: ['warehouse', 'jobs'], + queryFn: () => warehouseApi.preinstallJobs({ limit: 50, offset: 0 }), + refetchInterval: 5000, + }); + const importsQ = useQuery({ + queryKey: ['warehouse', 'imports'], + queryFn: () => warehouseApi.listImports({ limit: 50, offset: 0 }), + refetchInterval: 5000, + }); + + const activeJobs = [ + ...(jobsQ.data?.jobs ?? []).filter((j) => j.status === 'pending' || j.status === 'running'), + ...(importsQ.data?.jobs ?? []).filter((j) => j.status === 'pending' || j.status === 'running'), + ]; + + return ( +
+
+
+

+ + {t('title')} +

+

{t('subtitle')}

+
+
+ + + +
+
+ +
+ + {t('timeoutNote')} +
+ + {listQ.isLoading && ( +
+ {Array.from({ length: 4 }).map((_, i) => ( + + ))} +
+ )} + {listQ.isError &&

{t('error')}

} + {listQ.data && ( +
+ {listQ.data.components.map((row) => ( + + ))} +
+ )} + + {showImport && + createPortal( +
setShowImport(false)} + > +
e.stopPropagation()} + > + { + void qc.invalidateQueries({ queryKey: ['warehouse'] }); + setShowImport(false); + }} + onCancel={() => setShowImport(false)} + /> +
+
, + document.body, + )} +
+ ); +} + +function ComponentCard({ row, t }: { row: WarehouseComponentSummary; t: TFunction<'warehouse'> }) { + const about = t(`about.${row.name}`, { defaultValue: '' }); + + let Icon = Box; + if (row.name === 'cube-image') Icon = Disc; + if (row.name === 'cube-agent') Icon = Cpu; + if (row.name === 'cube-kernel-scf') Icon = Microchip; + + return ( + + +
+
+
+ +
+
+

{row.name}

+ {about ? ( +

{about}

+ ) : null} +
+
+ +
+ +
+
+ + {t('versionCount')} + + {row.versionCount} +
+
+ + {t('arch')} + + + {row.arches.length ? row.arches.join(' · ') : '—'} + +
+
+ + {t('size')} + + + {formatArtifactBytes(row.sizeBytes)} + +
+
+ +
+ + {t('coverage')} + + {row.nodesMissing == null ? ( + + + {t('coverageUnavailable')} + + ) : row.versionCount === 0 ? ( + + + {t('noArtifacts')} + + ) : row.nodesMissing === 0 ? ( + + + {t('allCovered')} + + ) : ( + + + {t('nodesMissingCount', { count: row.nodesMissing })} + + )} +
+ + + ); +} diff --git a/web/src/pages/WarehouseComponent.tsx b/web/src/pages/WarehouseComponent.tsx new file mode 100644 index 000000000..ac79169cb --- /dev/null +++ b/web/src/pages/WarehouseComponent.tsx @@ -0,0 +1,397 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Tencent. All rights reserved. + +import { useState } from 'react'; +import { createPortal } from 'react-dom'; +import { Link, useParams } from 'react-router-dom'; +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { useTranslation } from 'react-i18next'; +import { + ArrowLeft, + Download, + Trash2, + CheckCircle2, + AlertCircle, + Box, + Disc, + Cpu, + Microchip, + Layers, + Info, + RefreshCw, +} from 'lucide-react'; +import { warehouseApi, type WarehouseArtifact } from '@/api/client'; +import { ApiError } from '@/lib/api'; +import { Button } from '@/components/ui/button'; +import { Card } from '@/components/ui/card'; +import { Skeleton } from '@/components/ui/skeleton'; +import { showToast } from '@/components/ui/ToastProvider'; +import { cn, formatArtifactBytes } from '@/lib/utils'; + +export default function WarehouseComponentPage() { + const { component = '' } = useParams(); + const { t } = useTranslation('warehouse'); + const qc = useQueryClient(); + const name = decodeURIComponent(component); + const [target, setTarget] = useState<{ version: string; artifact: WarehouseArtifact } | null>( + null, + ); + + const q = useQuery({ + queryKey: ['warehouse', 'component', name], + queryFn: () => warehouseApi.getComponent(name), + enabled: name !== '', + }); + + const del = useMutation({ + mutationFn: (row: { version: string; arch: string }) => + warehouseApi.deleteVersion(name, row.version, row.arch), + onSuccess: () => { + void qc.invalidateQueries({ queryKey: ['warehouse'] }); + }, + onError: (err: unknown) => { + showToast(err instanceof ApiError ? err.message : String(err), 'warn'); + }, + }); + + const about = t(`about.${name}`, { defaultValue: '' }); + + let Icon = Box; + if (name === 'cube-image') Icon = Disc; + if (name === 'cube-agent') Icon = Cpu; + if (name === 'cube-kernel-scf') Icon = Microchip; + + if (q.isLoading) { + return ( +
+ + +
+ ); + } + + if (q.isError || !q.data) { + const msg = q.error instanceof ApiError ? q.error.message : t('error'); + return ( +
+ +

{msg}

+
+ ); + } + + const versions = q.data.versions ?? []; + + return ( +
+ +
+
+ +
+
+

{q.data.name}

+ {about ?

{about}

: null} +
+
+ + {versions.length === 0 ? ( + +
+ +
+
{t('componentEmpty')}
+
+ {t('componentEmptyHint')} +
+ + + + + ) : ( +
+ {versions.map((group) => ( + +
+
+ + {group.version} + +
+
+
+ + + + + + + + + + + + {group.artifacts.map((art) => ( + + + + + + + + + ))} + +
{t('arch')}{t('size')}{t('source')}{t('installed')}{t('missing')} +
{art.arch} + {formatArtifactBytes(art.sizeBytes)} + {art.source} + {art.nodesInstalled == null ? ( + + {t('coverageUnavailable')} + + ) : art.nodesInstalled.length === 0 ? ( + — + ) : ( +
+ {art.nodesInstalled.map((n) => ( + + {n} + + ))} +
+ )} + 		+ {art.nodesMissing == null ? ( + + {t('coverageUnavailable')} + + ) : art.nodesMissing.length === 0 ? ( + + + {t('allCovered')} + + ) : ( +
+ {art.nodesMissing.map((n) => ( + + + {n} + + ))} +
+ )} + 		+
+ + +
+
+
+ + ))} +
+ )} + + {target && ( + setTarget(null)} + /> + )} +
+ ); +} + +function BackLink() { + const { t } = useTranslation('warehouse'); + return ( + + + {t('backToWarehouse')} + + ); +} + +function PreinstallDialog({ + component, + version, + artifact, + onClose, +}: { + component: string; + version: string; + artifact: WarehouseArtifact; + onClose: () => void; +}) { + const { t } = useTranslation('warehouse'); + const qc = useQueryClient(); + const missing = artifact.nodesMissing ?? []; + const [selected, setSelected] = useState(missing); + + const mut = useMutation({ + mutationFn: () => + warehouseApi.preinstall({ + nodeIds: selected, + arch: artifact.arch, + component, + version, + }), + onSuccess: () => { + showToast(t('preinstallHint'), 'success'); + void qc.invalidateQueries({ queryKey: ['warehouse'] }); + onClose(); + }, + onError: (err: unknown) => + showToast(err instanceof ApiError ? err.message : String(err), 'warn'), + }); + + return createPortal( +
+ e.stopPropagation()} + > +
+

{t('preinstall')}

+
+ + {component} + + / + + {version} + + / + {artifact.arch} +
+
+ +
+
+ + {t('preinstallHint')} +
+ +
+
+ +
+ {selected.length} / {missing.length} selected +
+
+ +
+ {missing.map((n) => { + const isSelected = selected.includes(n); + return ( + + ); + })} +
+
+
+ +
+ + +
+ +
, + document.body, + ); +} diff --git a/web/src/pages/WarehouseJobs.tsx b/web/src/pages/WarehouseJobs.tsx new file mode 100644 index 000000000..20b79cd4c --- /dev/null +++ b/web/src/pages/WarehouseJobs.tsx @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Tencent. All rights reserved. + +import { useQuery } from '@tanstack/react-query'; +import { Link, useSearchParams } from 'react-router-dom'; +import { useTranslation } from 'react-i18next'; +import type { ElementType, ReactNode } from 'react'; +import { ArrowLeft, RefreshCw, FileClock, Download } from 'lucide-react'; +import { warehouseApi, type WarehouseImportJob, type WarehousePreinstallJob } from '@/api/client'; +import { Button } from '@/components/ui/button'; +import { Card } from '@/components/ui/card'; +import { Pagination } from '@/components/ui/pagination'; +import { Skeleton } from '@/components/ui/skeleton'; +import { cn, formatArtifactBytes, getStatusBadge } from '@/lib/utils'; + +const PAGE_SIZE = 20; + +type JobsTab = 'import' | 'preinstall'; + +function parseTab(raw: string | null): JobsTab { + return raw === 'preinstall' ? 'preinstall' : 'import'; +} + +function parsePage(raw: string | null): number { + const n = Number(raw); + return Number.isInteger(n) && n > 0 ? n : 1; +} + +export default function WarehouseJobsPage() { + const { t } = useTranslation('warehouse'); + const [params, setParams] = useSearchParams(); + const tab = parseTab(params.get('tab')); + const page = parsePage(params.get('page')); + const offset = (page - 1) * PAGE_SIZE; + + const setTab = (next: JobsTab) => { + const nextParams = new URLSearchParams(); + nextParams.set('tab', next); + setParams(nextParams); + }; + + const setPage = (next: number) => { + const nextParams = new URLSearchParams(); + nextParams.set('tab', tab); + if (next > 1) { + nextParams.set('page', String(next)); + } + setParams(nextParams); + }; + + const importsQ = useQuery({ + queryKey: ['warehouse', 'imports', page], + queryFn: () => warehouseApi.listImports({ limit: PAGE_SIZE, offset }), + refetchInterval: 5000, + enabled: tab === 'import', + }); + const jobsQ = useQuery({ + queryKey: ['warehouse', 'jobs', page], + queryFn: () => warehouseApi.preinstallJobs({ limit: PAGE_SIZE, offset }), + refetchInterval: 5000, + enabled: tab === 'preinstall', + }); + + const activeQ = tab === 'import' ? importsQ : jobsQ; + const importJobs = importsQ.data?.jobs ?? []; + const preinstallJobs = jobsQ.data?.jobs ?? []; + const total = activeQ.data?.total ?? 0; + const loading = + activeQ.isLoading && (tab === 'import' ? importJobs.length === 0 : preinstallJobs.length === 0); + const pager = + total > 0 ? ( + + ) : null; + + return ( +
+
+ + + {t('title')} + + / + {t('jobs')} +
+ +
+
+

+ + {t('jobs')} +

+

{t('jobsSubtitle')}

+
+ +
+ +
+ {( + [ + { key: 'import', label: t('importJobs') }, + { key: 'preinstall', label: t('preinstallJobs') }, + ] as const + ).map(({ key, label }) => ( + + ))} +
+ + {loading ? ( + + ) : tab === 'import' ? ( + + {importJobs.length === 0 ? null : } + + ) : ( + + {preinstallJobs.length === 0 ? null : ( + + )} + + )} +
+ ); +} + +function JobSection({ + emptyTitle, + emptyDesc, + icon: Icon, + children, +}: { + emptyTitle: string; + emptyDesc: string; + icon: ElementType; + children: ReactNode; +}) { + return ( +
+ {children ?? ( + +
+ +
+

{emptyTitle}

+

{emptyDesc}

+ + )} +
+ ); +} + +function JobsTableCard({ children, footer }: { children: ReactNode; footer?: ReactNode }) { + return ( + +
+ {children}
+
+ {footer ?
{footer}
: null} + + ); +} + +function ImportJobsTable({ jobs, footer }: { jobs: WarehouseImportJob[]; footer?: ReactNode }) { + const { t } = useTranslation('warehouse'); + return ( + + + + {t('source')} + {t('repo')} + {t('version')} + {t('arch')} + {t('size')} + {t('status')} + + + + {jobs.map((job) => ( + + {t(`source.${job.source}`, { defaultValue: job.source })} + + {job.source === 'upload' ? '—' : job.sourceRef} + + {job.tag || '—'} + {job.arch} + + {formatArtifactBytes(job.bytesTotal)} + + {getStatusBadge(job.status, t, job.error)} + + ))} + + + ); +} + +function PreinstallJobsTable({ + jobs, + footer, +}: { + jobs: WarehousePreinstallJob[]; + footer?: ReactNode; +}) { + const { t } = useTranslation('warehouse'); + return ( + + + + {t('node')} + {t('component')} + {t('version')} + {t('status')} + + + + {jobs.map((job) => ( + + {job.nodeId} + {job.component} + + {job.version} / {job.arch} + + {getStatusBadge(job.status, t, job.error)} + + ))} + + + ); +}