From 2daae3644a59747fa7560175065b156cf0f9db9a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 12 Oct 2026 23:38:04 -0700 Subject: [PATCH 1/5] feat: guard direct lease expiry and gate remote allocations --- CHANGELOG.md | 2 + docs/commands/cleanup.md | 21 +- docs/features/configuration.md | 40 +- docs/features/lifecycle-cleanup.md | 86 ++-- docs/features/providers.md | 37 +- docs/providers/gcp.md | 15 +- internal/cli/actions.go | 2 +- internal/cli/aws.go | 11 + internal/cli/aws_fixed_create_intent_test.go | 1 + internal/cli/bootstrap.go | 107 +---- internal/cli/bootstrap_test.go | 10 +- internal/cli/checkpoint.go | 6 + internal/cli/config.go | 23 ++ internal/cli/config_cmd.go | 3 + internal/cli/config_show_projection.go | 2 +- internal/cli/config_test.go | 2 + internal/cli/connect.go | 4 +- internal/cli/direct_leases.go | 94 +++++ internal/cli/direct_leases_test.go | 141 +++++++ internal/cli/guest_expiry.go | 261 ++++++++++++ internal/cli/guest_expiry_test.go | 377 ++++++++++++++++++ internal/cli/hcloud.go | 5 +- internal/cli/heartbeat.go | 2 +- internal/cli/pool.go | 11 +- internal/cli/pool_test.go | 26 +- internal/cli/prewarm.go | 3 + internal/cli/provider_backend.go | 2 +- internal/cli/run.go | 17 +- internal/cli/shard.go | 3 + internal/cli/status.go | 2 +- internal/cli/warmup_expiry_guard_test.go | 10 +- internal/cli/warmup_policy_test.go | 4 +- internal/providers/digitalocean/client.go | 2 +- .../providers/digitalocean/heartbeat_test.go | 91 ++++- internal/providers/firecracker/backend.go | 2 +- internal/providers/incus/backend.go | 2 +- internal/providers/lambda/backend.go | 2 +- internal/providers/linode/backend.go | 2 +- internal/providers/linode/fixed.go | 2 +- internal/providers/multipass/backend.go | 3 +- internal/providers/multipass/backend_test.go | 13 + internal/providers/nebius/backend.go | 2 +- internal/providers/ovh/backend.go | 2 +- internal/providers/scaleway/fixed.go | 2 +- internal/providers/scaleway/provider.go | 2 +- internal/providers/tencentcloud/backend.go | 2 +- internal/providers/vultr/client.go | 2 +- 47 files changed, 1251 insertions(+), 210 deletions(-) create mode 100644 internal/cli/direct_leases.go create mode 100644 internal/cli/direct_leases_test.go create mode 100644 internal/cli/guest_expiry.go create mode 100644 internal/cli/guest_expiry_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e7414d31b6..a3b06d6eaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,14 @@ - CI: run CLI coverage in three isolated shards, retaining atomic whole-suite coverage and the 90% core gate; report the slowest top-level tests without summing nested timings. - Coordinator monthly reserved-USD budgets are raised to $100,000 fleet-wide, $50,000 per owner and $100,000 per org; reservations from finished leases had exhausted the per-owner cap at roughly a quarter of the estimated spend. - CLI: `crabbox list` streams and shows live leases by default; `--retained` includes kept terminal leases. Add `--jsonl` streaming and uncapped `--all` lease lookups while preserving the buffered `--json` array. +- Direct cleanup now expires `keep=true` leases and reduces running/provisioning expiry grace from 12 hours to 1 hour; unowned machines remain untouched. ### Added - Blacksmith Testbox: configure the native total readiness wait (queue and hydration) through `blacksmith.waitTimeout`, `CRABBOX_BLACKSMITH_WAIT_TIMEOUT`, or `--blacksmith-wait-timeout`, preserving the native 15-minute default when unset. - Serve leases can fall back to an owner-authorized, end-to-end encrypted coordinator relay when LAN and tailnet routes are unavailable. - Coordinator: reconcile retained terminal AWS leases from the current index with scoped cleanup proof, bounded resumable passes, opt-in owned deletion, and `crabbox admin retained-reconcile` status. +- Direct Linux guest expiry guards across shared-bootstrap providers, plus `directLeases: deny` and trusted provider exceptions to require coordinator allocation. - Serve agents advertise LAN and tailnet front-door candidates, with pinned-key LAN-first client selection and per-lease route caching. - Portal: show owners their donated serve machines, online status, capacity, allocated leases, and runtime details. diff --git a/docs/commands/cleanup.md b/docs/commands/cleanup.md index da2e7e2a21..a8b40b2c80 100644 --- a/docs/commands/cleanup.md +++ b/docs/commands/cleanup.md @@ -81,20 +81,29 @@ What cleanup does depends on the selected provider: ### Deletion decisions for direct machines -Selection is label-driven. Cleanup reads the `keep`, `state`, `expires_at`, and +Selection is label-driven. Cleanup requires `crabbox=true` and reads `state`, `expires_at`, and `ttl` labels written when the machine was created. The decision is conservative: -- skip machines labeled `keep=true`, even after recorded expiry; +- `keep=true` does not exempt a machine from expiry; it only suppresses automatic release after `run`; - for `running` or `provisioning` machines, skip until well past expiry — delete - only once the expiry time plus a 12-hour stale window has elapsed; + only once the expiry time plus a 1-hour stale window has elapsed; - for `leased`, `ready`, or `active` machines, delete once expired; - always delete machines in `failed`, `released`, or `expired` states; - for any other machine, delete only if `expires_at`/`ttl` parses and has passed; skip if the expiry label is missing or still in the future. -Resources without these labels are skipped (`reason=missing labels`), so +Resources without `crabbox=true` are skipped (`reason=missing labels`), so non-Crabbox machines are never touched. +Supported direct Linux guests also run an expiry guard every two minutes. +GCP self-deletes; AWS powers off and terminates; other covered providers power +off only. Stopped machines can remain billable (including Hetzner and Azure), +so run cleanup to delete them. See the +[provider coverage table](../features/lifecycle-cleanup.md#guest-side-expiry-guard). + +`directLeases: deny` blocks new remote allocations, not cleanup, stop, list, or +status of existing machines. + ### Namespace local cleanup For `provider=namespace-devbox`, cleanup removes only Crabbox-owned Namespace SSH @@ -115,12 +124,12 @@ For direct machine providers, each candidate prints one decision line. `--dry-ru prints the same lines but makes no provider calls: ```text -skip server id=12345 name=crabbox-blue-lobster reason=keep=true +skip server id=12345 name=crabbox-blue-lobster reason=not expired delete server id=67890 name=crabbox-amber-crab stop server id=11223 name=crabbox-green-heron ``` -`skip` lines include a `reason=` (for example `keep=true`, `state=running`, +`skip` lines include a `reason=` (for example `state=running`, `missing expires_at`, `not expired`). Without `--dry-run`, each `delete` line is followed by the actual provider delete call; a failed delete returns the provider error and stops the sweep. Stop-only providers such as Hostinger print `stop` diff --git a/docs/features/configuration.md b/docs/features/configuration.md index 9f23022ebe..bfcfa77d9c 100644 --- a/docs/features/configuration.md +++ b/docs/features/configuration.md @@ -25,6 +25,42 @@ repository remain valid. An inherited user-config or environment credential is not sent to a repository-defined destination unless the operator explicitly overrides that destination through its environment variable or CLI flag. +## Direct lease policy + +`directLeases: allow|deny` defaults to `allow`. Set it in trusted user/system +configuration to require coordinator-managed remote allocation on this machine: + +```yaml +directLeases: deny +directLeaseAllow: [hetzner] +``` + +`CRABBOX_DIRECT_LEASES` overrides the policy; `CRABBOX_DIRECT_LEASE_ALLOW` +overrides the allowlist with comma-separated provider names. Names and aliases +resolve to canonical provider names. Repository configuration may tighten the +policy to `deny`; it cannot override an inherited deny with `allow` or add +allowlist entries. Repository `directLeaseAllow` is ignored. Environment is a +trusted operator override and can explicitly restore `allow`. + +When denied, creation fails before configuring or calling the direct provider: + +```text +direct-provider leases are disabled (directLeases: deny); configure a coordinator or add to directLeaseAllow +``` + +This applies to warmup/prewarm, new runs, fixed lease IDs, checkpoint forks, +and shards. Stop, cleanup, status, list, and runs against existing leases remain +available. A stale reused lease cannot be replaced with a new direct allocation. +A coordinator URL exempts only providers actually allocating through the +coordinator; registered-direct mode and providers without coordinator support +still obey the policy. + +Providers that provision nothing remote are always exempt: `ssh`, +`local-container`, `docker-sandbox`, `apple-container`, `apple-machine`, +`apple-vm`, `anthropic-sandbox-runtime`, `mxc`, `multipass`, `firecracker`, +`lume`, `tart`, `hyperv`, and `windows-sandbox`. Providers capable of selecting +remote hosts, such as Parallels and Incus, are not automatically exempt. + ## Precedence ```text @@ -32,8 +68,8 @@ flags > env > repo-local crabbox.yaml/.crabbox.yaml > user config > defaults ``` Lowest precedence is applied first: defaults, then user config, then repo -config, then env vars, then flags. Each layer only overrides fields that are -explicitly set; unset fields fall through to the layer below. +config, then env vars, then flags. Except for the tightening-only repository direct-lease policy above, each layer +only overrides fields that are explicitly set; unset fields fall through to the layer below. Lease-creating commands validate OS selector compatibility after resolving the provider, target, and Windows mode from all layers. For example, diff --git a/docs/features/lifecycle-cleanup.md b/docs/features/lifecycle-cleanup.md index 6242604694..f6afe415d1 100644 --- a/docs/features/lifecycle-cleanup.md +++ b/docs/features/lifecycle-cleanup.md @@ -67,8 +67,8 @@ Read this when: A lease holds a remote box until it is released or expires. Two independent paths reclaim the underlying resources: the **brokered** path, owned by the -coordinator, and the **direct** path, owned by the local CLI (and, for GCP, a -guest-side guard). Which one applies depends on whether the provider runs +coordinator, and the **direct** path, owned by the local CLI and supported Linux +guest-side guards. Which one applies depends on whether the provider runs through a coordinator. ## Brokered lifecycle @@ -587,36 +587,58 @@ Use `--dry-run` to print what would be deleted without touching anything. The sweep is conservative; for each candidate machine `shouldCleanupServer` (`internal/cli/pool.go`) decides from the machine's Crabbox labels: -- skip machines with no labels, or labeled `keep=true`, even after recorded expiry; -- `running` / `provisioning`: delete only when stale — past `expires_at` plus a - 12-hour safety window; -- `leased` / `ready` / `active`: delete once past `expires_at`; -- `failed` / `released` / `expired`: delete; -- otherwise: delete once past `expires_at`, skip if `expires_at` is missing or - still in the future. - -For this to work, every direct-provider machine must carry Crabbox labels/tags -(at least `crabbox`, `state`, and `expires_at`) so the sweep can identify owned -resources without touching unrelated infrastructure. - -### GCP guest-side expiry guard - -A direct GCP lease can outlive the local CLI that created it — if `cleanup` -never runs, the VM would leak. To guard against this, direct GCP leases install -a self-deleting guard (`cloudInitGCPExpiryGuardFiles` in -`internal/cli/bootstrap.go`): a systemd timer runs every 2 minutes, reads the -instance's own labels via the GCP metadata server, and deletes the instance when -it is clearly expired. Unlike the manual CLI sweep, this automatic expiry path -also reaps kept leases: - -- exits unless `crabbox=true` and the provider label is absent or `gcp`; -- retains `keep=true` machines until their recorded expiry, leaving those without valid expiry metadata alone; -- `failed` / `released` / `expired`: delete; -- `running` / `provisioning`: delete only past `expires_at` plus 12 hours; -- `leased` / `ready` / `active` (and unlabeled state): delete once past - `expires_at`. - -So an expired GCP box can reclaim itself even if the operator's machine is gone. +- skip machines unless `crabbox=true`; unrelated or unlabeled machines are never reclaimed; +- `running` / `provisioning`: reclaim only after `expires_at` plus **1 hour**; +- `leased` / `ready` / `active`: reclaim after `expires_at`; +- `failed` / `released` / `expired`: reclaim immediately; +- other states: reclaim only after a valid expiry; missing or invalid expiry is left alone. + +`keep=true` only suppresses automatic release after `run`. It does **not** +exempt a direct lease from expiry, matching brokered leases. This changes the +old direct sweep behavior, which skipped kept leases and allowed a 12-hour +running/provisioning grace. Cleanup retains its legacy `ttl` expiry-label +fallback. New bootstraps use `expires_at` (Unix seconds). + +### Guest-side expiry guard + +Direct Linux VMs on the shared cloud-init path install one root-owned script, +`/usr/local/sbin/crabbox-expiry-guard`, and a systemd service/timer. The timer +runs every two minutes, beginning two minutes after boot, before optional +bootstrap work completes. It uses the same decision cases as the CLI sweep; +tests execute these cases through both Go and the shell guard. No provider +credentials are stored in guard files. Missing, ambiguous, or unavailable +ownership metadata fails closed. Expiry-based decisions require a valid deadline; +terminal `failed` / `released` / `expired` states remain immediately reclaimable. + +| Direct Linux provider | Deadline source | Reclaim action | +| --- | --- | --- | +| GCP | Current labels fetched through the metadata server's service-account token and Compute API (existing mechanism) | Delete this VM through the Compute API | +| AWS | Instance tags via IMDSv2; new direct Linux launches enable instance metadata tags | Power off; `InstanceInitiatedShutdownBehavior=terminate` terminates the instance | +| Azure | IMDS `compute/tagsList` | `systemctl poweroff` | +| Hetzner | Root-owned `/etc/crabbox/lease-expiry` | `systemctl poweroff` | +| DigitalOcean, Vultr, Linode | Root-owned `/etc/crabbox/lease-expiry` | `systemctl poweroff` | +| OVHcloud, Scaleway, Nebius, Lambda, Tencent Cloud | Root-owned `/etc/crabbox/lease-expiry` | `systemctl poweroff` | +| Incus, Multipass, Firecracker | Root-owned `/etc/crabbox/lease-expiry` | `systemctl poweroff` | + +Powering off stops the workload; it does **not** guarantee that billing stops. +Providers that bill stopped machines, including Hetzner, still require +`crabbox cleanup` to delete them. Azure guest shutdown does not deallocate its +VM. Provider disks and other retained resources can also remain billable. +GCP retains its existing self-delete permissions requirement. + +File-backed bootstraps write exactly four nonsecret fields: `crabbox`, `state`, +`keep`, and `expires_at`. `RefreshGuestLeaseExpiry(ctx, target, fields)` in +`internal/cli/guest_expiry.go` (internal implementation `refreshGuestLeaseExpiry`) +is the hook for a file-backed provider's lease-lifetime update: it validates the +fields and writes mode 0600 through the existing SSH transport, publishing by +atomic rename in `/etc/crabbox`. Ordinary CLI state touches and heartbeats refresh this file as well as provider +labels. Other deadline updates must use this hook, not only change provider labels. No new `extend` command is introduced here. + +Brokered leases never install this guard; the coordinator owns their lifecycle. +Windows and macOS targets do not install it. Providers using prepared templates, +custom bootstrap, hosted sandboxes, delegated execution, or static hosts rather +than the shared cloud-init path also do not install it. The explicit uncovered +provider list is in [Providers](providers.md#direct-lease-expiry-coverage). ## Claims and `--reclaim` diff --git a/docs/features/providers.md b/docs/features/providers.md index bb9e0fe70d..ca0e76b6d2 100644 --- a/docs/features/providers.md +++ b/docs/features/providers.md @@ -54,6 +54,40 @@ selection metadata, so registration and documentation drift fail the docs gate. `crabbox providers --json` remains the low-level compiled spec. `crabbox doctor` checks whether the selected provider is usable from the current environment. +## Direct lease expiry coverage + +The shared Linux guest guard covers **AWS, Azure, GCP, Hetzner, DigitalOcean, +Vultr, Linode, OVHcloud, Scaleway, Nebius, Lambda, Tencent Cloud, Incus, +Multipass, and Firecracker**. See the [deadline source and reclaim action +table](lifecycle-cleanup.md#guest-side-expiry-guard). Coordinator-managed leases +and Windows/macOS targets are excluded. + +The following providers do not use that shared Linux bootstrap and receive no +guard (some have provider-native expiry instead): + +- `agent-sandbox`, `anthropic-sandbox-runtime`, `apple-container`, `apple-machine`, `apple-vm`, `ascii-box`, `aws-lambda-microvm`, `azure-dynamic-sessions`. +- `azure-sandbox`, `blacksmith-testbox`, `blaxel`, `boxd`, `cloud-run-sandbox`, `cloudflare`, `cloudflare-dynamic-workers`, `cloudflare-sandbox`. +- `coder`, `codesandbox`, `crownest`, `cua`, `cubesandbox`, `daytona`, `docker-sandbox`, `e2b`. +- `exe-dev`, `external`, `fastapi-cloud`, `freestyle`, `github-codespaces`, `hostinger`, `hyperv`, `islo`. +- `kubevirt`, `local-container`, `lume`, `machine0`, `modal`, `morph`, `mxc`, `namespace-devbox`. +- `namespace-instance`, `nomad`, `nvidia-brev`, `opencomputer`, `opensandbox`, `orgo`, `parallels`, `phala`. +- `proxmox`, `railway`, `runpod`, `sealos-devbox`, `semaphore`, `serve`, `smolvm`, `sprites`. +- `ssh`, `superserve`, `tart`, `tenki`, `tensorlake`, `unikraft-cloud`, `upstash-box`, `vast`. +- `vercel-sandbox`, `wandb`, `windows-sandbox`, `xcp-ng`. + +In particular, prepared/custom-bootstrapped VMs such as Proxmox, XCP-ng, +Parallels, and Apple VM remain uncovered; their existing provider cleanup still +applies. This guard does not add a bootstrap path to hosted sandboxes or static +hosts. A guest shutdown is not deletion: stopped machines can remain billable +and still need `crabbox cleanup`. + +To forbid new direct remote allocations, set +[`directLeases: deny`](configuration.md#direct-lease-policy), with optional +trusted `directLeaseAllow` exceptions. The always-exempt local/static set is +`ssh`, `local-container`, `docker-sandbox`, `apple-container`, `apple-machine`, +`apple-vm`, `anthropic-sandbox-runtime`, `mxc`, `multipass`, `firecracker`, +`lume`, `tart`, `hyperv`, and `windows-sandbox`. + ## Machine classes An explicitly selected canonical `--class tiny|small|standard|fast|large|beast` maps to an @@ -166,7 +200,8 @@ describes retry order. `crabbox list` marks brokered machines as `orphan=no-active-lease` when their provider label references a lease no longer active in the coordinator. This is an -operator hint only; `keep=true` machines are never deleted automatically. +operator hint only; listing does not delete machines. Recorded expiry still +applies regardless of `keep=true`. The structured quota preflight and `provisioningAttempts` metadata belong to the brokered Worker path; direct AWS fallback can still retry provider types but diff --git a/docs/providers/gcp.md b/docs/providers/gcp.md index 834e094fdc..39f3bcc076 100644 --- a/docs/providers/gcp.md +++ b/docs/providers/gcp.md @@ -466,13 +466,14 @@ Three independent safety nets enforce expiry: - Direct GCP VMs set Compute Engine `maxRunDuration` with termination action `DELETE`, so the TTL hard cap is enforced by the platform. -- Each VM installs a guest-side `crabbox-gcp-expiry-guard` systemd timer that - reads live instance labels through the metadata service and Compute API, then - self-deletes expired leases, including kept warmups, when the attached service - account can delete the VM. New VMs receive this behavior; existing VMs retain - their installed guard and must be recreated to receive the updated idle reaper. - Manual `crabbox cleanup` continues to skip kept leases; use explicit `stop` - to remove an existing kept VM. +- New direct Linux VMs install the shared `crabbox-expiry-guard` systemd timer. + It reads live instance labels through the metadata service and Compute API, + then self-deletes expired leases, including kept warmups, when the attached + service account can delete the VM. The timer runs every two minutes; running + and provisioning states have a one-hour expiry grace. Existing VMs retain + their installed guard and must be recreated to receive this version. + Manual `crabbox cleanup` also expires kept leases. Brokered leases do not + install this guard; their coordinator owns expiry. - Cleanup removes stale local GCP claim files whose lease IDs no longer appear in provider inventory. diff --git a/internal/cli/actions.go b/internal/cli/actions.go index a61a17ffe7..bef5e8e311 100644 --- a/internal/cli/actions.go +++ b/internal/cli/actions.go @@ -185,7 +185,7 @@ func (a App) actionsHydrate(ctx context.Context, args []string) (err error) { } defer stopHeartbeat() } else if sshBackend, ok := backend.(SSHLeaseBackend); ok { - _, err := sshBackend.Touch(ctx, TouchRequest{Lease: LeaseTarget{Server: server, SSH: target, LeaseID: leaseID}, State: blank(server.Labels["state"], "ready"), IdleTimeout: cfg.IdleTimeout}) + _, err := touchLeaseWithGuestExpiry(ctx, sshBackend, cfg, TouchRequest{Lease: LeaseTarget{Server: server, SSH: target, LeaseID: leaseID}, State: blank(server.Labels["state"], "ready"), IdleTimeout: cfg.IdleTimeout}) if err != nil { fmt.Fprintf(a.Stderr, "warning: touch failed for %s: %v\n", leaseID, err) } diff --git a/internal/cli/aws.go b/internal/cli/aws.go index bb96eed81b..9b28d3ad91 100644 --- a/internal/cli/aws.go +++ b/internal/cli/aws.go @@ -783,6 +783,7 @@ func (c *AWSClient) createServer(ctx context.Context, cfg Config, publicKey, lea } } applyAWSRunInstanceTargetOptions(input, cfg) + applyAWSGuestExpiryOptions(input, cfg) if cfg.TargetOS == targetMacOS { hostID := cfg.HostID if hostID == "" { @@ -1851,3 +1852,13 @@ func parsePort32(port string) (int32, bool) { } return int32(n), true } + +// Only direct Linux guests own their expiry. IMDSv2 exposes the same launch +// tags used by cleanup; guest poweroff must terminate rather than stop billing. +func applyAWSGuestExpiryOptions(input *ec2.RunInstancesInput, cfg Config) { + if guestExpiryAdapter(cfg) != "aws" { + return + } + input.InstanceInitiatedShutdownBehavior = types.ShutdownBehaviorTerminate + input.MetadataOptions = &types.InstanceMetadataOptionsRequest{HttpEndpoint: types.InstanceMetadataEndpointStateEnabled, HttpTokens: types.HttpTokensStateRequired, InstanceMetadataTags: types.InstanceMetadataTagsStateEnabled} +} diff --git a/internal/cli/aws_fixed_create_intent_test.go b/internal/cli/aws_fixed_create_intent_test.go index 4ff6fb9624..3a5090e6f6 100644 --- a/internal/cli/aws_fixed_create_intent_test.go +++ b/internal/cli/aws_fixed_create_intent_test.go @@ -224,6 +224,7 @@ func TestFixedAWSCreateIntentClassifiesEveryExportedConfigField(t *testing.T) { `) classify("command default resolved into the explicit create request Keep field", `WarmupKeep`) classify("coordinator-only lifetime enrollment; direct creation rejects canary flags", `LifetimePolicy`) + classify("local allocation admission policy, not remote resource intent", `DirectLeases DirectLeaseAllow`) configType := reflect.TypeOf(Config{}) known := map[string]bool{} diff --git a/internal/cli/bootstrap.go b/internal/cli/bootstrap.go index df1725b6c7..35d268e223 100644 --- a/internal/cli/bootstrap.go +++ b/internal/cli/bootstrap.go @@ -530,8 +530,8 @@ func cloudInitOptionalReadyChecks(cfg Config) string { func cloudInitOptionalWriteFiles(cfg Config) string { var parts []string - if cfg.Provider == "gcp" { - parts = append(parts, cloudInitGCPExpiryGuardFiles()) + if files := cloudInitGuestExpiryGuardFiles(cfg); files != "" { + parts = append(parts, files) } if cfg.Desktop && isWaylandDesktopEnv(cfg.DesktopEnv) { parts = append(parts, cloudInitWaylandDesktopWriteFiles(normalizedDesktopEnv(cfg.DesktopEnv))) @@ -670,11 +670,7 @@ func cloudInitWaylandDesktopWriteFiles(desktopEnv string) string { } func cloudInitEarlyBootstrap(cfg Config) string { - // The expiry timer needs only networking; it must not defer default GCP readiness. - if cfg.Provider == "gcp" { - return cloudInitGCPExpiryGuardBootstrap() - } - return "" + return cloudInitGuestExpiryGuardBootstrap(cfg) } func cloudInitOptionalBootstrap(cfg Config) string { @@ -883,103 +879,6 @@ func indentCloudInitRuncmd(script string) string { return strings.Join(lines, "") } -func cloudInitGCPExpiryGuardFiles() string { - return ` - path: /usr/local/sbin/crabbox-gcp-expiry-guard - permissions: '0755' - content: | - #!/usr/bin/env bash - set -euo pipefail - metadata() { - curl -fsS -H 'Metadata-Flavor: Google' "http://metadata.google.internal/computeMetadata/v1/$1" - } - project="$(metadata project/project-id 2>/dev/null || true)" - name="$(metadata instance/name 2>/dev/null || true)" - zone_path="$(metadata instance/zone 2>/dev/null || true)" - zone="${zone_path##*/}" - token_json="$(metadata instance/service-accounts/default/token 2>/dev/null || true)" - token="$(printf '%s' "$token_json" | jq -r '.access_token // empty' 2>/dev/null || true)" - if [ -z "$project" ] || [ -z "$name" ] || [ -z "$zone" ] || [ -z "$token" ]; then - exit 0 - fi - instance="$(curl -fsS -H "Authorization: Bearer $token" "https://compute.googleapis.com/compute/v1/projects/$project/zones/$zone/instances/$name" 2>/dev/null || true)" - if [ -z "$instance" ]; then - exit 0 - fi - label() { - printf '%s' "$instance" | jq -r --arg key "$1" '.labels[$key] // empty' - } - crabbox="$(label crabbox)" - provider="$(label provider)" - if [ "$crabbox" != "true" ] || { [ -n "$provider" ] && [ "$provider" != "gcp" ]; }; then - exit 0 - fi - keep="$(label keep | tr '[:upper:]' '[:lower:]')" - expires_at="$(label expires_at)" - state="$(label state | tr '[:upper:]' '[:lower:]')" - now="$(date -u +%s)" - if [ "$keep" = "true" ]; then - if ! [[ "$expires_at" =~ ^[1-9][0-9]{0,17}$ ]] || [ "$now" -le "$expires_at" ]; then - exit 0 - fi - fi - delete=false - case "$state" in - failed|released|expired) - delete=true - ;; - running|provisioning) - if [[ "$expires_at" =~ ^[0-9]+$ ]] && [ "$now" -gt $((expires_at + 43200)) ]; then - delete=true - fi - ;; - leased|ready|active|"") - if [[ "$expires_at" =~ ^[0-9]+$ ]] && [ "$now" -gt "$expires_at" ]; then - delete=true - fi - ;; - *) - if [[ "$expires_at" =~ ^[0-9]+$ ]] && [ "$now" -gt "$expires_at" ]; then - delete=true - fi - ;; - esac - if [ "$delete" != "true" ]; then - exit 0 - fi - logger -t crabbox-gcp-expiry-guard "deleting expired lease=$(label lease) state=${state:-unknown} expires_at=${expires_at:-missing}" - curl -fsS -X DELETE -H "Authorization: Bearer $token" "https://compute.googleapis.com/compute/v1/projects/$project/zones/$zone/instances/$name" >/dev/null || true - - path: /etc/systemd/system/crabbox-gcp-expiry-guard.service - permissions: '0644' - content: | - [Unit] - Description=Crabbox GCP direct lease expiry guard - After=network-online.target - Wants=network-online.target - - [Service] - Type=oneshot - ExecStart=/usr/local/sbin/crabbox-gcp-expiry-guard - - path: /etc/systemd/system/crabbox-gcp-expiry-guard.timer - permissions: '0644' - content: | - [Unit] - Description=Run Crabbox GCP direct lease expiry guard - - [Timer] - OnBootSec=2min - OnUnitActiveSec=2min - RandomizedDelaySec=30s - Persistent=true - - [Install] - WantedBy=timers.target` -} - -func cloudInitGCPExpiryGuardBootstrap() string { - return ` systemctl daemon-reload - systemctl enable --now crabbox-gcp-expiry-guard.timer` -} - func cloudInitTailscaleBootstrap(cfg Config) string { authKey := strings.TrimSpace(cfg.Tailscale.AuthKey) hostname := strings.TrimSpace(cfg.Tailscale.Hostname) diff --git a/internal/cli/bootstrap_test.go b/internal/cli/bootstrap_test.go index 9fb7a5f443..cbf71a6160 100644 --- a/internal/cli/bootstrap_test.go +++ b/internal/cli/bootstrap_test.go @@ -146,13 +146,13 @@ func TestCloudInitGCPInstallsExpiryGuard(t *testing.T) { cfg.Provider = "gcp" got := cloudInit(cfg, "ssh-ed25519 test") for _, want := range []string{ - "/usr/local/sbin/crabbox-gcp-expiry-guard", + "/usr/local/sbin/crabbox-expiry-guard", "computeMetadata/v1/$1", "compute/v1/projects/$project/zones/$zone/instances/$name", - "crabbox-gcp-expiry-guard.service", - "crabbox-gcp-expiry-guard.timer", + "crabbox-expiry-guard.service", + "crabbox-expiry-guard.timer", "OnUnitActiveSec=2min", - "systemctl enable --now crabbox-gcp-expiry-guard.timer", + "systemctl enable --now crabbox-expiry-guard.timer", "expires_at", "failed|released|expired", "running|provisioning", @@ -162,7 +162,7 @@ func TestCloudInitGCPInstallsExpiryGuard(t *testing.T) { } } early, final, found := strings.Cut(got, "\nruncmd:") - if !found || !strings.Contains(early, "touch /var/lib/crabbox/bootstrapped") || strings.Contains(final, "crabbox-gcp-expiry-guard.timer") { + if !found || !strings.Contains(early, "touch /var/lib/crabbox/bootstrapped") || strings.Contains(final, "crabbox-expiry-guard.timer") { t.Fatal("default GCP expiry guard must be armed before early readiness") } diff --git a/internal/cli/checkpoint.go b/internal/cli/checkpoint.go index 7f333ba09d..4a57c09332 100644 --- a/internal/cli/checkpoint.go +++ b/internal/cli/checkpoint.go @@ -1326,6 +1326,9 @@ func (a App) checkpointFork(ctx context.Context, args []string) (err error) { if *jsonOut { operationApp.Stdout = a.Stderr } + if err := validateDirectLeaseCreation(cfg); err != nil { + return err + } backend, err := loadBackend(cfg, runtimeForApp(operationApp)) if err != nil { return err @@ -1909,6 +1912,9 @@ func (a App) checkpointForkParallelsSnapshot(ctx context.Context, fs *flag.FlagS if jsonOut { operationApp.Stdout = a.Stderr } + if err := validateDirectLeaseCreation(cfg); err != nil { + return err + } backend, err := loadBackend(cfg, runtimeForApp(operationApp)) if err != nil { return err diff --git a/internal/cli/config.go b/internal/cli/config.go index d9de824c97..f9a3fb1183 100644 --- a/internal/cli/config.go +++ b/internal/cli/config.go @@ -18,6 +18,9 @@ import ( ) type Config struct { + DirectLeases string + DirectLeaseAllow []string + guestLeaseExpiry *GuestLeaseExpiryFields LifetimePolicy *LeaseLifetimeRequest `json:"-" yaml:"-"` RecordLocal bool `json:"-" yaml:"-"` Profile string @@ -1581,6 +1584,7 @@ func baseConfig() Config { multipassImage, _ := osImageDefaultMultipassImage(osImage) return Config{ Profile: "default", + DirectLeases: "allow", Provider: provider, providerSelectionSource: providerSelectionCompiledDefault, TargetOS: "linux", @@ -1745,6 +1749,8 @@ type ServeConfig struct { } type fileConfig struct { + DirectLeases string `yaml:"directLeases,omitempty"` + DirectLeaseAllow *[]string `yaml:"directLeaseAllow,omitempty"` Serve *ServeConfig `yaml:"serve,omitempty"` Claims *fileClaimsConfig `yaml:"claims,omitempty"` History *fileLocalHistoryPolicy `yaml:"history,omitempty"` @@ -2549,6 +2555,9 @@ func applyFileConfigWithTrust(cfg *Config, file fileConfig, trusted bool) error } func applyFileConfigWithTrustAndProviderSource(cfg *Config, file fileConfig, trusted bool, providerSource providerSelectionSource) error { + if err := applyDirectLeaseFileConfig(cfg, file, trusted); err != nil { + return err + } if file.Serve != nil { cfg.Serve = *file.Serve } @@ -3935,6 +3944,20 @@ func applyNonNegativeLeaseDuration(target *time.Duration, value string) bool { } func applyEnv(cfg *Config) error { + if value := os.Getenv("CRABBOX_DIRECT_LEASES"); strings.TrimSpace(value) != "" { + policy, err := normalizeDirectLeases(value) + if err != nil { + return err + } + cfg.DirectLeases = policy + } + if value, ok := os.LookupEnv("CRABBOX_DIRECT_LEASE_ALLOW"); ok { + allow, err := normalizeDirectLeaseAllow(strings.Split(value, ",")) + if err != nil { + return err + } + cfg.DirectLeaseAllow = allow + } if value, ok := os.LookupEnv("CRABBOX_SERVE_ADVERTISE_ADDRESSES"); ok { cfg.Serve.AdvertiseAddresses = strings.Split(value, ",") } diff --git a/internal/cli/config_cmd.go b/internal/cli/config_cmd.go index b928eb29b0..7e0ce47355 100644 --- a/internal/cli/config_cmd.go +++ b/internal/cli/config_cmd.go @@ -132,6 +132,8 @@ func configShowView(cfg Config) map[string]any { "providerStatus": providerConfigStatus(cfg), "profile": cfg.Profile, "provider": provider, + "directLeases": cfg.DirectLeases, + "directLeaseAllow": cfg.DirectLeaseAllow, "providerSelected": providerSelected, "providerSource": cfg.providerSelectionSource, "target": cfg.TargetOS, @@ -683,6 +685,7 @@ func writeConfigShowText(w io.Writer, cfg Config) error { if err := layout.writeRemaining(w); err != nil { return err } + fmt.Fprintf(w, "directLeases=%s directLeaseAllow=%s\n", cfg.DirectLeases, strings.Join(cfg.DirectLeaseAllow, ",")) writeProviderConfigStatus(w, providerConfigStatus(cfg)) return output.err } diff --git a/internal/cli/config_show_projection.go b/internal/cli/config_show_projection.go index b2545cbd1a..14fd5dbf3b 100644 --- a/internal/cli/config_show_projection.go +++ b/internal/cli/config_show_projection.go @@ -38,7 +38,7 @@ func ConfigShowSecretState(value string) string { return tokenState(value) } // Names only: no legacy value projection or environment reads are needed to // protect existing text slots. Retire a name only when its legacy row migrates. -const legacyConfigShowTextLabels = "config provider lease broker access_auth ssh sync env run warmup claims capacity actions phala namespace namespace_instance morph e2b cubesandbox blaxel nomad ascii_box superserve machine0 cloudflare fastapi_cloud cloudflare_dynamic_workers cloudflare_sandbox results cache jobs aws_lambda_microvm github_codespaces lambda nebius ovh scaleway tencentcloud azure_dynamic_sessions proxmox xcp_ng inspection provider_status" +const legacyConfigShowTextLabels = "config provider directLeases lease broker access_auth ssh sync env run warmup claims capacity actions phala namespace namespace_instance morph e2b cubesandbox blaxel nomad ascii_box superserve machine0 cloudflare fastapi_cloud cloudflare_dynamic_workers cloudflare_sandbox results cache jobs aws_lambda_microvm github_codespaces lambda nebius ovh scaleway tencentcloud azure_dynamic_sessions proxmox xcp_ng inspection provider_status" func collectProviderConfigShowSections(cfg Config) ([]ProviderConfigShowSection, error) { return collectProviderConfigShowSectionsFrom(cfg, registeredProviders()) diff --git a/internal/cli/config_test.go b/internal/cli/config_test.go index 51f9f87d9f..ee191334b1 100644 --- a/internal/cli/config_test.go +++ b/internal/cli/config_test.go @@ -722,6 +722,8 @@ func clearConfigEnv(t *testing.T) { isolateTestUserDirs(t) for _, key := range []string{ "CRABBOX_WARMUP_KEEP", + "CRABBOX_DIRECT_LEASES", + "CRABBOX_DIRECT_LEASE_ALLOW", "CRABBOX_CLAIMS_AUTO_PRUNE", "CRABBOX_SYNC_SOURCE", "CRABBOX_SYNC_GIT_SEED_SOURCE", diff --git a/internal/cli/connect.go b/internal/cli/connect.go index 51974b06da..a30ebd06eb 100644 --- a/internal/cli/connect.go +++ b/internal/cli/connect.go @@ -60,7 +60,7 @@ func (a App) startInteractiveSSHLeaseActivity(ctx context.Context, cfg Config, l fmt.Fprintf(a.Stderr, "warning: provider=%s does not support lease touch\n", backend.Spec().Name) return stopInteractiveSSHHeartbeat(stopHeartbeat) } - if touched, touchErr := sshBackend.Touch(ctx, TouchRequest{Lease: lease, State: "running", IdleTimeout: cfg.IdleTimeout}); touchErr == nil { + if touched, touchErr := touchLeaseWithGuestExpiry(ctx, sshBackend, cfg, TouchRequest{Lease: lease, State: "running", IdleTimeout: cfg.IdleTimeout}); touchErr == nil { lease.Server = touched } else { fmt.Fprintf(a.Stderr, "warning: direct touch state=running: %v\n", touchErr) @@ -71,7 +71,7 @@ func (a App) startInteractiveSSHLeaseActivity(ctx context.Context, cfg Config, l } cleanupCtx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() - if touched, touchErr := sshBackend.Touch(cleanupCtx, TouchRequest{Lease: lease, State: "ready", IdleTimeout: cfg.IdleTimeout}); touchErr == nil { + if touched, touchErr := touchLeaseWithGuestExpiry(cleanupCtx, sshBackend, cfg, TouchRequest{Lease: lease, State: "ready", IdleTimeout: cfg.IdleTimeout}); touchErr == nil { lease.Server = touched } else { fmt.Fprintf(a.Stderr, "warning: direct touch state=ready: %v\n", touchErr) diff --git a/internal/cli/direct_leases.go b/internal/cli/direct_leases.go new file mode 100644 index 0000000000..5e6c9c20ed --- /dev/null +++ b/internal/cli/direct_leases.go @@ -0,0 +1,94 @@ +package cli + +import ( + "fmt" + "strings" +) + +func normalizeDirectLeases(value string) (string, error) { + switch strings.ToLower(strings.TrimSpace(value)) { + case "", "allow": + return "allow", nil + case "deny": + return "deny", nil + default: + return "", fmt.Errorf("directLeases must be allow or deny") + } +} + +func normalizeDirectLeaseAllow(values []string) ([]string, error) { + var result []string + seen := map[string]bool{} + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + provider, err := ProviderFor(value) + if err != nil { + return nil, err + } + name := provider.Spec().Name + if !seen[name] { + result = append(result, name) + seen[name] = true + } + } + return result, nil +} + +func applyDirectLeaseFileConfig(cfg *Config, file fileConfig, trusted bool) error { + if file.DirectLeases != "" { + policy, err := normalizeDirectLeases(file.DirectLeases) + if err != nil { + return err + } + if trusted || policy == "deny" { + cfg.DirectLeases = policy + } + } + // Repository automation cannot grant a machine-policy exception, even if + // it also sets deny in this layer. Only the operator owns the allowlist. + if trusted && file.DirectLeaseAllow != nil { + allow, err := normalizeDirectLeaseAllow(*file.DirectLeaseAllow) + if err != nil { + return err + } + cfg.DirectLeaseAllow = allow + } + return nil +} + +// Local-only backends never allocate remote infrastructure. Backends capable +// of selecting a remote host (for example parallels/incus) are not exempt. +func directLeaseExempt(provider string) bool { + switch provider { + case "ssh", "local-container", "docker-sandbox", "multipass", "firecracker", "lume", "tart", "hyperv", "windows-sandbox", "apple-container", "apple-machine", "apple-vm", "anthropic-sandbox-runtime", "mxc": + return true + } + return false +} + +func validateDirectLeaseCreation(cfg Config) error { + policy, err := normalizeDirectLeases(cfg.DirectLeases) + if err != nil { + return err + } + if policy != "deny" { + return nil + } + provider, err := ProviderFor(cfg.Provider) + if err != nil { + return err + } + name := provider.Spec().Name + if directLeaseExempt(name) || ShouldUseCoordinator(cfg, provider.Spec()) { + return nil + } + for _, allowed := range cfg.DirectLeaseAllow { + if allowed == name { + return nil + } + } + return Exit(2, "direct-provider leases are disabled (directLeases: deny); configure a coordinator or add %s to directLeaseAllow", name) +} diff --git a/internal/cli/direct_leases_test.go b/internal/cli/direct_leases_test.go new file mode 100644 index 0000000000..0164782757 --- /dev/null +++ b/internal/cli/direct_leases_test.go @@ -0,0 +1,141 @@ +package cli + +import ( + "bytes" + "context" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestDirectLeasePolicy(t *testing.T) { + for _, tc := range []struct { + name, provider, policy, coordinator string + allow []string + registered, denied bool + }{ + {name: "default", provider: "aws"}, + {name: "allow", provider: "aws", policy: "allow"}, + {name: "deny", provider: "aws", policy: "deny", denied: true}, + {name: "allowlist", provider: "aws", policy: "deny", allow: []string{"aws"}}, + {name: "other allowlist", provider: "aws", policy: "deny", allow: []string{"gcp"}, denied: true}, + {name: "broker", provider: "aws", policy: "deny", coordinator: "https://coordinator.example"}, + {name: "registered direct", provider: "aws", policy: "deny", coordinator: "https://coordinator.example", registered: true, denied: true}, + {name: "unsupported broker", provider: "run-env-profile-test", policy: "deny", coordinator: "https://coordinator.example", denied: true}, + {name: "static", provider: "ssh", policy: "deny"}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := Config{Provider: tc.provider, DirectLeases: tc.policy, DirectLeaseAllow: tc.allow, Coordinator: tc.coordinator} + if tc.registered { + cfg.BrokerMode = BrokerModeRegistered + } + err := validateDirectLeaseCreation(cfg) + if (err != nil) != tc.denied { + t.Fatalf("err=%v denied=%t", err, tc.denied) + } + if tc.denied && err.Error() != "direct-provider leases are disabled (directLeases: deny); configure a coordinator or add "+tc.provider+" to directLeaseAllow" { + t.Fatalf("error=%v", err) + } + }) + } + for _, name := range []string{"ssh", "local-container", "docker-sandbox", "multipass", "firecracker", "lume", "tart", "hyperv", "windows-sandbox", "apple-container", "apple-machine", "apple-vm", "anthropic-sandbox-runtime", "mxc"} { + if !directLeaseExempt(name) { + t.Fatalf("missing local exemption %s", name) + } + } +} + +func TestDirectLeaseConfigTrust(t *testing.T) { + clearConfigEnv(t) + t.Setenv("CRABBOX_DIRECT_LEASES", "") + t.Setenv("CRABBOX_DIRECT_LEASE_ALLOW", "") + for _, trusted := range []bool{false, true} { + cfg := baseConfig() + cfg.DirectLeases = "deny" + cfg.DirectLeaseAllow = []string{"aws"} + allow := []string{"gcp"} + if err := applyFileConfigWithTrust(&cfg, fileConfig{DirectLeases: "allow", DirectLeaseAllow: &allow}, trusted); err != nil { + t.Fatal(err) + } + wantPolicy, wantAllow := "deny", []string{"aws"} + if trusted { + wantPolicy, wantAllow = "allow", allow + } + if cfg.DirectLeases != wantPolicy || !reflect.DeepEqual(cfg.DirectLeaseAllow, wantAllow) { + t.Fatalf("trusted=%t config=%s %v", trusted, cfg.DirectLeases, cfg.DirectLeaseAllow) + } + } + cfg := baseConfig() + if err := applyFileConfigWithTrust(&cfg, fileConfig{DirectLeases: "deny"}, false); err != nil || cfg.DirectLeases != "deny" { + t.Fatal("repository cannot tighten") + } + t.Setenv("CRABBOX_DIRECT_LEASES", "allow") + t.Setenv("CRABBOX_DIRECT_LEASE_ALLOW", " aws, gcp, aws ") + if err := applyEnv(&cfg); err != nil || cfg.DirectLeases != "allow" || !reflect.DeepEqual(cfg.DirectLeaseAllow, []string{"aws", "gcp"}) { + t.Fatalf("env config: %v %s %v", err, cfg.DirectLeases, cfg.DirectLeaseAllow) + } + t.Setenv("CRABBOX_DIRECT_LEASES", "bogus") + if err := applyEnv(&cfg); err == nil { + t.Fatal("invalid policy accepted") + } +} + +func TestDirectDenyCommandsBeforeProviderConfigure(t *testing.T) { + for _, command := range []string{"warmup", "run", "prewarm"} { + t.Run(command, func(t *testing.T) { + clearConfigEnv(t) + t.Setenv("CRABBOX_DIRECT_LEASES", "deny") + dir := t.TempDir() + t.Chdir(dir) + isolateRunTestUserDirs(t, dir) + previous := runEnvProfileTestConfigureHook + defer func() { runEnvProfileTestConfigureHook = previous }() + called := false + runEnvProfileTestConfigureHook = func(Config) { called = true } + args := []string{"--provider", "run-env-profile-test", "--network", "public"} + var out, diagnostics bytes.Buffer + app := App{Stdout: &out, Stderr: &diagnostics} + var err error + switch command { + case "warmup": + err = app.warmup(context.Background(), args) + case "run": + err = app.runCommand(context.Background(), append(args, "--", "true")) + case "prewarm": + err = app.prewarm(context.Background(), args) + } + if err == nil || !strings.Contains(err.Error(), "direct-provider leases are disabled") || called { + t.Fatalf("err=%v configure=%t diagnostics=%s", err, called, diagnostics.String()) + } + }) + } +} + +func TestDirectLeaseConfigLayering(t *testing.T) { + clearConfigEnv(t) + if err := os.Unsetenv("CRABBOX_DIRECT_LEASE_ALLOW"); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + t.Chdir(dir) + isolateRunTestUserDirs(t, dir) + user := userConfigPath() + if err := os.MkdirAll(filepath.Dir(user), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(user, []byte("directLeases: deny\ndirectLeaseAllow: [aws]\n"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, ".crabbox.yaml"), []byte("directLeases: allow\ndirectLeaseAllow: [gcp]\n"), 0600); err != nil { + t.Fatal(err) + } + cfg, err := loadConfigInputs() + if err != nil { + t.Fatal(err) + } + if cfg.DirectLeases != "deny" || !reflect.DeepEqual(cfg.DirectLeaseAllow, []string{"aws"}) { + t.Fatalf("repository loosened policy: %s %v", cfg.DirectLeases, cfg.DirectLeaseAllow) + } +} diff --git a/internal/cli/guest_expiry.go b/internal/cli/guest_expiry.go new file mode 100644 index 0000000000..522cfc068f --- /dev/null +++ b/internal/cli/guest_expiry.go @@ -0,0 +1,261 @@ +package cli + +import ( + "context" + "crypto/sha256" + "fmt" + "strconv" + "strings" + "time" +) + +// GuestLeaseExpiryFields is the nonsecret bootstrap/refresh contract. Keep only +// controls automatic release after run; it never changes the expiry decision. +type GuestLeaseExpiryFields struct { + Crabbox string + State string + Keep string + ExpiresAt string +} + +func GuestLeaseExpiryFromLabels(labels map[string]string) GuestLeaseExpiryFields { + return GuestLeaseExpiryFields{labels["crabbox"], labels["state"], labels["keep"], labels["expires_at"]} +} + +// WithGuestLeaseExpiry binds file-backed bootstrap to the actual allocation. +func WithGuestLeaseExpiry(cfg Config, labels map[string]string) Config { + fields := GuestLeaseExpiryFromLabels(labels) + cfg.guestLeaseExpiry = &fields + return cfg +} + +// guestExpiryAdapter enumerates shared-cloud-init Linux providers. Metadata and +// reclaim differences stay here rather than leaking into the decision policy. +func guestExpiryAdapter(cfg Config) string { + if cfg.TargetOS != "" && cfg.TargetOS != targetLinux { + return "" + } + if p, err := ProviderFor(cfg.Provider); err == nil && ShouldUseCoordinator(cfg, p.Spec()) { + return "" + } + switch cfg.Provider { + case "gcp", "aws", "azure": + return cfg.Provider + case "hetzner", "digitalocean", "vultr", "linode", "ovh", "scaleway", "nebius", "lambda", "tencentcloud", "incus", "multipass", "firecracker": + return "file" + } + return "" +} + +// POSIX shell counterpart of shouldCleanupServer. Tests execute the same table +// through both implementations. Inputs are data, never evaluated as shell code. +const guestExpiryDecision = `expiry_due() { + [ "$crabbox" = true ] || return 1 + case "$state" in + failed|released|expired) return 0 ;; + esac + case "$expires_at" in + ''|*[!0-9]*) + case "$expires_at" in + ????-??-??T??:??:??*Z|????-??-??T??:??:??*+??:??|????-??-??T??:??:??*-??:??) + expires_at=$(date -u -d "$expires_at" +%s 2>/dev/null) || return 1 ;; + *) return 1 ;; + esac ;; + esac + expires_at=$(printf '%s' "$expires_at" | sed 's/^0*//') + # Bound arithmetic to a safe signed range, including the grace addition. + [ -n "$expires_at" ] && [ "${#expires_at}" -le 12 ] && [ "$expires_at" -gt 0 ] || return 1 + case "$state" in + running|provisioning) [ "$now" -gt "$((expires_at + 3600))" ] ;; + *) [ "$now" -gt "$expires_at" ] ;; + esac +} +` + +const guestExpiryGCPSource = `metadata() { + curl --connect-timeout 5 --max-time 15 -fsS -H 'Metadata-Flavor: Google' "http://metadata.google.internal/computeMetadata/v1/$1" +} +project="$(metadata project/project-id)" || exit 0 +name="$(metadata instance/name)" || exit 0 +zone_path="$(metadata instance/zone)" || exit 0 +zone="${zone_path##*/}" +token_json="$(metadata instance/service-accounts/default/token)" || exit 0 +token="$(printf '%s' "$token_json" | jq -er '.access_token | select(type == "string" and length > 0)')" || exit 0 +[ -n "$project" ] && [ -n "$name" ] && [ -n "$zone" ] || exit 0 +instance="$(curl --connect-timeout 5 --max-time 15 -fsS -H "Authorization: Bearer $token" "https://compute.googleapis.com/compute/v1/projects/$project/zones/$zone/instances/$name")" || exit 0 +label() { printf '%s' "$instance" | jq -er --arg key "$1" '.labels[$key] // "" | select(type == "string")'; } +provider="$(label provider)" || exit 0 +[ -z "$provider" ] || [ "$provider" = gcp ] || exit 0 +` +const guestExpiryAWSSource = `metadata_token="$(curl --connect-timeout 5 --max-time 15 -fsS -X PUT -H 'X-aws-ec2-metadata-token-ttl-seconds: 60' http://169.254.169.254/latest/api/token)" || exit 0 +[ -n "$metadata_token" ] || exit 0 +label() { + curl --connect-timeout 5 --max-time 15 -fsS -H "X-aws-ec2-metadata-token: $metadata_token" "http://169.254.169.254/latest/meta-data/tags/instance/$1" +} +` +const guestExpiryAzureSource = `instance="$(curl --connect-timeout 5 --max-time 15 -fsS -H 'Metadata: true' --noproxy '*' 'http://169.254.169.254/metadata/instance?api-version=2021-02-01')" || exit 0 +label() { + printf '%s' "$instance" | jq -er --arg key "$1" '[.compute.tagsList[] | select(.name == $key) | .value] | if length == 1 and (.[0] | type == "string") then .[0] else error("missing or ambiguous tag") end' +} +` +const guestExpiryFileSource = `file=/etc/crabbox/lease-expiry +[ -f "$file" ] && [ ! -L "$file" ] || exit 0 +[ "$(stat -c '%u:%a' "$file")" = '0:600' ] || exit 0 +file_fields="$(cat "$file")" || exit 0 +label() { + printf '%s\n' "$file_fields" | awk -F= -v key="$1" '$1 == key { count++; value=substr($0, length(key)+2) } END { if (count != 1) exit 1; print value }' +} +` + +func guestExpiryScript(adapter string) string { + source := guestExpiryFileSource + reclaim := "systemctl poweroff\n" + switch adapter { + case "gcp": + source = guestExpiryGCPSource + reclaim = `curl --connect-timeout 5 --max-time 15 -fsS -X DELETE -H "Authorization: Bearer $token" "https://compute.googleapis.com/compute/v1/projects/$project/zones/$zone/instances/$name" >/dev/null` + "\n" + case "aws": + source = guestExpiryAWSSource + case "azure": + source = guestExpiryAzureSource + } + return "#!/bin/sh\nset -eu\n" + guestExpiryDecision + source + `crabbox="$(label crabbox)" || exit 0 +state="$(label state)" || exit 0 +state="$(printf '%s' "$state" | tr '[:upper:]' '[:lower:]')" || exit 0 +keep="$(label keep)" || exit 0 +: "$keep" +expires_at="$(label expires_at)" || exit 0 +now="$(date -u +%s)" +expiry_due || exit 0 +logger -t crabbox-expiry-guard 'reclaiming expired direct lease' || true +` + reclaim +} + +func guestExpiryFile(fields GuestLeaseExpiryFields) (string, error) { + if fields.Crabbox != "true" { + return "", fmt.Errorf("guest lease expiry requires crabbox=true") + } + if fields.Keep != "true" && fields.Keep != "false" { + return "", fmt.Errorf("invalid guest lease keep value") + } + switch fields.State { + case "leased", "ready", "active", "running", "provisioning", "failed", "released", "expired": + default: + return "", fmt.Errorf("invalid guest lease state") + } + expires, ok := parseLeaseLabelTime(fields.ExpiresAt) + if !ok || expires.Unix() <= 0 || expires.Unix() > 999999999999 { + return "", fmt.Errorf("invalid guest lease expiry") + } + return fmt.Sprintf("crabbox=true\nstate=%s\nkeep=%s\nexpires_at=%d\n", fields.State, fields.Keep, expires.Unix()), nil +} + +func cloudInitGuestExpiryGuardFiles(cfg Config) string { + adapter := guestExpiryAdapter(cfg) + if adapter == "" { + return "" + } + files := guestExpiryCloudFile("/usr/local/sbin/crabbox-expiry-guard", "0755", guestExpiryScript(adapter)) + + guestExpiryCloudFile("/etc/systemd/system/crabbox-expiry-guard.service", "0644", `[Unit] +Description=Crabbox direct lease expiry guard +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=root +ExecStart=/usr/local/sbin/crabbox-expiry-guard +`) + guestExpiryCloudFile("/etc/systemd/system/crabbox-expiry-guard.timer", "0644", `[Unit] +Description=Run Crabbox direct lease expiry guard + +[Timer] +OnBootSec=2min +OnUnitActiveSec=2min +Persistent=true + +[Install] +WantedBy=timers.target +`) + if adapter == "file" && cfg.guestLeaseExpiry != nil { + if content, err := guestExpiryFile(*cfg.guestLeaseExpiry); err == nil { + files += guestExpiryCloudFile("/etc/crabbox/lease-expiry", "0600", content) + } + } + return strings.TrimRight(files, "\n") +} + +func guestExpiryCloudFile(path, mode, content string) string { + return fmt.Sprintf(" - path: %s\n owner: root:root\n permissions: '%s'\n content: |\n%s\n", path, mode, " "+strings.ReplaceAll(strings.TrimRight(content, "\n"), "\n", "\n ")) +} + +func cloudInitGuestExpiryGuardBootstrap(cfg Config) string { + if guestExpiryAdapter(cfg) == "" { + return "" + } + return " systemctl daemon-reload\n systemctl enable --now crabbox-expiry-guard.timer" +} + +// RefreshGuestLeaseExpiry is the extension hook for file-backed providers. It +// uses the existing authenticated SSH transport; the old deadline remains +// intact on failed writes, and readers see either complete version. +func RefreshGuestLeaseExpiry(ctx context.Context, target SSHTarget, fields GuestLeaseExpiryFields) error { + return refreshGuestLeaseExpiry(ctx, target, fields) +} +func refreshGuestLeaseExpiry(ctx context.Context, target SSHTarget, fields GuestLeaseExpiryFields) error { + if target.TargetOS != "" && target.TargetOS != targetLinux { + return fmt.Errorf("guest expiry refresh requires Linux") + } + content, err := guestExpiryFile(fields) + if err != nil { + return err + } + checksum := fmt.Sprintf("%x", sha256.Sum256([]byte(content))) + return refreshGuestLeaseExpirySSH(ctx, target, "sudo -n sh -c "+shellQuote(guestExpiryRefreshScript)+" crabbox-expiry "+shellQuote(checksum), content) +} + +var refreshGuestLeaseExpirySSH = runSSHInputQuiet + +const guestExpiryRefreshScript = `set -eu +[ ! -L /etc/crabbox ] +install -d -o root -g root -m 0755 /etc/crabbox +file=$(mktemp /etc/crabbox/.lease-expiry.XXXXXX) +trap 'rm -f -- "$file"' EXIT HUP INT TERM +cat >"$file" +# Never publish a truncated SSH input stream. +checksum=$(sha256sum "$file") +[ "${checksum%% *}" = "$1" ] +chown root:root "$file" +chmod 0600 "$file" +mv -fT -- "$file" /etc/crabbox/lease-expiry +trap - EXIT HUP INT TERM +` + +// Keep arithmetic in the same range as the shell guard. +func guestExpiryTimeValid(t time.Time) bool { + return t.Unix() > 0 && len(strconv.FormatInt(t.Unix(), 10)) <= 12 +} + +// Keep file-backed guards in sync with ordinary idle/state touches as well as +// explicit lifetime updates. Metadata-backed guards read the provider directly. +func touchLeaseWithGuestExpiry(ctx context.Context, backend LeaseTouchBackend, cfg Config, req TouchRequest) (Server, error) { + server, err := backend.Touch(ctx, req) + if err != nil || guestExpiryAdapter(cfg) != "file" { + return server, err + } + target := req.Lease.SSH + if target.Host == "" { + resolver, ok := backend.(SSHLoginBackend) + if !ok { + return server, fmt.Errorf("provider cannot resolve guest expiry SSH target") + } + lease, resolveErr := resolver.Resolve(ctx, ResolveRequest{ID: req.Lease.LeaseID, Options: leaseOptionsFromConfig(cfg)}) + if resolveErr != nil { + return server, resolveErr + } + target = lease.SSH + } + if err := refreshGuestLeaseExpiry(ctx, target, GuestLeaseExpiryFromLabels(server.Labels)); err != nil { + return server, fmt.Errorf("refresh guest lease expiry: %w", err) + } + return server, nil +} diff --git a/internal/cli/guest_expiry_test.go b/internal/cli/guest_expiry_test.go new file mode 100644 index 0000000000..fe4f73160b --- /dev/null +++ b/internal/cli/guest_expiry_test.go @@ -0,0 +1,377 @@ +package cli + +import ( + "context" + "crypto/sha256" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "gopkg.in/yaml.v3" +) + +func requireGuestExpiryShell(t *testing.T) { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("Linux guest fixtures require POSIX tools") + } + if _, err := exec.LookPath("sh"); err != nil { + t.Skip("POSIX shell unavailable") + } +} + +func TestGuestExpiryDecisionParity(t *testing.T) { + requireGuestExpiryShell(t) + now := time.Unix(1800000000, 0).UTC() + for _, tc := range []struct { + name, owner, state, keep, expiry string + want bool + }{ + {"unowned terminal", "", "failed", "false", "", false}, + {"foreign", "false", "ready", "false", "1", false}, + {"unlabeled", "", "", "", "", false}, + {"kept expired", "true", "ready", "true", "1799999999", true}, + {"kept future", "true", "ready", "true", "1800000001", false}, + {"at expiry", "true", "ready", "false", "1800000000", false}, + {"running grace", "true", "running", "false", "1799996400", false}, + {"running expired", "true", "running", "true", "1799996399", true}, + {"provisioning expired", "true", "provisioning", "true", "1799996399", true}, + {"failed", "true", "failed", "true", "", true}, + {"released", "true", "released", "true", "1800000001", true}, + {"expired", "true", "expired", "false", "", true}, + {"active", "true", "active", "false", "1", true}, + {"leased", "true", "leased", "false", "1", true}, + {"missing deadline", "true", "ready", "false", "", false}, + {"invalid deadline", "true", "ready", "false", "$(touch should-not-exist)", false}, + {"overflow", "true", "ready", "false", "999999999999999999999", false}, + {"negative", "true", "ready", "false", "-1", false}, + {"zero", "true", "ready", "false", "0", false}, + } { + t.Run(tc.name, func(t *testing.T) { + labels := map[string]string{"crabbox": tc.owner, "state": tc.state, "keep": tc.keep, "expires_at": tc.expiry} + got, reason := shouldCleanupServer(Server{Labels: labels}, now) + if got != tc.want { + t.Fatalf("Go=%t (%s) want=%t", got, reason, tc.want) + } + script := guestExpiryDecision + "\ncrabbox=" + shellQuote(tc.owner) + "\nstate=" + shellQuote(tc.state) + "\nkeep=" + shellQuote(tc.keep) + "\nexpires_at=" + shellQuote(tc.expiry) + "\nnow=1800000000\nif expiry_due; then echo due; else echo skip; fi\n" + output, err := exec.Command("sh", "-c", script).CombinedOutput() + if err != nil { + t.Fatalf("shell: %v: %s", err, output) + } + if (strings.TrimSpace(string(output)) == "due") != tc.want { + t.Fatalf("shell=%q want=%t", output, tc.want) + } + }) + } +} + +func TestGuestExpiryGuardProviders(t *testing.T) { + for _, name := range []string{"aws", "gcp", "azure", "hetzner", "digitalocean", "vultr", "linode", "ovh", "scaleway", "nebius", "lambda", "tencentcloud", "incus", "multipass", "firecracker"} { + t.Run(name, func(t *testing.T) { + cfg := WithGuestLeaseExpiry(Config{Provider: name, TargetOS: targetLinux}, map[string]string{"crabbox": "true", "state": "leased", "keep": "true", "expires_at": "1800000000"}) + data := cloudInit(cfg, "ssh-ed25519 fixture") + var document struct { + Files []struct{ Path, Owner, Permissions, Content string } `yaml:"write_files"` + } + if err := yaml.Unmarshal([]byte(data), &document); err != nil { + t.Fatal(err) + } + files := map[string]string{} + for _, file := range document.Files { + files[file.Path] = file.Content + if strings.Contains(file.Path, "expiry") && file.Owner != "root:root" { + t.Fatalf("owner=%q", file.Owner) + } + } + script := files["/usr/local/sbin/crabbox-expiry-guard"] + if script == "" || !strings.Contains(data, "OnUnitActiveSec=2min") || !strings.Contains(data, "systemctl enable --now crabbox-expiry-guard.timer") { + t.Fatal("missing guard") + } + switch name { + case "gcp": + if !strings.Contains(script, "-X DELETE") || strings.Contains(script, "systemctl poweroff") { + t.Fatal("GCP reclaim") + } + case "aws": + if !strings.Contains(script, "tags/instance/$1") || !strings.Contains(script, "X-aws-ec2-metadata-token") { + t.Fatal("AWS source") + } + case "azure": + if !strings.Contains(script, ".compute.tagsList[]") { + t.Fatal("Azure source") + } + default: + if !strings.Contains(files["/etc/crabbox/lease-expiry"], "expires_at=1800000000") { + t.Fatal("missing file deadline") + } + } + if name != "gcp" && !strings.Contains(script, "systemctl poweroff") { + t.Fatal("missing poweroff") + } + if shellcheck, err := exec.LookPath("shellcheck"); err == nil { + cmd := exec.Command(shellcheck, "-s", "sh", "-") + cmd.Stdin = strings.NewReader(script) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("shellcheck: %v\n%s", err, output) + } + } + for _, target := range []string{targetWindows, targetMacOS} { + cfg.TargetOS = target + if cloudInitGuestExpiryGuardFiles(cfg) != "" { + t.Fatal("non-Linux guard") + } + } + }) + } + for _, name := range []string{"aws", "azure", "gcp", "hetzner"} { + cfg := Config{Provider: name, TargetOS: targetLinux, Coordinator: "https://coordinator.example"} + if cloudInitGuestExpiryGuardFiles(cfg) != "" { + t.Fatalf("brokered %s got guard", name) + } + } +} + +func TestAWSDirectGuestExpiryLaunchOptions(t *testing.T) { + for _, tc := range []struct { + target, coordinator string + want bool + }{{targetLinux, "", true}, {targetLinux, "https://coordinator.example", false}, {targetWindows, "", false}, {targetMacOS, "", false}} { + input := &ec2.RunInstancesInput{} + applyAWSGuestExpiryOptions(input, Config{Provider: "aws", TargetOS: tc.target, Coordinator: tc.coordinator}) + if !tc.want { + if input.MetadataOptions != nil || input.InstanceInitiatedShutdownBehavior != "" { + t.Fatal("unexpected launch options") + } + continue + } + if input.InstanceInitiatedShutdownBehavior != types.ShutdownBehaviorTerminate || input.MetadataOptions.InstanceMetadataTags != types.InstanceMetadataTagsStateEnabled || input.MetadataOptions.HttpTokens != types.HttpTokensStateRequired { + t.Fatalf("options=%+v", input) + } + for _, tag := range awsTagsWithName(DirectLeaseLabels(baseConfig(), "cbx_123456abcdef", "fixture", "aws", "", false, time.Now()), "fixture") { + if strings.ContainsAny(*tag.Key, " /\t\n") || *tag.Key == "." || *tag.Key == ".." || *tag.Key == "_index" { + t.Fatalf("IMDS-illegal tag key %q", *tag.Key) + } + } + } +} + +func TestRefreshGuestLeaseExpiryAtomic(t *testing.T) { + requireGuestExpiryShell(t) + old := refreshGuestLeaseExpirySSH + defer func() { refreshGuestLeaseExpirySSH = old }() + fields := GuestLeaseExpiryFields{"true", "ready", "true", "1800000000"} + called := false + refreshGuestLeaseExpirySSH = func(ctx context.Context, target SSHTarget, remote, input string) error { + called = true + if target.Host != "fixture" || remote != "sudo -n sh -c "+shellQuote(guestExpiryRefreshScript)+" crabbox-expiry "+shellQuote(fmt.Sprintf("%x", sha256.Sum256([]byte(input)))) { + t.Fatal("incorrect SSH transport") + } + if input != "crabbox=true\nstate=ready\nkeep=true\nexpires_at=1800000000\n" { + t.Fatalf("input=%q", input) + } + return nil + } + if err := refreshGuestLeaseExpiry(context.Background(), SSHTarget{Host: "fixture"}, fields); err != nil || !called { + t.Fatalf("refresh: %v", err) + } + fields.State = "ready\n$(touch injected)" + called = false + if err := refreshGuestLeaseExpiry(context.Background(), SSHTarget{}, fields); err == nil || called { + t.Fatal("unsafe fields reached SSH") + } + // Execute the actual transaction locally, replacing only ownership operations + // unavailable to non-root tests. Observe atomic rename and retained old bytes + // when the input producer fails before publication. + dir := t.TempDir() + file := filepath.Join(dir, "lease-expiry") + if err := os.WriteFile(file, []byte("old"), 0600); err != nil { + t.Fatal(err) + } + script := strings.ReplaceAll(guestExpiryRefreshScript, "/etc/crabbox", shellQuote(dir)) + script = strings.ReplaceAll(script, " -o root -g root", "") + script = strings.ReplaceAll(script, "chown root:root \"$file\"", ":") + // macOS mv has no -T; both forms perform the same atomic file rename here. + script = strings.ReplaceAll(script, "mv -fT --", "mv -f --") + if _, err := exec.LookPath("sha256sum"); err != nil { + if _, err := exec.LookPath("shasum"); err != nil { + t.Skip("SHA-256 CLI unavailable") + } + script = strings.ReplaceAll(script, "sha256sum", "shasum -a 256") + } + checksum := fmt.Sprintf("%x", sha256.Sum256([]byte("new"))) + cmd := exec.Command("sh", "-c", script, "crabbox-expiry", checksum) + cmd.Stdin = strings.NewReader("new") + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("transaction: %v %s", err, output) + } + data, _ := os.ReadFile(file) + if string(data) != "new" { + t.Fatalf("data=%q", data) + } + info, _ := os.Stat(file) + if info.Mode().Perm() != 0600 { + t.Fatal("permissions") + } + failed := strings.Replace(script, "cat >\"$file\"", "false >\"$file\"", 1) + if exec.Command("sh", "-c", failed, "crabbox-expiry", checksum).Run() == nil { + t.Fatal("expected failed write") + } + data, _ = os.ReadFile(file) + if string(data) != "new" { + t.Fatal("failed write replaced deadline") + } + truncated := exec.Command("sh", "-c", script, "crabbox-expiry", checksum) + truncated.Stdin = strings.NewReader("ne") + if truncated.Run() == nil { + t.Fatal("truncated transfer was published") + } + data, _ = os.ReadFile(file) + if string(data) != "new" { + t.Fatal("truncated transfer replaced deadline") + } + entries, _ := os.ReadDir(dir) + if len(entries) != 1 { + t.Fatal("temporary file leaked") + } +} + +func TestGuestExpiryMetadataFailsClosed(t *testing.T) { + requireGuestExpiryShell(t) + if _, err := exec.LookPath("jq"); err != nil { + t.Skip("jq unavailable") + } + for _, adapter := range []string{"aws", "azure"} { + for _, mode := range []string{"expired", "future", "foreign", "missing", "injected", "duplicate"} { + t.Run(adapter+"/"+mode, func(t *testing.T) { + dir := t.TempDir() + mock := `#!/bin/sh +case "$*" in + *latest/api/token*) echo synthetic-token ;; + *latest/meta-data/tags/instance/*) + arguments="$*" + case "${arguments##*/}" in + crabbox) if [ "$MODE" = foreign ]; then echo false; else echo true; fi ;; + state) if [ "$MODE" = missing ]; then exit 22; else echo ready; fi ;; + keep) echo true ;; + expires_at) case "$MODE" in future) echo 999999999999 ;; injected) echo '$(touch injected)' ;; *) echo 1 ;; esac ;; + esac ;; + *metadata/instance*) + owner=true; expires=1 + [ "$MODE" != foreign ] || owner=false + [ "$MODE" != future ] || expires=999999999999 + if [ "$MODE" = injected ]; then expires='$(touch injected)'; fi + extra=''; [ "$MODE" != duplicate ] || extra=',{"name":"crabbox","value":"false"}' + state=',{"name":"state","value":"ready"}'; [ "$MODE" != missing ] || state='' + printf '{"compute":{"tagsList":[{"name":"crabbox","value":"%s"},{"name":"keep","value":"true"},{"name":"expires_at","value":"%s"}%s%s]}}\n' "$owner" "$expires" "$state" "$extra" ;; + *) exit 22 ;; +esac +` + for name, content := range map[string]string{"curl": mock, "systemctl": "#!/bin/sh\ntouch \"$RECLAIMED\"\n", "logger": "#!/bin/sh\nexit 0\n"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0700); err != nil { + t.Fatal(err) + } + } + cmd := exec.Command("sh", "-c", guestExpiryScript(adapter)) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "PATH="+dir+string(os.PathListSeparator)+os.Getenv("PATH"), "MODE="+mode, "RECLAIMED="+filepath.Join(dir, "reclaimed"), "INJECTED="+filepath.Join(dir, "injected")) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("guard: %v %s", err, output) + } + _, err := os.Stat(filepath.Join(dir, "reclaimed")) + want := mode == "expired" || adapter == "aws" && mode == "duplicate" + if (err == nil) != want { + t.Fatalf("reclaim=%t want=%t", err == nil, want) + } + if _, err := os.Stat(filepath.Join(dir, "injected")); err == nil { + t.Fatal("metadata executed") + } + }) + } + } +} + +func TestAWSDirectGuestExpiryLaunchRequest(t *testing.T) { + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + t.Error(err) + return + } + calls++ + for key, want := range map[string]string{"Action": "RunInstances", "InstanceInitiatedShutdownBehavior": "terminate", "MetadataOptions.InstanceMetadataTags": "enabled", "MetadataOptions.HttpTokens": "required"} { + if got := r.Form.Get(key); got != want { + t.Errorf("%s=%q want=%q", key, got, want) + } + } + writeEC2Error(w, "InsufficientInstanceCapacity", "synthetic rejection", http.StatusBadRequest) + })) + defer server.Close() + cfg := baseConfig() + cfg.Provider = "aws" + cfg.AWSRootGB = 40 + _, err := testAWSClient(server.URL).createServer(context.Background(), cfg, "ssh-ed25519 fixture", "cbx_123456abcdef", "fixture", true, "ami-fixture", "sg-fixture", false, nil) + if err == nil || calls != 1 { + t.Fatalf("calls=%d err=%v", calls, err) + } +} + +type guestExpiryTouchBackend struct{ fields map[string]string } + +func (b guestExpiryTouchBackend) Spec() ProviderSpec { return ProviderSpec{Name: "hetzner"} } +func (b guestExpiryTouchBackend) Touch(context.Context, TouchRequest) (Server, error) { + return Server{Labels: b.fields}, nil +} +func TestGuestExpiryTouchRefreshesFile(t *testing.T) { + old := refreshGuestLeaseExpirySSH + defer func() { refreshGuestLeaseExpirySSH = old }() + calls := 0 + refreshGuestLeaseExpirySSH = func(_ context.Context, target SSHTarget, _, content string) error { + calls++ + if target.Host != "fixture" || !strings.Contains(content, "state=running\n") || !strings.Contains(content, "expires_at=1800000000\n") { + t.Fatalf("target=%v content=%q", target.Host, content) + } + return nil + } + backend := guestExpiryTouchBackend{map[string]string{"crabbox": "true", "state": "running", "keep": "true", "expires_at": "1800000000"}} + req := TouchRequest{Lease: LeaseTarget{SSH: SSHTarget{Host: "fixture"}}} + if _, err := touchLeaseWithGuestExpiry(context.Background(), backend, Config{Provider: "hetzner", TargetOS: targetLinux}, req); err != nil { + t.Fatal(err) + } + if calls != 1 { + t.Fatal("file was not refreshed") + } + if _, err := touchLeaseWithGuestExpiry(context.Background(), backend, Config{Provider: "aws", TargetOS: targetLinux}, req); err != nil { + t.Fatal(err) + } + if calls != 1 { + t.Fatal("metadata source refreshed a file") + } +} + +func TestGuestExpiryFileSnapshot(t *testing.T) { + requireGuestExpiryShell(t) + dir := t.TempDir() + file := filepath.Join(dir, "lease-expiry") + if err := os.WriteFile(file, []byte("crabbox=true\nstate=running\nkeep=true\nexpires_at=1\n"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "stat"), []byte("#!/bin/sh\necho 0:600\n"), 0700); err != nil { + t.Fatal(err) + } + script := strings.ReplaceAll(guestExpiryFileSource, "/etc/crabbox/lease-expiry", file) + + "test \"$(label crabbox)\" = true\nprintf 'crabbox=true\\nstate=ready\\nkeep=true\\nexpires_at=999999999999\\n' >\"$file\"\ntest \"$(label expires_at)\" = 1\n" + cmd := exec.Command("sh", "-c", script) + cmd.Env = append(os.Environ(), "PATH="+dir+string(os.PathListSeparator)+os.Getenv("PATH")) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("snapshot changed during read: %v %s", err, output) + } +} diff --git a/internal/cli/hcloud.go b/internal/cli/hcloud.go index 9a3f1167f9..49ba0263cf 100644 --- a/internal/cli/hcloud.go +++ b/internal/cli/hcloud.go @@ -292,12 +292,13 @@ func (c *HetznerClient) CreateServer(ctx context.Context, cfg Config, publicKey, if cfg.Tailscale.Enabled && cfg.Tailscale.Hostname == "" { cfg.Tailscale.Hostname = RenderTailscaleHostname(cfg.Tailscale.HostnameTemplate, leaseID, slug, cfg.Provider) } + now := time.Now().UTC() + labels := DirectLeaseLabels(cfg, leaseID, slug, "hetzner", "", keep, now) + cfg = WithGuestLeaseExpiry(cfg, labels) userData, err := hetznerUserData(cloudInit(cfg, publicKey)) if err != nil { return Server{}, err } - now := time.Now().UTC() - labels := DirectLeaseLabels(cfg, leaseID, slug, "hetzner", "", keep, now) body := map[string]any{ "name": name, "server_type": cfg.ServerType, diff --git a/internal/cli/heartbeat.go b/internal/cli/heartbeat.go index 6a8b585f6b..fac6492829 100644 --- a/internal/cli/heartbeat.go +++ b/internal/cli/heartbeat.go @@ -164,7 +164,7 @@ func (a App) heartbeat(ctx context.Context, args []string) error { } registeredLease = &coordinatorLease } - touched, err := sshBackend.Touch(ctx, TouchRequest{ + touched, err := touchLeaseWithGuestExpiry(ctx, sshBackend, cfg, TouchRequest{ Lease: lease, State: state, IdleTimeout: cfg.IdleTimeout, diff --git a/internal/cli/pool.go b/internal/cli/pool.go index a04a640522..7260d64e28 100644 --- a/internal/cli/pool.go +++ b/internal/cli/pool.go @@ -580,17 +580,14 @@ func (a App) cleanup(ctx context.Context, args []string) error { func shouldCleanupServer(server Server, now time.Time) (bool, string) { labels := server.Labels - if labels == nil { + if labels == nil || labels["crabbox"] != "true" { return false, "missing labels" } - if strings.EqualFold(labels["keep"], "true") { - return false, "keep=true" - } state := strings.ToLower(labels["state"]) switch state { case "running", "provisioning": expiresAt, ok := cleanupExpiry(labels) - if ok && now.After(expiresAt.Add(12*time.Hour)) { + if ok && now.After(expiresAt.Add(time.Hour)) { return true, "stale state=" + state } return false, "state=" + state @@ -608,7 +605,7 @@ func shouldCleanupServer(server Server, now time.Time) (bool, string) { if !ok { return false, "missing expires_at" } - if now.Before(expiresAt) { + if !now.After(expiresAt) { return false, "not expired" } return true, "expired" @@ -624,7 +621,7 @@ func cleanupExpiry(labels map[string]string) (time.Time, bool) { if value == "" { continue } - if parsed, ok := parseLeaseLabelTime(value); ok { + if parsed, ok := parseLeaseLabelTime(value); ok && guestExpiryTimeValid(parsed) { return parsed, true } } diff --git a/internal/cli/pool_test.go b/internal/cli/pool_test.go index d501834d9b..b32127bf90 100644 --- a/internal/cli/pool_test.go +++ b/internal/cli/pool_test.go @@ -19,7 +19,7 @@ func TestShouldCleanupServerSkipsRunningAndProvisioningStates(t *testing.T) { now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) for _, state := range []string{"running", "provisioning"} { server := Server{Labels: map[string]string{ - "keep": "false", + "crabbox": "true", "keep": "false", "state": state, "expires_at": now.Add(-time.Hour).Format(time.RFC3339), }} @@ -33,7 +33,7 @@ func TestShouldCleanupServerDeletesExpiredIdleStates(t *testing.T) { now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) for _, state := range []string{"leased", "ready", "active"} { server := Server{Labels: map[string]string{ - "keep": "false", + "crabbox": "true", "keep": "false", "state": state, "expires_at": now.Add(-time.Minute).Format(time.RFC3339), }} @@ -46,7 +46,7 @@ func TestShouldCleanupServerDeletesExpiredIdleStates(t *testing.T) { func TestShouldCleanupServerDeletesStaleRunningStates(t *testing.T) { now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) server := Server{Labels: map[string]string{ - "keep": "false", + "crabbox": "true", "keep": "false", "state": "running", "expires_at": now.Add(-13 * time.Hour).Format(time.RFC3339), }} @@ -58,7 +58,7 @@ func TestShouldCleanupServerDeletesStaleRunningStates(t *testing.T) { func TestShouldCleanupServerDeletesExpiredInactive(t *testing.T) { now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) server := Server{Labels: map[string]string{ - "keep": "false", + "crabbox": "true", "keep": "false", "expires_at": now.Add(-time.Minute).Format(time.RFC3339), }} if ok, reason := shouldCleanupServer(server, now); !ok { @@ -69,10 +69,10 @@ func TestShouldCleanupServerDeletesExpiredInactive(t *testing.T) { func TestShouldCleanupServerKeepsUnexpiredAndMissingExpiry(t *testing.T) { now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) tests := []Server{ - {Labels: map[string]string{"keep": "true", "expires_at": now.Add(time.Hour).Format(time.RFC3339)}}, - {Labels: map[string]string{"keep": "true", "state": "released"}}, - {Labels: map[string]string{"keep": "false", "expires_at": now.Add(time.Hour).Format(time.RFC3339)}}, - {Labels: map[string]string{"keep": "false"}}, + {Labels: map[string]string{"crabbox": "true", "keep": "true", "expires_at": now.Add(time.Hour).Format(time.RFC3339)}}, + + {Labels: map[string]string{"crabbox": "true", "keep": "false", "expires_at": now.Add(time.Hour).Format(time.RFC3339)}}, + {Labels: map[string]string{"crabbox": "true", "keep": "false"}}, } for _, server := range tests { if ok, reason := shouldCleanupServer(server, now); ok { @@ -81,19 +81,19 @@ func TestShouldCleanupServerKeepsUnexpiredAndMissingExpiry(t *testing.T) { } } -func TestShouldCleanupServerPreservesKeptExpiredLeases(t *testing.T) { +func TestShouldCleanupServerReapsKeptExpiredLeases(t *testing.T) { now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) for _, state := range []string{"ready", "leased", "active", "failed", "released", "expired", "running", "provisioning"} { t.Run(state, func(t *testing.T) { server := Server{Labels: map[string]string{ - "keep": "true", "state": state, + "crabbox": "true", "keep": "true", "state": state, "expires_at": now.Add(-24 * time.Hour).Format(time.RFC3339), }} - if got, reason := shouldCleanupServer(server, now); got || reason != "keep=true" { - t.Fatalf("cleanup=%t (%s), want keep protection", got, reason) + if got, reason := shouldCleanupServer(server, now); !got { + t.Fatalf("cleanup=%t (%s), want expired lease cleanup", got, reason) } server.Labels["expires_at"] = now.Add(time.Minute).Format(time.RFC3339) - if got, reason := shouldCleanupServer(server, now); got { + if got, reason := shouldCleanupServer(server, now); got && state != "failed" && state != "released" && state != "expired" { t.Fatalf("cleanup before expiry: %s", reason) } }) diff --git a/internal/cli/prewarm.go b/internal/cli/prewarm.go index 1ac7542848..c84a354a37 100644 --- a/internal/cli/prewarm.go +++ b/internal/cli/prewarm.go @@ -126,6 +126,9 @@ func (a App) prewarmWithPoolFillClaim(ctx context.Context, args []string, poolFi if poolIdentityErr != nil { return poolIdentityErr } + if err := validateDirectLeaseCreation(cfg); err != nil { + return err + } backend, err := loadBackend(cfg, runtimeForApp(a)) if err != nil { return err diff --git a/internal/cli/provider_backend.go b/internal/cli/provider_backend.go index b220527ed2..726c2f4902 100644 --- a/internal/cli/provider_backend.go +++ b/internal/cli/provider_backend.go @@ -2267,7 +2267,7 @@ func (a App) touchLeaseTargetBestEffort(ctx context.Context, cfg Config, lease L } touchCtx, cancel := context.WithTimeout(ctx, bestEffortLeaseTouchTimeout) defer cancel() - server, err := sshBackend.Touch(touchCtx, TouchRequest{Lease: lease, State: state, IdleTimeout: cfg.IdleTimeout}) + server, err := touchLeaseWithGuestExpiry(touchCtx, sshBackend, cfg, TouchRequest{Lease: lease, State: state, IdleTimeout: cfg.IdleTimeout}) if err != nil { fmt.Fprintf(a.Stderr, "warning: touch failed for %s: %v\n", lease.LeaseID, err) return lease.Server diff --git a/internal/cli/run.go b/internal/cli/run.go index d20ce16f38..34ec5ab4cf 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -83,6 +83,9 @@ func (a App) warmupWithLeaseObserver(ctx context.Context, args []string, observe if err != nil { return err } + if err := validateDirectLeaseCreation(cfg); err != nil { + return err + } backend, err := loadBackend(cfg, runtimeForApp(a)) if err != nil { return err @@ -1008,6 +1011,11 @@ func (a App) runCommandWithBenchmarkRecord(ctx context.Context, args []string, b delegatedTimingCapture = &capturedTimingReportWriter{writer: a.Stderr} backendRuntime.Stderr = delegatedTimingCapture } + if strings.TrimSpace(*leaseIDFlag) == "" { + if err := validateDirectLeaseCreation(cfg); err != nil { + return err + } + } backend, err := loadBackend(cfg, backendRuntime) if err != nil { return err @@ -1791,7 +1799,7 @@ func (a App) runCommandWithBenchmarkRecord(ctx context.Context, args []string, b } a.startRegisteredWebVNCDaemonBestEffort(ctx, cfg, target, leaseID, acquired && *keep) if !useCoordinator && leaseID != "" { - if touched, touchErr := sshBackend.Touch(ctx, TouchRequest{Lease: LeaseTarget{Server: server, SSH: target, LeaseID: leaseID}, State: blank(server.Labels["state"], "ready"), IdleTimeout: cfg.IdleTimeout}); touchErr == nil { + if touched, touchErr := touchLeaseWithGuestExpiry(ctx, sshBackend, cfg, TouchRequest{Lease: LeaseTarget{Server: server, SSH: target, LeaseID: leaseID}, State: blank(server.Labels["state"], "ready"), IdleTimeout: cfg.IdleTimeout}); touchErr == nil { server = touched } else { fmt.Fprintf(a.Stderr, "warning: direct touch failed for %s: %v\n", leaseID, touchErr) @@ -2127,6 +2135,9 @@ func (a App) runCommandWithBenchmarkRecord(ctx context.Context, args []string, b !shouldReplaceLeaseAfterBeforeCommandSSHFailure(waitErr, acquired, useCoordinator, *leaseIDFlag != "", *keep, *keepOnFailure, *noSync, *syncOnly, *stopAfter, requestedSlug) { return false, nil } + if err := validateDirectLeaseCreation(cfg); err != nil { + return true, err + } beforeCommandLeaseReplacementAttempted = true oldLease := LeaseTarget{Server: server, SSH: target, LeaseID: leaseID, Coordinator: coord} oldLeaseID := leaseID @@ -2986,13 +2997,13 @@ afterSync: } } if !useCoordinator { - if touched, touchErr := sshBackend.Touch(context.Background(), TouchRequest{Lease: LeaseTarget{Server: server, SSH: target, LeaseID: leaseID}, State: "running", IdleTimeout: cfg.IdleTimeout}); touchErr == nil { + if touched, touchErr := touchLeaseWithGuestExpiry(context.Background(), sshBackend, cfg, TouchRequest{Lease: LeaseTarget{Server: server, SSH: target, LeaseID: leaseID}, State: "running", IdleTimeout: cfg.IdleTimeout}); touchErr == nil { server = touched } else { fmt.Fprintf(a.Stderr, "warning: direct touch state=running: %v\n", touchErr) } defer func() { - if touched, touchErr := sshBackend.Touch(context.Background(), TouchRequest{Lease: LeaseTarget{Server: server, SSH: target, LeaseID: leaseID}, State: "ready", IdleTimeout: cfg.IdleTimeout}); touchErr == nil { + if touched, touchErr := touchLeaseWithGuestExpiry(context.Background(), sshBackend, cfg, TouchRequest{Lease: LeaseTarget{Server: server, SSH: target, LeaseID: leaseID}, State: "ready", IdleTimeout: cfg.IdleTimeout}); touchErr == nil { server = touched } else { fmt.Fprintf(a.Stderr, "warning: direct touch state=ready: %v\n", touchErr) diff --git a/internal/cli/shard.go b/internal/cli/shard.go index 13403d6400..deae89824f 100644 --- a/internal/cli/shard.go +++ b/internal/cli/shard.go @@ -226,6 +226,9 @@ func (a App) shard(ctx context.Context, args []string) error { if err != nil { return err } + if err := validateDirectLeaseCreation(cfg); err != nil { + return err + } backend, err := loadBackend(cfg, runtimeForApp(a)) if err != nil { return err diff --git a/internal/cli/status.go b/internal/cli/status.go index 5fd6f8e202..f8a377e953 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -73,7 +73,7 @@ func (a App) status(ctx context.Context, args []string) error { fmt.Fprintf(a.Stderr, "warning: touch skipped for %s: %v\n", lease.LeaseID, claimErr) } else if claimed { SetServerLeaseClaimSnapshot(&lease.Server, claim, true) - _, touchErr := sshBackend.Touch(statusCtx, TouchRequest{Lease: lease, State: state.State, IdleTimeout: cfg.IdleTimeout}) + _, touchErr := touchLeaseWithGuestExpiry(statusCtx, sshBackend, cfg, TouchRequest{Lease: lease, State: state.State, IdleTimeout: cfg.IdleTimeout}) if touchErr != nil { fmt.Fprintf(a.Stderr, "warning: touch failed for %s: %v\n", lease.LeaseID, touchErr) } diff --git a/internal/cli/warmup_expiry_guard_test.go b/internal/cli/warmup_expiry_guard_test.go index 53366dd4f4..e66b184892 100644 --- a/internal/cli/warmup_expiry_guard_test.go +++ b/internal/cli/warmup_expiry_guard_test.go @@ -24,7 +24,7 @@ func TestWarmupGCPKeptIdleExpiryGuard(t *testing.T) { var document struct { Files []struct{ Path, Content string } `yaml:"write_files"` } - if err := yaml.Unmarshal([]byte("write_files:\n"+cloudInitGCPExpiryGuardFiles()), &document); err != nil { + if err := yaml.Unmarshal([]byte("write_files:\n"+cloudInitGuestExpiryGuardFiles(Config{Provider: "gcp", TargetOS: targetLinux})), &document); err != nil { t.Fatal(err) } now := time.Now() @@ -34,12 +34,12 @@ func TestWarmupGCPKeptIdleExpiryGuard(t *testing.T) { }{ {"kept idle", "ready", LeaseLabelTime(now.Add(-time.Minute)), "true", true}, {"kept future", "ready", LeaseLabelTime(now.Add(time.Hour)), "true", false}, - {"released future", "released", LeaseLabelTime(now.Add(time.Hour)), "true", false}, - {"missing expiry", "released", "", "true", false}, + {"released future", "released", LeaseLabelTime(now.Add(time.Hour)), "true", true}, + {"missing expiry", "released", "", "true", true}, {"invalid expiry", "ready", "invalid", "true", false}, {"overflow expiry", "ready", "999999999999999999999", "true", false}, - {"running grace", "running", LeaseLabelTime(now.Add(-time.Hour)), "true", false}, - {"stale running", "running", LeaseLabelTime(now.Add(-13 * time.Hour)), "true", true}, + {"running grace", "running", LeaseLabelTime(now.Add(-30 * time.Minute)), "true", false}, + {"stale running", "running", LeaseLabelTime(now.Add(-2 * time.Hour)), "true", true}, {"foreign instance", "ready", LeaseLabelTime(now.Add(-time.Hour)), "false", false}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/cli/warmup_policy_test.go b/internal/cli/warmup_policy_test.go index cbbc5c1513..b4b2d843e5 100644 --- a/internal/cli/warmup_policy_test.go +++ b/internal/cli/warmup_policy_test.go @@ -78,8 +78,8 @@ func TestWarmupKeepIdleConfigPrecedence(t *testing.T) { if due, reason := shouldCleanupServer(server, created.Add(tc.idle-time.Second)); due { t.Fatalf("cleanup before idle deadline: %s", reason) } - if due, reason := shouldCleanupServer(server, created.Add(tc.idle+time.Second)); due == tc.keep { - t.Fatalf("manual cleanup=%t (%s), want keep=%t respected", due, reason, tc.keep) + if due, reason := shouldCleanupServer(server, created.Add(tc.idle+time.Second)); !due { + t.Fatalf("manual cleanup=%t (%s), want expired lease cleanup", due, reason) } }) } diff --git a/internal/providers/digitalocean/client.go b/internal/providers/digitalocean/client.go index 7dfd5004f0..5c259ee17d 100644 --- a/internal/providers/digitalocean/client.go +++ b/internal/providers/digitalocean/client.go @@ -366,7 +366,7 @@ func (c *digitalOceanClient) createDroplet(ctx context.Context, cfg core.Config, "image": digitalOceanImage(cfg), "ssh_keys": []any{key.ID}, "tags": tags, - "user_data": core.CloudInitUserData(cfg, publicKey), + "user_data": core.CloudInitUserData(core.WithGuestLeaseExpiry(cfg, labelsFromTags(tags)), publicKey), "ipv6": false, } if cfg.DigitalOcean.VPCUUID != "" { diff --git a/internal/providers/digitalocean/heartbeat_test.go b/internal/providers/digitalocean/heartbeat_test.go index ade7a35166..41cfa50ef0 100644 --- a/internal/providers/digitalocean/heartbeat_test.go +++ b/internal/providers/digitalocean/heartbeat_test.go @@ -3,10 +3,13 @@ package digitalocean import ( "bytes" "context" + "crypto/ed25519" + "crypto/rand" "encoding/json" "errors" "fmt" "io" + "net" "net/http" "os" "path/filepath" @@ -16,6 +19,7 @@ import ( core "github.com/openclaw/crabbox/internal/cli" "github.com/openclaw/crabbox/internal/testutil" + "golang.org/x/crypto/ssh" ) type heartbeatTransport func(*http.Request) (*http.Response, error) @@ -28,6 +32,8 @@ func TestFixedDropletHeartbeatAndStatusWait(t *testing.T) { t.Run(command+"/"+change, func(t *testing.T) { api := &fakeDigitalOceanAPI{} b := newTestBackend(t, api) + port, refreshes := guestExpirySSHFixture(t) + b.Cfg.SSHPort = port // Ensure renewal changes activity tags even within one wall-clock second. b.RT.Clock = fixedClock{t: time.Now().Add(-time.Minute)} repo := t.TempDir() @@ -46,6 +52,7 @@ func TestFixedDropletHeartbeatAndStatusWait(t *testing.T) { t.Fatalf("fixed acquisition lost ownership: %+v, %v", claim, err) } if err := core.WithDurableLeaseClaimLock(lease.LeaseID, func(c *core.LeaseClaim, _ bool, persist func() error) error { + c.SSHHost = "127.0.0.1" switch change { case "scope": c.ProviderScope = "team:foreign" @@ -61,7 +68,7 @@ func TestFixedDropletHeartbeatAndStatusWait(t *testing.T) { t.Fatal(err) } config := filepath.Join(t.TempDir(), "config.yaml") - if err := os.WriteFile(config, []byte("provider: digitalocean\n"), 0600); err != nil { + if err := os.WriteFile(config, []byte(fmt.Sprintf("provider: digitalocean\nssh:\n port: %q\n", port)), 0600); err != nil { t.Fatal(err) } t.Setenv("CRABBOX_CONFIG", config) @@ -70,6 +77,9 @@ func TestFixedDropletHeartbeatAndStatusWait(t *testing.T) { t.Setenv(name, "") } item := api.created[0] + for i := range item.Networks.V4 { + item.Networks.V4[i].IPAddress = "127.0.0.1" + } labels := labelsFromTags(item.Tags) if command == "status" { labels["state"] = "provisioning" @@ -126,6 +136,14 @@ func TestFixedDropletHeartbeatAndStatusWait(t *testing.T) { } err = (core.App{Stdout: output, Stderr: &stderr}).Run(ctx, args) if change == "owned" || change == "ordinary" { + select { + case input := <-refreshes: + if !strings.HasPrefix(input, "crabbox=true\nstate=") || !strings.Contains(input, "\nkeep=true\nexpires_at=") { + t.Fatalf("invalid guest expiry refresh: %q", input) + } + default: + t.Fatalf("owned lease did not refresh guest expiry: err=%v stderr=%s", err, &stderr) + } if writes == 0 || (command == "heartbeat" && err != nil) { t.Fatalf("owned lease not touched: writes=%d err=%v stderr=%s", writes, err, &stderr) } @@ -143,7 +161,78 @@ func TestFixedDropletHeartbeatAndStatusWait(t *testing.T) { } else if writes != 0 || (command == "heartbeat" && err == nil) { t.Fatalf("foreign/incomplete claim touched: writes=%d err=%v", writes, err) } + if len(refreshes) != 0 { + t.Fatal("unexpected guest expiry refresh") + } }) } } } + +// Model the guest without executing its root-owned refresh command locally. +func guestExpirySSHFixture(t *testing.T) (string, <-chan string) { + t.Helper() + _, key, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + signer, err := ssh.NewSignerFromKey(key) + if err != nil { + t.Fatal(err) + } + config := &ssh.ServerConfig{NoClientAuth: true} + config.AddHostKey(signer) + listener, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + refreshes := make(chan string, 8) + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func() { + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(time.Minute)) + server, channels, requests, err := ssh.NewServerConn(conn, config) + if err != nil { + return // Readiness probes connect without an SSH handshake. + } + defer server.Close() + go ssh.DiscardRequests(requests) + for channel := range channels { + ch, requests, err := channel.Accept() + if err != nil { + return + } + defer ch.Close() + for req := range requests { + var command struct{ Command string } + valid := req.Type == "exec" && ssh.Unmarshal(req.Payload, &command) == nil + _ = req.Reply(valid, nil) + if !valid { + continue + } + input, err := io.ReadAll(ch) + if err != nil { + return + } + if strings.Contains(command.Command, "/etc/crabbox/.lease-expiry.") && strings.Contains(command.Command, "mv -fT") { + refreshes <- string(input) + } + _, _ = ch.SendRequest("exit-status", false, ssh.Marshal(struct{ Code uint32 }{0})) + return + } + } + }() + } + }() + _, port, err := net.SplitHostPort(listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + return port, refreshes +} diff --git a/internal/providers/firecracker/backend.go b/internal/providers/firecracker/backend.go index 9e60987479..38a4272747 100644 --- a/internal/providers/firecracker/backend.go +++ b/internal/providers/firecracker/backend.go @@ -185,7 +185,7 @@ func (b *backend) acquireOnce(ctx context.Context, req core.AcquireRequest) (cor } }() cfg.SSHKey = keyPath - payload, err := buildCloudInitPayload(cfg, leaseID, slug, publicKey) + payload, err := buildCloudInitPayload(core.WithGuestLeaseExpiry(cfg, core.TouchDirectLeaseLabels(core.DirectLeaseLabels(cfg, leaseID, slug, providerName, "", req.Keep, now), cfg, "provisioning", now)), leaseID, slug, publicKey) if err != nil { _ = b.removeStateDir(leaseStateRecord{LeaseID: leaseID}) return core.LeaseTarget{}, err diff --git a/internal/providers/incus/backend.go b/internal/providers/incus/backend.go index bf250486ef..fea52ff44a 100644 --- a/internal/providers/incus/backend.go +++ b/internal/providers/incus/backend.go @@ -688,7 +688,7 @@ func instanceConfigForCreate(cfg core.Config, labels map[string]string, publicKe bootstrapCfg.SSHFallbackPorts = nil } config := map[string]string{ - "cloud-init.user-data": core.CloudInitUserData(bootstrapCfg, publicKey), + "cloud-init.user-data": core.CloudInitUserData(core.WithGuestLeaseExpiry(bootstrapCfg, labels), publicKey), } for key, value := range labels { config[labelKey(key)] = value diff --git a/internal/providers/lambda/backend.go b/internal/providers/lambda/backend.go index 330f759ccb..bc9a7ad77a 100644 --- a/internal/providers/lambda/backend.go +++ b/internal/providers/lambda/backend.go @@ -230,7 +230,7 @@ func (b *backend) launchRequest(cfg core.Config, leaseID, slug, publicKey string InstanceTypeName: typeForConfig(cfg), Quantity: 1, SSHKeyNames: []string{key.Name}, - UserData: core.CloudInitUserData(cfg, publicKey), + UserData: core.CloudInitUserData(core.WithGuestLeaseExpiry(cfg, core.DirectLeaseLabels(cfg, leaseID, slug, providerName, "", keep, now)), publicKey), } if image := imageForConfig(cfg); image != "" { req.ImageID = image diff --git a/internal/providers/linode/backend.go b/internal/providers/linode/backend.go index c2c4b988cc..2eb6bd1429 100644 --- a/internal/providers/linode/backend.go +++ b/internal/providers/linode/backend.go @@ -172,7 +172,7 @@ func (b *linodeLeaseBackend) acquireOnce(ctx context.Context, req core.AcquireRe Tags: leaseTags(cfg, leaseID, slug, "provisioning", req.Keep, now), AuthorizedKeys: []string{publicKey}, RootPass: rootPass, - Metadata: &linodeMetadata{UserData: linodeUserData(cfg, publicKey)}, + Metadata: &linodeMetadata{UserData: linodeUserData(core.WithGuestLeaseExpiry(cfg, labelsFromTags(leaseTags(cfg, leaseID, slug, "provisioning", req.Keep, now))), publicKey)}, } if firewallID > 0 { settings, settingsErr := client.AccountSettings(ctx) diff --git a/internal/providers/linode/fixed.go b/internal/providers/linode/fixed.go index 72d1291d1a..56dd0a006c 100644 --- a/internal/providers/linode/fixed.go +++ b/internal/providers/linode/fixed.go @@ -124,7 +124,7 @@ func (b *linodeLeaseBackend) acquireFixed(ctx context.Context, req core.AcquireR claim := tx.Claim createReq.Label, createReq.Tags = core.LeaseProviderName(claim.LeaseID, claim.Slug), tagsFromLabels(tx.CreateLabels()) createReq.AuthorizedKeys = []string{publicKey} - createReq.Metadata = &linodeMetadata{UserData: linodeUserData(cfg, publicKey)} + createReq.Metadata = &linodeMetadata{UserData: linodeUserData(core.WithGuestLeaseExpiry(cfg, tx.CreateLabels()), publicKey)} createReq.RootPass, err = generateLinodeRootPass() if err != nil { return linodeInstance{}, err diff --git a/internal/providers/multipass/backend.go b/internal/providers/multipass/backend.go index f657e8d3df..74147ffcc4 100644 --- a/internal/providers/multipass/backend.go +++ b/internal/providers/multipass/backend.go @@ -129,6 +129,8 @@ func (b *backend) Acquire(ctx context.Context, req core.AcquireRequest) (core.Le cfg.SSHKey = keyPath name := core.LeaseProviderName(leaseID, slug) fmt.Fprintf(b.rt.Stderr, "provisioning provider=%s lease=%s slug=%s image=%s cpus=%d memory=%s disk=%s keep=%v\n", providerName, leaseID, slug, cfg.Multipass.Image, cfg.Multipass.CPUs, core.Blank(cfg.Multipass.Memory, "-"), core.Blank(cfg.Multipass.Disk, "-"), req.Keep) + labels := core.DirectLeaseLabels(cfg, leaseID, slug, providerName, "", req.Keep, time.Now().UTC()) + cfg = core.WithGuestLeaseExpiry(cfg, labels) if err := b.createInstance(ctx, cfg, name, leaseID, slug, publicKey); err != nil { _ = b.removeInstance(context.Background(), name) return core.LeaseTarget{}, err @@ -154,7 +156,6 @@ func (b *backend) Acquire(ctx context.Context, req core.AcquireRequest) (core.Le if err != nil { return core.LeaseTarget{}, rollbackProvisioned(err) } - labels := core.DirectLeaseLabels(cfg, leaseID, slug, providerName, "", req.Keep, time.Now().UTC()) labels["instance"] = name labels["image"] = cfg.Multipass.Image labels["ssh_user"] = cfg.Multipass.User diff --git a/internal/providers/multipass/backend_test.go b/internal/providers/multipass/backend_test.go index 6023276df5..6b8bfa1dfe 100644 --- a/internal/providers/multipass/backend_test.go +++ b/internal/providers/multipass/backend_test.go @@ -1138,6 +1138,14 @@ func TestMultipassLifecycleHeartbeatCLI(t *testing.T) { t.Skip("POSIX CLI fixture, not native Multipass") } _, lease, _ := acquiredLifecycleFixture(t) + binDir := t.TempDir() + refreshPath := filepath.Join(t.TempDir(), "lease-expiry") + sshFixture := "#!/bin/sh\ncase \"$*\" in\n*'/etc/crabbox/.lease-expiry.'*) cat >\"$CRABBOX_TEST_EXPIRY_FILE\";;\nesac\n" + if err := os.WriteFile(filepath.Join(binDir, "ssh"), []byte(sshFixture), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("CRABBOX_TEST_EXPIRY_FILE", refreshPath) helper := filepath.Join(t.TempDir(), "multipass-fixture") data := sampleInfoJSON(lease.Server.Name) script := "#!/bin/sh\ncase \"$1\" in\ninfo) printf '%s\\n' '" + data + "';;\n*) exit 91;;\nesac\n" @@ -1173,6 +1181,11 @@ func TestMultipassLifecycleHeartbeatCLI(t *testing.T) { if output.IdleTimeout != "1h30m0s" || claim.IdleTimeoutSeconds != 5400 { t.Fatal("public renewal lost override") } + refresh, err := os.ReadFile(refreshPath) + want := fmt.Sprintf("crabbox=true\nstate=%s\nkeep=%s\nexpires_at=%s\n", claim.Labels["state"], claim.Labels["keep"], claim.Labels["expires_at"]) + if err != nil || string(refresh) != want { + t.Fatalf("guest expiry refresh=%q want=%q err=%v", refresh, want, err) + } } } diff --git a/internal/providers/nebius/backend.go b/internal/providers/nebius/backend.go index be69088916..0cc115b922 100644 --- a/internal/providers/nebius/backend.go +++ b/internal/providers/nebius/backend.go @@ -75,7 +75,7 @@ func (b *backend) acquireOnce(ctx context.Context, req core.AcquireRequest) (tar now := b.now().UTC() labels := nebiusLeaseLabels(cfg, leaseID, slug, "provisioning", req.Keep, now) created := nebiusInstance{} - userData, err := renderNebiusCloudInit(cfg, publicKey) + userData, err := renderNebiusCloudInit(core.WithGuestLeaseExpiry(cfg, labels), publicKey) if err != nil { return core.LeaseTarget{}, err } diff --git a/internal/providers/ovh/backend.go b/internal/providers/ovh/backend.go index bb1e70a1a6..6886adc589 100644 --- a/internal/providers/ovh/backend.go +++ b/internal/providers/ovh/backend.go @@ -230,7 +230,7 @@ func (b *Backend) acquireOnce(ctx context.Context, req core.AcquireRequest) (tar FlavorID: cfg.ServerType, ImageID: cfg.OVH.Image, SSHKeyID: createdKey.ID, - UserData: core.CloudInitUserData(cfg, publicKey), + UserData: core.CloudInitUserData(core.WithGuestLeaseExpiry(cfg, labels), publicKey), }) if err != nil { if !isDeterminateCreateError(err) { diff --git a/internal/providers/scaleway/fixed.go b/internal/providers/scaleway/fixed.go index 548b361ec5..6a464ae450 100644 --- a/internal/providers/scaleway/fixed.go +++ b/internal/providers/scaleway/fixed.go @@ -146,7 +146,7 @@ func (b *Backend) prepareFixedServer(ctx context.Context, client Client, cfg cor labels := maps.Clone(claim.Labels) if claim.FixedCreateIntent.Attempt["userdata"] == "" { if err := client.Instance().SetServerUserData(&instance.SetServerUserDataRequest{Zone: scw.Zone(client.Zone()), ServerID: item.ID, - Key: "cloud-init", Content: strings.NewReader(core.CloudInitUserData(cfg, publicKey))}, scw.WithContext(ctx)); err != nil { + Key: "cloud-init", Content: strings.NewReader(core.CloudInitUserData(core.WithGuestLeaseExpiry(cfg, labels), publicKey))}, scw.WithContext(ctx)); err != nil { return core.LeaseTarget{}, err } if err := tx.Observe(core.FixedResourceBinding{AttemptValues: map[string]string{"userdata": "written"}}); err != nil { diff --git a/internal/providers/scaleway/provider.go b/internal/providers/scaleway/provider.go index 29ef2fd693..0174c7f33a 100644 --- a/internal/providers/scaleway/provider.go +++ b/internal/providers/scaleway/provider.go @@ -381,7 +381,7 @@ func (b *Backend) acquireOnce(ctx context.Context, req core.AcquireRequest) (tar Zone: scw.Zone(client.Zone()), ServerID: created.ID, Key: "cloud-init", - Content: strings.NewReader(core.CloudInitUserData(cfg, publicKey)), + Content: strings.NewReader(core.CloudInitUserData(core.WithGuestLeaseExpiry(cfg, labels), publicKey)), }, scw.WithContext(ctx)); err != nil { return core.LeaseTarget{}, err } diff --git a/internal/providers/tencentcloud/backend.go b/internal/providers/tencentcloud/backend.go index aef5c46e95..3e776958a6 100644 --- a/internal/providers/tencentcloud/backend.go +++ b/internal/providers/tencentcloud/backend.go @@ -179,7 +179,7 @@ func buildRunInstanceRequest(cfg core.Config, leaseID, slug, publicKey string, t InstanceType: serverTypeForConfig(cfg), InstanceCount: 1, InstanceName: core.LeaseProviderName(leaseID, slug), - UserData: base64.StdEncoding.EncodeToString([]byte(core.CloudInitUserData(cfg, publicKey))), + UserData: base64.StdEncoding.EncodeToString([]byte(core.CloudInitUserData(core.WithGuestLeaseExpiry(cfg, labelsFromTags(tags)), publicKey))), ClientToken: strings.ReplaceAll(core.ProviderKeyForLease(leaseID), "_", "-"), TagSpecification: []tagSpecification{{ ResourceType: "instance", diff --git a/internal/providers/vultr/client.go b/internal/providers/vultr/client.go index 92f760f34b..195578c9f4 100644 --- a/internal/providers/vultr/client.go +++ b/internal/providers/vultr/client.go @@ -375,7 +375,7 @@ func (c *vultrClient) createInstanceBody(ctx context.Context, cfg core.Config, p "region": vultrRegion(cfg), "plan": cfg.ServerType, "sshkey_id": []string{keyID}, - "user_data": base64.StdEncoding.EncodeToString([]byte(core.CloudInitUserData(cfg, publicKey))), + "user_data": base64.StdEncoding.EncodeToString([]byte(core.CloudInitUserData(core.WithGuestLeaseExpiry(cfg, labelsFromTags(leaseTags(cfg, leaseID, slug, "provisioning", keep, now))), publicKey))), "label": name, "hostname": name, "tags": leaseTags(cfg, leaseID, slug, "provisioning", keep, now), From 84942ca4fff971441c85b66ef1aafa7e6dde5d4b Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 13 Oct 2026 00:38:49 -0700 Subject: [PATCH 2/5] fix: preserve direct lease cleanup and bootstrap contracts Keep AWS raw-gzip and Hetzner MIME transport intact through a shared encoder, bind the Hetzner round-trip fixture to allocation labels, and bound Tencent Cloud compressed user data. Compact the shared guard and use the exported expiry refresh hook directly. Normalize Sealos native ownership and recheck live expiry before deletion. Update ownership fixtures and test renewed kept leases, expired kept leases, and non-mutating cleanup dry runs. Verified targeted failing tests and affected guard/transport/size regressions with GOTOOLCHAIN=go1.26.5 across seven packages. Documentation links pass. Codex autoreview against the origin/main merge base is scoped-clean through P2. --- CHANGELOG.md | 1 + docs/features/lifecycle-cleanup.md | 2 +- docs/providers/tencentcloud.md | 4 ++ internal/cli/aws.go | 11 +--- internal/cli/cloudinit_transport.go | 20 ++++++ internal/cli/cloudinit_transport_test.go | 61 +++++++++++++++++++ internal/cli/guest_expiry.go | 28 ++++----- internal/cli/guest_expiry_test.go | 4 +- internal/cli/hcloud_userdata.go | 12 +--- internal/cli/hcloud_userdata_test.go | 10 ++- internal/providers/lume/provider_test.go | 3 +- internal/providers/parallels/provider_test.go | 1 + internal/providers/proxmox/cleanup_test.go | 37 ++++++++++- internal/providers/sealosdevbox/claims.go | 5 ++ internal/providers/sealosdevbox/cleanup.go | 3 + .../providers/sealosdevbox/cleanup_test.go | 57 +++++++++++++++++ internal/providers/shared/direct_test.go | 48 +++++++++++++-- internal/providers/tencentcloud/backend.go | 20 +++++- .../providers/tencentcloud/cloudinit_test.go | 58 ++++++++++++++++++ .../providers/tencentcloud/provider_test.go | 10 +++ 20 files changed, 349 insertions(+), 46 deletions(-) create mode 100644 internal/cli/cloudinit_transport.go create mode 100644 internal/cli/cloudinit_transport_test.go create mode 100644 internal/providers/tencentcloud/cloudinit_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index a3b06d6eaf..f729316a77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Coordinator monthly reserved-USD budgets are raised to $100,000 fleet-wide, $50,000 per owner and $100,000 per org; reservations from finished leases had exhausted the per-owner cap at roughly a quarter of the estimated spend. - CLI: `crabbox list` streams and shows live leases by default; `--retained` includes kept terminal leases. Add `--jsonl` streaming and uncapped `--all` lease lookups while preserving the buffered `--json` array. - Direct cleanup now expires `keep=true` leases and reduces running/provisioning expiry grace from 12 hours to 1 hour; unowned machines remain untouched. +- Direct leases: preserve provider user-data limits with compressed Tencent Cloud bootstrap, retain renewed leases during cleanup, and normalize Sealos ownership labels. [PR 2806](https://github.com/openclaw/crabbox/pull/2806). ### Added diff --git a/docs/features/lifecycle-cleanup.md b/docs/features/lifecycle-cleanup.md index f6afe415d1..fe27555ddb 100644 --- a/docs/features/lifecycle-cleanup.md +++ b/docs/features/lifecycle-cleanup.md @@ -628,7 +628,7 @@ GCP retains its existing self-delete permissions requirement. File-backed bootstraps write exactly four nonsecret fields: `crabbox`, `state`, `keep`, and `expires_at`. `RefreshGuestLeaseExpiry(ctx, target, fields)` in -`internal/cli/guest_expiry.go` (internal implementation `refreshGuestLeaseExpiry`) +`internal/cli/guest_expiry.go` is the hook for a file-backed provider's lease-lifetime update: it validates the fields and writes mode 0600 through the existing SSH transport, publishing by atomic rename in `/etc/crabbox`. Ordinary CLI state touches and heartbeats refresh this file as well as provider diff --git a/docs/providers/tencentcloud.md b/docs/providers/tencentcloud.md index 3a7e4c7ad5..9049e0c8d7 100644 --- a/docs/providers/tencentcloud.md +++ b/docs/providers/tencentcloud.md @@ -12,6 +12,10 @@ through CVM user data, writes Crabbox ownership metadata as Tencent Cloud tags, waits for public IPv4 and SSH readiness, then uses the normal Crabbox SSH/sync/run/stop/cleanup path. +Linux cloud-init is gzip-compressed before base64 encoding. Crabbox keeps the +encoded user-data field within CVM's 16 KiB limit and rejects oversized payloads +before requesting an instance. + The provider is **direct-only** in this release. It does not run through the coordinator, so the local CLI must have Tencent Cloud API credentials and direct cleanup remains the operator's responsibility. diff --git a/internal/cli/aws.go b/internal/cli/aws.go index 9b28d3ad91..93006fbe20 100644 --- a/internal/cli/aws.go +++ b/internal/cli/aws.go @@ -2,7 +2,6 @@ package cli import ( "bytes" - "compress/gzip" "context" "crypto/md5" "crypto/rsa" @@ -671,15 +670,11 @@ func (c *AWSClient) resolveLaunchAMI(ctx context.Context, cfg Config, staticImag func awsRunInstancesUserData(cfg Config, publicKey string) (string, error) { userData := []byte(awsUserData(cfg, publicKey)) if cfg.TargetOS != targetWindows && cfg.TargetOS != targetMacOS { - var compressed bytes.Buffer - writer := gzip.NewWriter(&compressed) - if _, err := writer.Write(userData); err != nil { + compressed, err := GzipCloudInit(string(userData)) + if err != nil { return "", fmt.Errorf("gzip AWS Linux cloud-init user data: %w", err) } - if err := writer.Close(); err != nil { - return "", fmt.Errorf("finish gzip AWS Linux cloud-init user data: %w", err) - } - userData = compressed.Bytes() + userData = compressed if len(userData) > awsEC2UserDataLimit { return "", fmt.Errorf("EC2 user-data limit is 16 KiB (%d raw bytes); compressed Linux cloud-init is %d bytes", awsEC2UserDataLimit, len(userData)) } diff --git a/internal/cli/cloudinit_transport.go b/internal/cli/cloudinit_transport.go new file mode 100644 index 0000000000..e61232eb08 --- /dev/null +++ b/internal/cli/cloudinit_transport.go @@ -0,0 +1,20 @@ +package cli + +import ( + "bytes" + "compress/gzip" +) + +// GzipCloudInit preserves cloud-init's binary gzip transport, without a timestamp. +// Callers apply their provider's size limit and outer encoding after compression. +func GzipCloudInit(cloudConfig string) ([]byte, error) { + var compressed bytes.Buffer + writer := gzip.NewWriter(&compressed) + if _, err := writer.Write([]byte(cloudConfig)); err != nil { + return nil, err + } + if err := writer.Close(); err != nil { + return nil, err + } + return compressed.Bytes(), nil +} diff --git a/internal/cli/cloudinit_transport_test.go b/internal/cli/cloudinit_transport_test.go new file mode 100644 index 0000000000..7ab125e9a1 --- /dev/null +++ b/internal/cli/cloudinit_transport_test.go @@ -0,0 +1,61 @@ +package cli + +import ( + "encoding/base64" + "testing" +) + +func TestDirectProviderUserDataSizeBudgets(t *testing.T) { + for _, tc := range []struct { + provider string + limit int + base64 bool + desktop bool + }{ + // Azure's decoded customData limit is 64 KB; GCP metadata values are 256 KB. + // https://learn.microsoft.com/en-us/azure/virtual-machines/custom-data + // https://cloud.google.com/compute/docs/metadata/setting-custom-metadata + {"azure", 64 * 1024, false, true}, + {"gcp", 256 * 1024, false, true}, + // DigitalOcean accepts 64 KiB of text; Linode caps the base64 field at 65535. + // https://docs.digitalocean.com/products/droplets/how-to/provide-user-data/ + // https://techdocs.akamai.com/linode-api/reference/post-linode-instance + {"digitalocean", 64 * 1024, false, false}, + {"linode", 65535, true, false}, + // Conservative regression budgets for the remaining shared bootstrap adapters. + {"vultr", 64 * 1024, true, false}, + {"scaleway", 64 * 1024, false, false}, + {"ovh", 64 * 1024, false, false}, + {"nebius", 64 * 1024, false, false}, + {"lambda", 64 * 1024, false, false}, + {"incus", 64 * 1024, false, false}, + {"multipass", 64 * 1024, false, false}, + {"firecracker", 64 * 1024, false, false}, + } { + desktops := []string{""} + if tc.desktop { + desktops = append(desktops, "xfce", "wayland", "gnome") + } + for _, desktop := range desktops { + t.Run(tc.provider+"/"+blank(desktop, "headless"), func(t *testing.T) { + cfg := baseConfig() + cfg.Provider = tc.provider + cfg.Desktop, cfg.DesktopEnv = desktop != "", desktop + cfg.Browser, cfg.Code = cfg.Desktop, cfg.Desktop + cfg.Tailscale.Enabled, cfg.Tailscale.AuthKey = true, "synthetic-auth-key" + cfg = WithGuestLeaseExpiry(cfg, map[string]string{"crabbox": "true", "state": "provisioning", "keep": "true", "expires_at": "1800000000"}) + data := cloudInit(cfg, "ssh-ed25519 fixture") + if tc.provider == "azure" { + data = azureLinuxCloudInit(cfg, "ssh-ed25519 fixture") + } + if tc.base64 { + data = base64.StdEncoding.EncodeToString([]byte(data)) + } + if len(data) > tc.limit { + t.Fatalf("user-data is %d bytes; budget is %d", len(data), tc.limit) + } + t.Logf("user-data=%d budget=%d", len(data), tc.limit) + }) + } + } +} diff --git a/internal/cli/guest_expiry.go b/internal/cli/guest_expiry.go index 522cfc068f..ced15ae08c 100644 --- a/internal/cli/guest_expiry.go +++ b/internal/cli/guest_expiry.go @@ -72,8 +72,11 @@ const guestExpiryDecision = `expiry_due() { } ` +const guestExpiryRequest = `request() { curl --connect-timeout 5 --max-time 15 -fsS "$@"; } +` + const guestExpiryGCPSource = `metadata() { - curl --connect-timeout 5 --max-time 15 -fsS -H 'Metadata-Flavor: Google' "http://metadata.google.internal/computeMetadata/v1/$1" + request -H 'Metadata-Flavor: Google' "http://metadata.google.internal/computeMetadata/v1/$1" } project="$(metadata project/project-id)" || exit 0 name="$(metadata instance/name)" || exit 0 @@ -82,18 +85,18 @@ zone="${zone_path##*/}" token_json="$(metadata instance/service-accounts/default/token)" || exit 0 token="$(printf '%s' "$token_json" | jq -er '.access_token | select(type == "string" and length > 0)')" || exit 0 [ -n "$project" ] && [ -n "$name" ] && [ -n "$zone" ] || exit 0 -instance="$(curl --connect-timeout 5 --max-time 15 -fsS -H "Authorization: Bearer $token" "https://compute.googleapis.com/compute/v1/projects/$project/zones/$zone/instances/$name")" || exit 0 +instance="$(request -H "Authorization: Bearer $token" "https://compute.googleapis.com/compute/v1/projects/$project/zones/$zone/instances/$name")" || exit 0 label() { printf '%s' "$instance" | jq -er --arg key "$1" '.labels[$key] // "" | select(type == "string")'; } provider="$(label provider)" || exit 0 [ -z "$provider" ] || [ "$provider" = gcp ] || exit 0 ` -const guestExpiryAWSSource = `metadata_token="$(curl --connect-timeout 5 --max-time 15 -fsS -X PUT -H 'X-aws-ec2-metadata-token-ttl-seconds: 60' http://169.254.169.254/latest/api/token)" || exit 0 +const guestExpiryAWSSource = `metadata_token="$(request -X PUT -H 'X-aws-ec2-metadata-token-ttl-seconds: 60' http://169.254.169.254/latest/api/token)" || exit 0 [ -n "$metadata_token" ] || exit 0 label() { - curl --connect-timeout 5 --max-time 15 -fsS -H "X-aws-ec2-metadata-token: $metadata_token" "http://169.254.169.254/latest/meta-data/tags/instance/$1" + request -H "X-aws-ec2-metadata-token: $metadata_token" "http://169.254.169.254/latest/meta-data/tags/instance/$1" } ` -const guestExpiryAzureSource = `instance="$(curl --connect-timeout 5 --max-time 15 -fsS -H 'Metadata: true' --noproxy '*' 'http://169.254.169.254/metadata/instance?api-version=2021-02-01')" || exit 0 +const guestExpiryAzureSource = `instance="$(request -H 'Metadata: true' --noproxy '*' 'http://169.254.169.254/metadata/instance?api-version=2021-02-01')" || exit 0 label() { printf '%s' "$instance" | jq -er --arg key "$1" '[.compute.tagsList[] | select(.name == $key) | .value] | if length == 1 and (.[0] | type == "string") then .[0] else error("missing or ambiguous tag") end' } @@ -112,18 +115,16 @@ func guestExpiryScript(adapter string) string { reclaim := "systemctl poweroff\n" switch adapter { case "gcp": - source = guestExpiryGCPSource - reclaim = `curl --connect-timeout 5 --max-time 15 -fsS -X DELETE -H "Authorization: Bearer $token" "https://compute.googleapis.com/compute/v1/projects/$project/zones/$zone/instances/$name" >/dev/null` + "\n" + source = guestExpiryRequest + guestExpiryGCPSource + reclaim = `request -X DELETE -H "Authorization: Bearer $token" "https://compute.googleapis.com/compute/v1/projects/$project/zones/$zone/instances/$name" >/dev/null` + "\n" case "aws": - source = guestExpiryAWSSource + source = guestExpiryRequest + guestExpiryAWSSource case "azure": - source = guestExpiryAzureSource + source = guestExpiryRequest + guestExpiryAzureSource } return "#!/bin/sh\nset -eu\n" + guestExpiryDecision + source + `crabbox="$(label crabbox)" || exit 0 state="$(label state)" || exit 0 state="$(printf '%s' "$state" | tr '[:upper:]' '[:lower:]')" || exit 0 -keep="$(label keep)" || exit 0 -: "$keep" expires_at="$(label expires_at)" || exit 0 now="$(date -u +%s)" expiry_due || exit 0 @@ -199,9 +200,6 @@ func cloudInitGuestExpiryGuardBootstrap(cfg Config) string { // uses the existing authenticated SSH transport; the old deadline remains // intact on failed writes, and readers see either complete version. func RefreshGuestLeaseExpiry(ctx context.Context, target SSHTarget, fields GuestLeaseExpiryFields) error { - return refreshGuestLeaseExpiry(ctx, target, fields) -} -func refreshGuestLeaseExpiry(ctx context.Context, target SSHTarget, fields GuestLeaseExpiryFields) error { if target.TargetOS != "" && target.TargetOS != targetLinux { return fmt.Errorf("guest expiry refresh requires Linux") } @@ -254,7 +252,7 @@ func touchLeaseWithGuestExpiry(ctx context.Context, backend LeaseTouchBackend, c } target = lease.SSH } - if err := refreshGuestLeaseExpiry(ctx, target, GuestLeaseExpiryFromLabels(server.Labels)); err != nil { + if err := RefreshGuestLeaseExpiry(ctx, target, GuestLeaseExpiryFromLabels(server.Labels)); err != nil { return server, fmt.Errorf("refresh guest lease expiry: %w", err) } return server, nil diff --git a/internal/cli/guest_expiry_test.go b/internal/cli/guest_expiry_test.go index fe4f73160b..9bf81e9717 100644 --- a/internal/cli/guest_expiry_test.go +++ b/internal/cli/guest_expiry_test.go @@ -180,12 +180,12 @@ func TestRefreshGuestLeaseExpiryAtomic(t *testing.T) { } return nil } - if err := refreshGuestLeaseExpiry(context.Background(), SSHTarget{Host: "fixture"}, fields); err != nil || !called { + if err := RefreshGuestLeaseExpiry(context.Background(), SSHTarget{Host: "fixture"}, fields); err != nil || !called { t.Fatalf("refresh: %v", err) } fields.State = "ready\n$(touch injected)" called = false - if err := refreshGuestLeaseExpiry(context.Background(), SSHTarget{}, fields); err == nil || called { + if err := RefreshGuestLeaseExpiry(context.Background(), SSHTarget{}, fields); err == nil || called { t.Fatal("unsafe fields reached SSH") } // Execute the actual transaction locally, replacing only ownership operations diff --git a/internal/cli/hcloud_userdata.go b/internal/cli/hcloud_userdata.go index 7fbd628b5e..5ab7529a78 100644 --- a/internal/cli/hcloud_userdata.go +++ b/internal/cli/hcloud_userdata.go @@ -1,8 +1,6 @@ package cli import ( - "bytes" - "compress/gzip" "encoding/base64" "fmt" "strings" @@ -13,18 +11,14 @@ func hetznerUserData(cloudConfig string) (string, error) { if len(cloudConfig) <= limit { return cloudConfig, nil } - var compressed bytes.Buffer - writer := gzip.NewWriter(&compressed) - if _, err := writer.Write([]byte(cloudConfig)); err != nil { - return "", err - } - if err := writer.Close(); err != nil { + compressed, err := GzipCloudInit(cloudConfig) + if err != nil { return "", err } // Cloud-init decodes MIME transfer encoding, then detects the gzipped cloud-config. var envelope strings.Builder envelope.WriteString("MIME-Version: 1.0\nContent-Type: application/gzip\nContent-Transfer-Encoding: base64\n\n") - encoded := base64.StdEncoding.EncodeToString(compressed.Bytes()) + encoded := base64.StdEncoding.EncodeToString(compressed) for len(encoded) > 76 { envelope.WriteString(encoded[:76] + "\n") encoded = encoded[76:] diff --git a/internal/cli/hcloud_userdata_test.go b/internal/cli/hcloud_userdata_test.go index 8c3718688d..f1cb9cdd0b 100644 --- a/internal/cli/hcloud_userdata_test.go +++ b/internal/cli/hcloud_userdata_test.go @@ -19,14 +19,17 @@ func TestHetznerDesktopUserDataFitsProviderLimit(t *testing.T) { cfg := baseConfig() cfg.Provider, cfg.Desktop, cfg.Browser, cfg.DesktopEnv = "hetzner", true, true, desktop var userData string + var labels map[string]string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var body struct { - UserData string `json:"user_data"` + UserData string `json:"user_data"` + Labels map[string]string `json:"labels"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { t.Error(err) } userData = body.UserData + labels = body.Labels _, _ = io.WriteString(w, `{"server":{"id":123}}`) })) defer server.Close() @@ -54,7 +57,10 @@ func TestHetznerDesktopUserDataFitsProviderLimit(t *testing.T) { if err != nil { t.Fatal(err) } - if string(decoded) != cloudInit(cfg, "ssh-ed25519 fixture") { + if labels["crabbox"] != "true" || labels["expires_at"] == "" { + t.Fatal("missing allocation ownership or deadline") + } + if string(decoded) != cloudInit(WithGuestLeaseExpiry(cfg, labels), "ssh-ed25519 fixture") { t.Fatal("MIME transport changed cloud-config content") } }) diff --git a/internal/providers/lume/provider_test.go b/internal/providers/lume/provider_test.go index 4b89bdab71..82126b74e4 100644 --- a/internal/providers/lume/provider_test.go +++ b/internal/providers/lume/provider_test.go @@ -197,6 +197,7 @@ func TestSpecAndAliases(t *testing.T) { func TestCleanupExpiry(t *testing.T) { now := time.Date(2026, 7, 17, 12, 0, 0, 0, time.UTC) server := core.Server{Status: "running", Labels: labels{ + "crabbox": "true", "state": "ready", "expires_at": core.LeaseLabelTime(now), }} @@ -220,7 +221,7 @@ func TestCleanupExpiry(t *testing.T) { func TestCleanupKeepsStartup(t *testing.T) { now := time.Date(2026, 7, 17, 12, 0, 0, 0, time.UTC) - server := core.Server{Status: "starting", Labels: labels{"state": "starting", "expires_at": core.LeaseLabelTime(now.Add(time.Hour))}} + server := core.Server{Status: "starting", Labels: labels{"crabbox": "true", "state": "starting", "expires_at": core.LeaseLabelTime(now.Add(time.Hour))}} if cleanup, reason := shouldCleanup(server, core.LeaseClaim{}, now); cleanup { t.Fatalf("active startup cleanup=%v reason=%q", cleanup, reason) } diff --git a/internal/providers/parallels/provider_test.go b/internal/providers/parallels/provider_test.go index 334cbd4146..ff448f5faf 100644 --- a/internal/providers/parallels/provider_test.go +++ b/internal/providers/parallels/provider_test.go @@ -834,6 +834,7 @@ func seedParallelsCleanupState(t *testing.T) (string, string) { t.Fatal(err) } labels := map[string]string{ + "crabbox": "true", "provider": "parallels", "lease": leaseID, "slug": "blue", diff --git a/internal/providers/proxmox/cleanup_test.go b/internal/providers/proxmox/cleanup_test.go index 6378be144e..1fdd14443e 100644 --- a/internal/providers/proxmox/cleanup_test.go +++ b/internal/providers/proxmox/cleanup_test.go @@ -247,6 +247,7 @@ func TestProxmoxCleanupDryRunAndKeptLease(t *testing.T) { b, c, claim, output := cleanupFixture(t) if !dry { c.servers[0].Labels["keep"] = "true" + c.servers[0].Labels["expires_at"] = core.LeaseLabelTime(time.Now().Add(time.Hour)) } if err := b.Cleanup(context.Background(), core.CleanupRequest{DryRun: dry}); err != nil { t.Fatal(err) @@ -270,8 +271,14 @@ func TestProxmoxCleanupRevalidatesLiveState(t *testing.T) { }{ {"generation before stop", 0, func(s *core.Server) { s.ImmutableID = replacementGeneration }}, {"generation after stop", 1, func(s *core.Server) { s.ImmutableID = replacementGeneration }}, - {"keep before stop", 0, func(s *core.Server) { s.Labels["keep"] = "true" }}, - {"keep after stop", 1, func(s *core.Server) { s.Labels["keep"] = "true" }}, + {"renewed kept before stop", 0, func(s *core.Server) { + s.Labels["keep"] = "true" + s.Labels["expires_at"] = core.LeaseLabelTime(time.Now().Add(time.Hour)) + }}, + {"renewed kept after stop", 1, func(s *core.Server) { + s.Labels["keep"] = "true" + s.Labels["expires_at"] = core.LeaseLabelTime(time.Now().Add(time.Hour)) + }}, {"ownership after stop", 1, func(s *core.Server) { s.Labels["lease"] = "cbx_other" }}, {"node changed", 0, func(s *core.Server) { s.HostID = "pve2" }}, } { @@ -293,6 +300,32 @@ func TestProxmoxCleanupRevalidatesLiveState(t *testing.T) { } } +func TestProxmoxCleanupDeletesExpiredKeptLease(t *testing.T) { + for _, step := range []int{-1, 0, 1} { + t.Run(fmt.Sprintf("keep at step %d", step), func(t *testing.T) { + b, c, claim, _ := cleanupFixture(t) + if step == -1 { + c.servers[0].Labels["keep"] = "true" + } else { + c.checkHook = func(current int) { + if current == step { + c.servers[0].Labels["keep"] = "true" + } + } + } + if err := b.Cleanup(context.Background(), core.CleanupRequest{}); err != nil { + t.Fatal(err) + } + if c.stopCalls != 1 || c.deleteCalls != 1 { + t.Fatalf("stop=%d delete=%d", c.stopCalls, c.deleteCalls) + } + if _, exists, err := core.ReadLeaseClaimWithPresence(claim.LeaseID); err != nil || exists { + t.Fatalf("claim exists=%t err=%v", exists, err) + } + }) + } +} + func TestProxmoxCleanupPreservesChangedClaim(t *testing.T) { b, c, claim, _ := cleanupFixture(t) c.listHook = func(call int) error { diff --git a/internal/providers/sealosdevbox/claims.go b/internal/providers/sealosdevbox/claims.go index c48ae200f4..0adb26fac1 100644 --- a/internal/providers/sealosdevbox/claims.go +++ b/internal/providers/sealosdevbox/claims.go @@ -386,6 +386,11 @@ func leaseLabelsFromDevbox(item devboxItem) map[string]string { labels["provider"] = value } } + // Native ownership is authoritative; annotations cannot grant it. + delete(labels, "crabbox") + if item.Metadata.Labels[managedByLabel] == "crabbox" && item.Metadata.Labels[providerLabel] == providerName { + labels["crabbox"] = "true" + } if item.Metadata.CreationTimestamp != "" { labels["created"] = item.Metadata.CreationTimestamp } diff --git a/internal/providers/sealosdevbox/cleanup.go b/internal/providers/sealosdevbox/cleanup.go index bbc162e864..e2819072f4 100644 --- a/internal/providers/sealosdevbox/cleanup.go +++ b/internal/providers/sealosdevbox/cleanup.go @@ -65,6 +65,9 @@ func (b *backend) Cleanup(ctx context.Context, req core.CleanupRequest) error { if err := b.validateClaimBinding(claim, validated); err != nil { return err } + if eligible, reason := core.ShouldCleanupServer(b.serverFromDevbox(validated), b.now()); !eligible { + return core.Exit(4, "refusing to delete Sealos DevBox %q: %s", server.Name, reason) + } if err := b.deleteDevbox(ctx, validated); err != nil { return err } diff --git a/internal/providers/sealosdevbox/cleanup_test.go b/internal/providers/sealosdevbox/cleanup_test.go index e089a37f33..232bc85c01 100644 --- a/internal/providers/sealosdevbox/cleanup_test.go +++ b/internal/providers/sealosdevbox/cleanup_test.go @@ -3,7 +3,9 @@ package sealosdevbox import ( "bytes" "context" + "encoding/json" "errors" + "fmt" "os" "strings" "testing" @@ -100,6 +102,61 @@ func TestCleanupRefusesResourceWhoseScopeChangesBeforeDelete(t *testing.T) { } } +func TestCleanupPreservesRenewedDevbox(t *testing.T) { + for _, duringRevalidation := range []bool{false, true} { + t.Run(fmt.Sprintf("during revalidation=%t", duringRevalidation), func(t *testing.T) { + isolateSealosState(t) + cfg := lifecycleConfig() + leaseID, slug, name := "cbx_renewed0000", "renewed", "devbox-renewed" + claimExactSealosTarget(t, cfg, leaseID, slug, name, t.TempDir(), core.SSHTarget{}) + old := cleanupDevboxJSON(cfg, leaseID, slug, name, sealosClaimScopeID(cfg), "2026-06-24T00:00:00Z") + fresh := cleanupDevboxJSON(cfg, leaseID, slug, name, sealosClaimScopeID(cfg), "2026-06-24T13:00:00Z") + fresh = strings.Replace(fresh, `"annotations":{`, `"annotations":{"crabbox.dev/keep":"true",`, 1) + listed := fresh + if duringRevalidation { + listed = old + } + runner := &lifecycleRunner{outputs: []string{`{"items":[` + listed + `]}`, fresh}} + backend := lifecycleBackend(cfg, runner) + err := backend.Cleanup(context.Background(), core.CleanupRequest{}) + if duringRevalidation && (err == nil || !strings.Contains(err.Error(), "not expired")) { + t.Fatalf("renewal error=%v", err) + } + if !duringRevalidation && err != nil { + t.Fatal(err) + } + for _, req := range runner.requests { + if strings.Contains(strings.Join(req.Args, " "), " delete ") { + t.Fatal("renewed devbox was deleted") + } + } + if _, exists, err := core.ReadLeaseClaimWithPresence(leaseID); err != nil || !exists { + t.Fatalf("claim exists=%t err=%v", exists, err) + } + }) + } +} + +func TestCleanupOwnershipUsesNativeLabels(t *testing.T) { + cfg := lifecycleConfig() + backend := lifecycleBackend(cfg, &lifecycleRunner{}) + for _, managedBy := range []string{"crabbox", "dashboard", ""} { + t.Run("managed by "+managedBy, func(t *testing.T) { + data := cleanupDevboxJSON(cfg, "cbx_owned000000", "owned", "devbox-owned", sealosClaimScopeID(cfg), "2026-06-24T00:00:00Z") + data = strings.Replace(data, `"app.kubernetes.io/managed-by":"crabbox"`, `"app.kubernetes.io/managed-by":"`+managedBy+`"`, 1) + data = strings.Replace(data, `"annotations":{`, `"annotations":{"crabbox.dev/crabbox":"true",`, 1) + var item devboxItem + if err := json.Unmarshal([]byte(data), &item); err != nil { + t.Fatal(err) + } + eligible, reason := core.ShouldCleanupServer(backend.serverFromDevbox(item), backend.now()) + if eligible != (managedBy == "crabbox") { + t.Fatalf("eligible=%t reason=%s", eligible, reason) + } + }) + } +} + func TestCleanupRemovesOnlySameScopeStaleClaims(t *testing.T) { isolateSealosState(t) cfg := lifecycleConfig() diff --git a/internal/providers/shared/direct_test.go b/internal/providers/shared/direct_test.go index 0d46a888b8..3eab4f6a1c 100644 --- a/internal/providers/shared/direct_test.go +++ b/internal/providers/shared/direct_test.go @@ -319,19 +319,22 @@ func TestCleanupServersSkipsDeletesAndDryRuns(t *testing.T) { servers := []core.Server{ testCleanupServerWithLabels("kept", map[string]string{ "keep": "true", - "expires_at": clock.now.Add(-time.Hour).Format(time.RFC3339Nano), + "expires_at": clock.now.Add(time.Hour).Format(time.RFC3339Nano), }), testCleanupServerWithLabels("failed", map[string]string{ "state": "failed", }), + testCleanupServerWithLabels("expired-kept", map[string]string{ + "keep": "true", "expires_at": clock.now.Add(-time.Hour).Format(time.RFC3339Nano), + }), } if err := backend.CleanupServers(context.Background(), core.CleanupRequest{}, servers); err != nil { t.Fatalf("CleanupServers returned error: %v", err) } - if strings.Join(deleted, ",") != "failed" { - t.Fatalf("deleted=%v, want failed", deleted) + if strings.Join(deleted, ",") != "failed,expired-kept" { + t.Fatalf("deleted=%v, want failed,expired-kept", deleted) } - if got := stderr.String(); !strings.Contains(got, "skip server id=kept name=kept reason=keep=true") || !strings.Contains(got, "delete server id=failed name=failed") { + if got := stderr.String(); !strings.Contains(got, "skip server id=kept name=kept reason=not expired") || !strings.Contains(got, "delete server id=failed name=failed") { t.Fatalf("stderr=%q, want skip and delete lines", got) } @@ -350,6 +353,28 @@ func TestCleanupServersSkipsDeletesAndDryRuns(t *testing.T) { } } +func TestCleanupServersRejectsUnowned(t *testing.T) { + for _, owner := range []string{"", "false"} { + t.Run("owner="+owner, func(t *testing.T) { + backend := DirectSSHBackend{ + RT: core.Runtime{Stderr: io.Discard}, + PrepareCleanup: func(context.Context, core.Server) (core.Server, bool, CleanupSkipReason, error) { + t.Fatal("unowned server reached preparation") + return core.Server{}, false, "", nil + }, + Delete: func(context.Context, core.Config, core.Server) error { + t.Fatal("unowned server deleted") + return nil + }, + } + server := core.Server{Labels: map[string]string{"crabbox": owner, "state": "failed", "expires_at": "1"}} + if err := backend.CleanupServers(context.Background(), core.CleanupRequest{}, []core.Server{server}); err != nil { + t.Fatal(err) + } + }) + } +} + func TestCleanupServersRequiresDeleteAndPropagatesDeleteErrors(t *testing.T) { clock := &testCleanupClock{now: time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)} server := testCleanupServer("old", clock.now.Add(-time.Hour)) @@ -477,9 +502,10 @@ func TestCleanupServersRechecksPreparedServerAndClaimEligibility(t *testing.T) { prepare: func(server core.Server) core.Server { server.Labels = CloneLabels(server.Labels) server.Labels["keep"] = "true" + server.Labels["expires_at"] = clock.now.Add(time.Hour).Format(time.RFC3339Nano) return server }, - reason: "keep=true", + reason: "not expired", }, { name: "prepared claim was renewed", @@ -489,6 +515,7 @@ func TestCleanupServersRechecksPreparedServerAndClaimEligibility(t *testing.T) { Provider: "example", Revision: "renewed", Labels: map[string]string{ + "crabbox": "true", "lease": "cbx_aaaaaaaaaaaa", "keep": "false", "expires_at": clock.now.Add(time.Hour).Format(time.RFC3339Nano), @@ -501,6 +528,15 @@ func TestCleanupServersRechecksPreparedServerAndClaimEligibility(t *testing.T) { }, reason: "not expired", }, + { + name: "refreshed server lost ownership", + prepare: func(server core.Server) core.Server { + server.Labels = CloneLabels(server.Labels) + delete(server.Labels, "crabbox") + return server + }, + reason: "missing labels", + }, } { t.Run(test.name, func(t *testing.T) { var stderr bytes.Buffer @@ -655,6 +691,8 @@ func testCleanupServer(name string, expiresAt time.Time) core.Server { } func testCleanupServerWithLabels(name string, labels map[string]string) core.Server { + labels = CloneLabels(labels) + labels["crabbox"] = "true" return core.Server{CloudID: name, Name: name, Labels: labels} } diff --git a/internal/providers/tencentcloud/backend.go b/internal/providers/tencentcloud/backend.go index 3e776958a6..dad5c4896a 100644 --- a/internal/providers/tencentcloud/backend.go +++ b/internal/providers/tencentcloud/backend.go @@ -172,6 +172,10 @@ func buildRunInstanceRequest(cfg core.Config, leaseID, slug, publicKey string, t if err != nil { return runInstanceRequest{}, err } + userData, err := tencentCloudUserData(core.WithGuestLeaseExpiry(cfg, labelsFromTags(tags)), publicKey) + if err != nil { + return runInstanceRequest{}, err + } req := runInstanceRequest{ InstanceChargeType: chargeType, Placement: placement{Zone: zoneForConfig(cfg)}, @@ -179,7 +183,7 @@ func buildRunInstanceRequest(cfg core.Config, leaseID, slug, publicKey string, t InstanceType: serverTypeForConfig(cfg), InstanceCount: 1, InstanceName: core.LeaseProviderName(leaseID, slug), - UserData: base64.StdEncoding.EncodeToString([]byte(core.CloudInitUserData(core.WithGuestLeaseExpiry(cfg, labelsFromTags(tags)), publicKey))), + UserData: userData, ClientToken: strings.ReplaceAll(core.ProviderKeyForLease(leaseID), "_", "-"), TagSpecification: []tagSpecification{{ ResourceType: "instance", @@ -205,6 +209,20 @@ func buildRunInstanceRequest(cfg core.Config, leaseID, slug, publicKey string, t return req, nil } +func tencentCloudUserData(cfg core.Config, publicKey string) (string, error) { + compressed, err := core.GzipCloudInit(core.CloudInitUserData(cfg, publicKey)) + if err != nil { + return "", fmt.Errorf("gzip Tencent Cloud user data: %w", err) + } + encoded := base64.StdEncoding.EncodeToString(compressed) + // Keep even the encoded field within CVM's documented 16 KB user-data limit. + // https://www.tencentcloud.com/document/api/213/15730 + if len(encoded) > 16*1024 { + return "", fmt.Errorf("Tencent Cloud user-data limit is 16 KiB; compressed base64 cloud-init is %d bytes", len(encoded)) + } + return encoded, nil +} + func tencentCloudChargeType(cfg core.Config) (string, error) { // Tencent historically used hourly billing despite the global Spot default. market := "on-demand" diff --git a/internal/providers/tencentcloud/cloudinit_test.go b/internal/providers/tencentcloud/cloudinit_test.go new file mode 100644 index 0000000000..de208ef3b0 --- /dev/null +++ b/internal/providers/tencentcloud/cloudinit_test.go @@ -0,0 +1,58 @@ +package tencentcloud + +import ( + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/base64" + "io" + "strconv" + "strings" + "testing" + "time" + + core "github.com/openclaw/crabbox/internal/cli" +) + +func TestTencentCloudUserDataFitsProviderLimit(t *testing.T) { + cfg := cfgForRun(core.BaseConfig()) + cfg.Tailscale.Enabled, cfg.Tailscale.AuthKey = true, "synthetic-auth-key" + tags := leaseTags(cfg, "cbx_123456abcdef", "fixture", "provisioning", true, time.Unix(1800000000, 0)) + req, err := buildRunInstanceRequest(cfg, "cbx_123456abcdef", "fixture", "ssh-ed25519 fixture", tags) + if err != nil { + t.Fatal(err) + } + if len(req.UserData) > 16*1024 { + t.Fatalf("encoded user-data is %d bytes; limit is 16384", len(req.UserData)) + } + compressed, err := base64.StdEncoding.DecodeString(req.UserData) + if err != nil { + t.Fatal(err) + } + gz, err := gzip.NewReader(bytes.NewReader(compressed)) + if err != nil { + t.Fatal(err) + } + defer gz.Close() + decoded, err := io.ReadAll(gz) + if err != nil { + t.Fatal(err) + } + want := core.CloudInitUserData(core.WithGuestLeaseExpiry(cfg, labelsFromTags(tags)), "ssh-ed25519 fixture") + if string(decoded) != want || !gz.ModTime.IsZero() { + t.Fatal("gzip changed cloud-init content or included a timestamp") + } + t.Logf("encoded user-data=%d limit=16384", len(req.UserData)) +} + +func TestTencentCloudUserDataRejectsOversizedPayload(t *testing.T) { + var entropy bytes.Buffer + for i := range 1024 { + digest := sha256.Sum256([]byte(strconv.Itoa(i))) + entropy.Write(digest[:]) + } + _, err := tencentCloudUserData(cfgForRun(core.BaseConfig()), "ssh-rsa "+base64.StdEncoding.EncodeToString(entropy.Bytes())) + if err == nil || !strings.Contains(err.Error(), "user-data limit") { + t.Fatalf("oversized payload error=%v", err) + } +} diff --git a/internal/providers/tencentcloud/provider_test.go b/internal/providers/tencentcloud/provider_test.go index 6942e7fdf3..32c1e35e2b 100644 --- a/internal/providers/tencentcloud/provider_test.go +++ b/internal/providers/tencentcloud/provider_test.go @@ -2,6 +2,7 @@ package tencentcloud import ( "bytes" + "compress/gzip" "context" "encoding/base64" "encoding/json" @@ -239,6 +240,15 @@ func TestBuildRunInstanceRequest(t *testing.T) { if err != nil { t.Fatal(err) } + gz, err := gzip.NewReader(bytes.NewReader(userData)) + if err != nil { + t.Fatal(err) + } + defer gz.Close() + userData, err = io.ReadAll(gz) + if err != nil { + t.Fatal(err) + } if !strings.Contains(string(userData), "ssh-ed25519 AAAATEST") { t.Fatalf("user data does not contain public key") } From a4b8e76ce4b25f347923ff78e7828aa2ae6729d9 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 13 Oct 2026 01:00:29 -0700 Subject: [PATCH 3/5] test: close expiry fixture SSH channels gracefully Wait for the client channel-close acknowledgement before closing TCP, so native Windows OpenSSH receives the fixture exit status instead of intermittently returning 255 after a successful refresh. The targeted DigitalOcean heartbeat/status test passes with GOTOOLCHAIN=go1.26.5. Codex autoreview is scoped-clean through P2. --- internal/providers/digitalocean/heartbeat_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/providers/digitalocean/heartbeat_test.go b/internal/providers/digitalocean/heartbeat_test.go index 41cfa50ef0..36419c8574 100644 --- a/internal/providers/digitalocean/heartbeat_test.go +++ b/internal/providers/digitalocean/heartbeat_test.go @@ -224,6 +224,10 @@ func guestExpirySSHFixture(t *testing.T) (string, <-chan string) { refreshes <- string(input) } _, _ = ch.SendRequest("exit-status", false, ssh.Marshal(struct{ Code uint32 }{0})) + _ = ch.Close() + // Native OpenSSH must receive the exit status before TCP closes. + for range requests { + } return } } From 0e3ab66715f3dadac66c4bf7ae11a465636e617d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 13 Oct 2026 01:28:54 -0700 Subject: [PATCH 4/5] ci: align Go module test timeout with coverage budget --- scripts/test-go-modules.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test-go-modules.sh b/scripts/test-go-modules.sh index d30450c199..1076096c11 100755 --- a/scripts/test-go-modules.sh +++ b/scripts/test-go-modules.sh @@ -23,8 +23,8 @@ while IFS= read -r -d '' modfile; do rel="." fi found=1 - printf '+ (cd %q && go test -timeout=15m ./...)\n' "$rel" - (cd "$dir" && go test -timeout=15m ./...) + printf '+ (cd %q && go test -timeout=20m ./...)\n' "$rel" + (cd "$dir" && go test -timeout=20m ./...) done <"$modules_file" if [[ "$found" -eq 0 ]]; then From b7033352fbbe3d80acb8279746015c27df194661 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 13 Oct 2026 01:45:16 -0700 Subject: [PATCH 5/5] test: align module runner expectations with timeout budget --- scripts/test-go-modules.test.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/test-go-modules.test.js b/scripts/test-go-modules.test.js index 2a79455d27..d17919daa0 100644 --- a/scripts/test-go-modules.test.js +++ b/scripts/test-go-modules.test.js @@ -60,7 +60,7 @@ exit "\${GO_TEST_EXIT:-0}" } for (const discovery of ["real find", "fake find"]) { - test(`all modules including root run once with a 15m timeout and spaced paths (${discovery})`, (t) => { + test(`all modules including root run once with a 20m timeout and spaced paths (${discovery})`, (t) => { const fixture = moduleFixture( t, discovery === "fake find" @@ -76,10 +76,10 @@ for (const discovery of ["real find", "fake find"]) { assert.deepEqual( fixture.calls().sort(), fixture.modules - .map((module) => `${fs.realpathSync(path.join(fixture.root, module))}\ttest -timeout=15m ./...`) + .map((module) => `${fs.realpathSync(path.join(fixture.root, module))}\ttest -timeout=20m ./...`) .sort(), ); - assert.match(result.stdout, /go test -timeout=15m \.\/\.\.\./); + assert.match(result.stdout, /go test -timeout=20m \.\/\.\.\./); }); } @@ -113,5 +113,5 @@ test("ordinary module test failure propagates without running later modules", (t ); const result = fixture.run(23); assert.equal(result.status, 23, result.stdout + result.stderr); - assert.deepEqual(fixture.calls(), [`${fs.realpathSync(fixture.root)}\ttest -timeout=15m ./...`]); + assert.deepEqual(fixture.calls(), [`${fs.realpathSync(fixture.root)}\ttest -timeout=20m ./...`]); });