diff --git a/.agents/reference/domain-index.md b/.agents/reference/domain-index.md index bb0c83bc9a..ca12e89c3d 100644 --- a/.agents/reference/domain-index.md +++ b/.agents/reference/domain-index.md @@ -64,7 +64,7 @@ This index describes what is catalogued, not what is usable now. Query `scripts/ | Google Cloud operations | Google Cloud, GCP, gcloud, Google Cloud project, Google OAuth consent screen, OAuth client, OAuth verification, Google Auth Platform | `services/hosting/google-cloud.md` | | Site operations | website access, app access, site inventory, mapped domain, multisite child, hosting account | `reference/site-operations.md`, `scripts/site-context-helper.sh` | | Networking/VPN | VPN, mesh, WireGuard, NetBird, Tailscale, Nostr VPN, Obscura, MPR, multi-party relay, Mullvad, QUIC obfuscation, FIPS, remote compute network, remote OpenCode workers, Headscale | `reference/mesh-remote-workers.md`, `services/networking/netbird.md`, `services/networking/tailscale.md`, `services/networking/nostr-vpn.md`, `services/networking/obscuravpn.md`, `tools/containers/remote-dispatch.md` | -| Infrastructure | GPU, containers, agent sandbox, durable compute session, OrbStack, remote dispatch, servers | `reference/agent-sandbox-lifecycle.md`, `tools/infrastructure/cloud-gpu.md`, `tools/containers/orbstack.md`, `tools/containers/remote-dispatch.md` | +| Infrastructure | GPU, containers, agent sandbox, durable compute session, OrbStack, remote dispatch, servers, Docker cleanup, disposable test sites, address pools | `reference/agent-sandbox-lifecycle.md`, `tools/infrastructure/cloud-gpu.md`, `tools/containers/orbstack.md`, `tools/containers/remote-dispatch.md`, `tools/containers/disposable-resources.md` | | Accessibility | accessibility, WCAG, a11y, contrast, screen reader | `tools/accessibility/accessibility-audit.md` | | OpenAPI exploration | OpenAPI, API spec, endpoint search, schema discovery | `tools/context/openapi-search.md` | | Local models | local model, llama.cpp, GGUF, Hugging Face, offline | `tools/local-models/local-models.md`, `tools/local-models/huggingface.md`, `scripts/local-model-helper.sh` | diff --git a/.agents/scripts/docker-resource-helper.sh b/.agents/scripts/docker-resource-helper.sh new file mode 100755 index 0000000000..387b27d318 --- /dev/null +++ b/.agents/scripts/docker-resource-helper.sh @@ -0,0 +1,860 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2025-2026 Marcus Quinn +# docker-resource-helper.sh — ownership labels and cleanup for disposable +# Docker resources (test sites, plugin-check stacks, review stacks) that +# aidevops sessions and scripts start (GH#34285). +# +# Usage: +# docker-resource-helper.sh labels [--repo OWNER/REPO] [--ref GH#N|PR#N] +# [--worktree PATH] [--ttl-hours N] # one --label=k=v per line +# docker-resource-helper.sh compose-labels [same options] [--indent N] +# docker-resource-helper.sh inventory # labelled resources as TSV +# docker-resource-helper.sh teardown --worktree PATH [--dry-run] [--force] +# docker-resource-helper.sh reap [--apply] [--max-seconds N] [--verbose] +# docker-resource-helper.sh report [--max-seconds N] +# +# Safety rules (see tools/containers/disposable-resources.md): +# - Only resources labelled sh.aidevops.owner=1 are ever removed. +# Unlabelled resources are only counted by `report`. +# - `reap` is report-only unless --apply is given. It removes a labelled +# resource older than 30 minutes only when (a) its worktree path no +# longer exists AND its issue/PR is closed, or (b) its TTL expired and no +# running container uses it. A failed issue/PR lookup means "keep". +# - Removal order: containers, then networks, then volumes. A network or +# volume still attached to a container that is not being removed is kept. +# - Docker missing or the daemon not answering within a short timeout: +# every entry point logs one line and exits 0. +# +# Environment: +# AIDEVOPS_DOCKER_TTL_HOURS default TTL label value (72) +# AIDEVOPS_DOCKER_TIMEOUT_SECONDS daemon probe timeout (5) +# AIDEVOPS_DOCKER_CMD_TIMEOUT_SECONDS per docker command timeout (30) +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" || exit 1 +# shellcheck source=./shared-constants.sh +source "${SCRIPT_DIR}/shared-constants.sh" + +readonly DRH_PREFIX="sh.aidevops" +readonly DRH_TAG="[docker-resource]" +readonly DRH_US=$'\x1f' +# Never reap anything younger than this: another session may be starting it. +readonly DRH_GRACE_SECONDS=1800 +readonly DRH_LABEL_KEYS="owner repo ref worktree created ttl-hours" +readonly DRH_MODE_APPLY="apply" +readonly DRH_MODE_DRY_RUN="dry-run" +readonly DRH_KIND_CONTAINER="container" +readonly DRH_DECISION_REMOVE="remove" +readonly DRH_STATE_UNKNOWN="unknown" + +DRH_DEFAULT_TTL="${AIDEVOPS_DOCKER_TTL_HOURS:-72}" +[[ "$DRH_DEFAULT_TTL" =~ ^[0-9]+$ ]] || DRH_DEFAULT_TTL=72 +DRH_PROBE_TIMEOUT="${AIDEVOPS_DOCKER_TIMEOUT_SECONDS:-5}" +[[ "$DRH_PROBE_TIMEOUT" =~ ^[1-9][0-9]*$ ]] || DRH_PROBE_TIMEOUT=5 +DRH_CMD_TIMEOUT="${AIDEVOPS_DOCKER_CMD_TIMEOUT_SECONDS:-30}" +[[ "$DRH_CMD_TIMEOUT" =~ ^[1-9][0-9]*$ ]] || DRH_CMD_TIMEOUT=30 + +DRH_TMP="" +DRH_NOW=0 +DRH_START=0 +DRH_MAX_SECONDS=0 +DRH_BUDGET_EXHAUSTED=0 +DRH_FORCE=0 +DRH_VERBOSE=0 +DRH_REF_CACHE="" +DRH_REF_STATE="" +DRH_GH_LOOKUPS=0 +DRH_DECISION="" +DRH_REASON="" +DRH_REMOVED_IDS=" " +DRH_CANDIDATE_IDS=" " +DRH_COUNT_REMOVED=0 +DRH_COUNT_FAILED=0 +DRH_COUNT_KEPT=0 +DRH_COUNT_CANDIDATES=0 +# Label option values (set by _drh_parse_label_opts). +DRH_L_REPO="" +DRH_L_REF="" +DRH_L_WORKTREE="" +DRH_L_TTL="" +DRH_L_INDENT=0 + +_drh_log() { + printf '%s %s\n' "$DRH_TAG" "$*" + return 0 +} + +_drh_cleanup_tmp() { + if [[ -n "$DRH_TMP" && -d "$DRH_TMP" ]]; then + rm -rf "$DRH_TMP" + fi + return 0 +} + +_drh_init_tmp() { + [[ -z "$DRH_TMP" ]] || return 0 + DRH_TMP=$(mktemp -d "${TMPDIR:-/tmp}/docker-resource.XXXXXX") || return 1 + trap _drh_cleanup_tmp EXIT + return 0 +} + +_drh_docker() { + timeout_sec "$DRH_CMD_TIMEOUT" docker "$@" + return $? +} + +# Return 0 when docker is installed and the daemon answers quickly. +_drh_docker_ready() { + if ! command -v docker >/dev/null 2>&1; then + _drh_log "docker not installed; skipping" + return 1 + fi + if ! timeout_sec "$DRH_PROBE_TIMEOUT" docker info --format '{{.ServerVersion}}' >/dev/null 2>&1; then + _drh_log "docker daemon not reachable within ${DRH_PROBE_TIMEOUT}s; skipping" + return 1 + fi + return 0 +} + +_drh_budget_left() { + if [[ "$DRH_MAX_SECONDS" -gt 0 && $((SECONDS - DRH_START)) -ge "$DRH_MAX_SECONDS" ]]; then + DRH_BUDGET_EXHAUSTED=1 + return 1 + fi + return 0 +} + +_drh_set_max_seconds() { + local value="$1" + if [[ ! "$value" =~ ^[0-9]+$ ]]; then + printf 'Invalid value for --max-seconds: %s\n' "$value" >&2 + return 1 + fi + DRH_MAX_SECONDS="$value" + return 0 +} + +_drh_short() { + local id="$1" + printf '%s' "${id:0:12}" + return 0 +} + +# --- labels ------------------------------------------------------------------ + +_drh_default_repo() { + local url rest name owner + url=$(git remote get-url origin 2>/dev/null) || return 0 + url="${url%.git}" + url="${url%/}" + name="${url##*/}" + rest="${url%/*}" + owner="${rest##*[:/]}" + if [[ "${owner}/${name}" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then + printf '%s/%s' "$owner" "$name" + fi + return 0 +} + +_drh_default_ref() { + local branch="" + if [[ "${WORKER_ISSUE_NUMBER:-}" =~ ^[0-9]+$ ]]; then + printf 'GH#%s' "$WORKER_ISSUE_NUMBER" + return 0 + fi + branch=$(git branch --show-current 2>/dev/null) || return 0 + if [[ "$branch" =~ [Gg][Hh]-?([0-9]+) ]]; then + printf 'GH#%s' "${BASH_REMATCH[1]}" + fi + return 0 +} + +_drh_default_worktree() { + git rev-parse --show-toplevel 2>/dev/null || pwd -P + return 0 +} + +_drh_parse_label_opts() { + DRH_L_REPO="" + DRH_L_REF="" + DRH_L_WORKTREE="" + DRH_L_TTL="$DRH_DEFAULT_TTL" + DRH_L_INDENT=0 + local have_repo=0 have_ref=0 opt val + while [[ $# -gt 0 ]]; do + local opt="$1" val="${2:-}" + case "$opt" in + --repo | --ref | --worktree | --ttl-hours | --indent) + [[ $# -ge 2 ]] || { + printf 'Missing value for %s\n' "$opt" >&2 + return 1 + } + case "$opt" in + --repo) + DRH_L_REPO="$val" + have_repo=1 + ;; + --ref) + DRH_L_REF="$val" + have_ref=1 + ;; + --worktree) DRH_L_WORKTREE="$val" ;; + --ttl-hours) DRH_L_TTL="$val" ;; + --indent) DRH_L_INDENT="$val" ;; + esac + shift + ;; + *) + printf 'Unknown option: %s\n' "$opt" >&2 + return 1 + ;; + esac + shift + done + [[ "$have_repo" -eq 1 ]] || DRH_L_REPO=$(_drh_default_repo) + [[ "$have_ref" -eq 1 ]] || DRH_L_REF=$(_drh_default_ref) + [[ -n "$DRH_L_WORKTREE" ]] || DRH_L_WORKTREE=$(_drh_default_worktree) + while [[ "$DRH_L_WORKTREE" == */ && "$DRH_L_WORKTREE" != "/" ]]; do DRH_L_WORKTREE="${DRH_L_WORKTREE%/}"; done + [[ -z "$DRH_L_REPO" || "$DRH_L_REPO" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || + { + printf 'Invalid --repo (expected owner/repo): %s\n' "$DRH_L_REPO" >&2 + return 1 + } + [[ -z "$DRH_L_REF" || "$DRH_L_REF" =~ ^(GH|PR)#[0-9]+$ ]] || + { + printf 'Invalid --ref (expected GH#N or PR#N): %s\n' "$DRH_L_REF" >&2 + return 1 + } + [[ "$DRH_L_TTL" =~ ^[0-9]+$ ]] || { + printf 'Invalid --ttl-hours: %s\n' "$DRH_L_TTL" >&2 + return 1 + } + [[ "$DRH_L_INDENT" =~ ^[0-9]+$ ]] || { + printf 'Invalid --indent: %s\n' "$DRH_L_INDENT" >&2 + return 1 + } + case "$DRH_L_WORKTREE" in + /*) ;; + *) + printf 'Invalid --worktree (absolute path required): %s\n' "$DRH_L_WORKTREE" >&2 + return 1 + ;; + esac + case "$DRH_L_WORKTREE" in + *$'\n'* | *$'\t'* | *"$DRH_US"*) + printf 'Invalid --worktree (control characters)\n' >&2 + return 1 + ;; + esac + return 0 +} + +# Print "keyvalue" for the six owner labels. +_drh_label_pairs() { + local created="" + created=$(date +%s) + printf '%s.owner\t1\n' "$DRH_PREFIX" + printf '%s.repo\t%s\n' "$DRH_PREFIX" "$DRH_L_REPO" + printf '%s.ref\t%s\n' "$DRH_PREFIX" "$DRH_L_REF" + printf '%s.worktree\t%s\n' "$DRH_PREFIX" "$DRH_L_WORKTREE" + printf '%s.created\t%s\n' "$DRH_PREFIX" "$created" + printf '%s.ttl-hours\t%s\n' "$DRH_PREFIX" "$DRH_L_TTL" + return 0 +} + +cmd_labels() { + local key value + _drh_parse_label_opts "$@" || return 1 + while IFS=$'\t' read -r key value; do + printf -- '--label=%s=%s\n' "$key" "$value" + done < <(_drh_label_pairs) + return 0 +} + +cmd_compose_labels() { + local key value pad + _drh_parse_label_opts "$@" || return 1 + pad=$(printf '%*s' "$DRH_L_INDENT" '') + printf '%slabels:\n' "$pad" + while IFS=$'\t' read -r key value; do + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + printf '%s %s: "%s"\n' "$pad" "$key" "$value" + done < <(_drh_label_pairs) + return 0 +} + +# --- inventory --------------------------------------------------------------- + +_drh_label_format() { + local key fmt + fmt="" + for key in $DRH_LABEL_KEYS; do + fmt="${fmt}${DRH_US}{{.Label \"${DRH_PREFIX}.${key}\"}}" + done + printf '%s' "$fmt" + return 0 +} + +# Append "kind US [US state]" rows to $1. +_drh_collect_kind() { + local kind="$1" out="$2" + local filter="label=${DRH_PREFIX}.owner=1" + local labels output line suffix + suffix="" + labels=$(_drh_label_format) + case "$kind" in + container) + output=$(_drh_docker ps -a --no-trunc --filter "$filter" \ + --format "{{.ID}}${DRH_US}{{.Names}}${labels}${DRH_US}{{.State}}") || return 1 + ;; + network) + output=$(_drh_docker network ls --no-trunc --filter "$filter" \ + --format "{{.ID}}${DRH_US}{{.Name}}${labels}") || return 1 + suffix="${DRH_US}-" + ;; + volume) + output=$(_drh_docker volume ls --filter "$filter" \ + --format "{{.Name}}${DRH_US}{{.Name}}${labels}") || return 1 + suffix="${DRH_US}-" + ;; + *) return 1 ;; + esac + while IFS= read -r line; do + [[ -n "$line" ]] || continue + printf '%s%s%s%s\n' "$kind" "$DRH_US" "$line" "$suffix" >>"$out" + done <<<"$output" + return 0 +} + +# Write labelled resources to $1 (fields: kind id name owner repo ref +# worktree created ttl state, separated by \x1f). +_drh_collect() { + local out="$1" kind + : >"$out" + for kind in $DRH_KIND_CONTAINER network volume; do + _drh_collect_kind "$kind" "$out" || return 1 + done + return 0 +} + +cmd_inventory() { + local records kind id name owner repo ref worktree created ttl state + [[ $# -eq 0 ]] || { + printf 'Usage: %s inventory\n' "${0##*/}" >&2 + return 1 + } + _drh_docker_ready || return 0 + _drh_init_tmp || return 1 + records="${DRH_TMP}/records" + _drh_collect "$records" || { + _drh_log "inventory: docker listing failed; skipping" + return 0 + } + printf 'kind\tid\tname\trepo\tref\tworktree\tcreated\tttl\tstate\n' + while IFS="$DRH_US" read -r kind id name owner repo ref worktree created ttl state; do + [[ "$owner" == "1" ]] || continue + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$kind" "$(_drh_short "$id")" "$name" \ + "${repo:--}" "${ref:--}" "${worktree:--}" "${created:--}" "${ttl:--}" "${state:--}" + done <"$records" + return 0 +} + +# --- ownership rule ---------------------------------------------------------- + +# Return 0 only when the label names an absolute path that no longer exists. +_drh_worktree_gone() { + local path="$1" + case "$path" in + /?*) [[ ! -e "$path" ]] && return 0 ;; + esac + return 1 +} + +# Set DRH_REF_STATE to closed|open|unknown for repo+ref. One gh call per +# distinct ref per run; any failure means unknown (never treated as closed). +_drh_ref_state() { + local repo="$1" ref="$2" + local key cached number state + state="" + DRH_REF_STATE="$DRH_STATE_UNKNOWN" + [[ "$repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ && "$ref" =~ ^(GH|PR)#([0-9]+)$ ]] || return 0 + number="${BASH_REMATCH[2]}" + key="${repo}#${number}" + cached=$(printf '%s' "$DRH_REF_CACHE" | awk -F'\t' -v k="$key" '$1 == k { print $2; exit }') + if [[ -n "$cached" ]]; then + DRH_REF_STATE="$cached" + return 0 + fi + if command -v gh >/dev/null 2>&1; then + DRH_GH_LOOKUPS=$((DRH_GH_LOOKUPS + 1)) + state=$(timeout_sec 15 gh api "repos/${repo}/issues/${number}" --jq '.state' 2>/dev/null) || state="" + fi + case "$state" in + closed | CLOSED) DRH_REF_STATE="closed" ;; + open | OPEN) DRH_REF_STATE="open" ;; + *) DRH_REF_STATE="$DRH_STATE_UNKNOWN" ;; + esac + DRH_REF_CACHE="${DRH_REF_CACHE}${key}"$'\t'"${DRH_REF_STATE}"$'\n' + return 0 +} + +# Return 0 when the resource is (or is used by) a running container. Any +# docker failure counts as running so the TTL rule keeps the resource. +_drh_in_use_running() { + local kind="$1" id="$2" + local running + case "$kind" in + container) + running=$(_drh_docker inspect --format '{{.State.Status}}' "$id" 2>/dev/null) || return 0 + case "$running" in + exited | created | dead) return 1 ;; + esac + return 0 + ;; + network | volume) + running=$(_drh_docker ps -q --no-trunc --filter "${kind}=${id}") || return 0 + [[ -n "$running" ]] && return 0 + return 1 + ;; + esac + return 0 +} + +# Set DRH_DECISION (remove|keep) and DRH_REASON for one reap record. +_drh_decide() { + local kind="$1" id="$2" owner="$3" repo="$4" ref="$5" worktree="$6" created="$7" ttl="$8" state="$9" + local age=0 + DRH_DECISION="keep" + if [[ "$owner" != "1" ]]; then + DRH_REASON="unlabelled" + return 0 + fi + if [[ ! "$created" =~ ^[0-9]+$ ]]; then + DRH_REASON="invalid-created-label" + return 0 + fi + [[ "$ttl" =~ ^[0-9]+$ ]] || ttl="$DRH_DEFAULT_TTL" + age=$((DRH_NOW - created)) + if [[ "$age" -lt "$DRH_GRACE_SECONDS" ]]; then + DRH_REASON="created-within-30m" + return 0 + fi + DRH_REASON="worktree-present" + if _drh_worktree_gone "$worktree"; then + _drh_ref_state "$repo" "$ref" + if [[ "$DRH_REF_STATE" == "closed" ]]; then + DRH_DECISION="$DRH_DECISION_REMOVE" + DRH_REASON="worktree-gone-ref-closed" + return 0 + fi + DRH_REASON="ref-${DRH_REF_STATE}" + fi + [[ "$age" -gt $((ttl * 3600)) ]] || return 0 + # The TTL rule never removes anything running or used by a running container. + if [[ "$kind" == "$DRH_KIND_CONTAINER" ]]; then + case "$state" in + exited | created | dead) ;; + *) + DRH_REASON="ttl-expired-but-running" + return 0 + ;; + esac + elif _drh_in_use_running "$kind" "$id"; then + DRH_REASON="ttl-expired-but-running" + return 0 + fi + DRH_DECISION="$DRH_DECISION_REMOVE" + DRH_REASON="ttl-expired" + return 0 +} + +# --- removal ----------------------------------------------------------------- + +# Count containers attached to a network/volume that are not in $3 (a +# space-delimited id set). Prints the count; returns 1 on docker failure. +_drh_blocking_attachments() { + local kind="$1" id="$2" exclude="$3" + local attached cid blockers=0 + attached=$(_drh_docker ps -a -q --no-trunc --filter "${kind}=${id}") || return 1 + for cid in $attached; do + [[ "$exclude" == *" ${cid} "* ]] || blockers=$((blockers + 1)) + done + printf '%s' "$blockers" + return 0 +} + +# Remove one resource; "no such ..."/"not found" counts as already removed. +_drh_remove() { + local kind="$1" id="$2" + local err lower + case "$kind" in + container) err=$(_drh_docker rm -f "$id" 2>&1 >/dev/null) && return 0 ;; + network) err=$(_drh_docker network rm "$id" 2>&1 >/dev/null) && return 0 ;; + volume) err=$(_drh_docker volume rm "$id" 2>&1 >/dev/null) && return 0 ;; + *) return 1 ;; + esac + lower=$(printf '%s' "$err" | tr '[:upper:]' '[:lower:]') + case "$lower" in + *"no such"* | *"not found"*) return 0 ;; + esac + _drh_log "remove failed: ${kind} ${id:0:12}: ${err}" + return 1 +} + +# Re-check the removal condition immediately before acting (another session +# may have restarted the stack or recreated the worktree since planning). +_drh_still_eligible() { + local kind="$1" id="$2" reason="$3" worktree="$4" + case "$reason" in + ttl-expired) + _drh_in_use_running "$kind" "$id" && return 1 + ;; + worktree-gone-ref-closed) + _drh_worktree_gone "$worktree" || return 1 + ;; + worktree-teardown) + [[ "$DRH_FORCE" -eq 1 ]] || _drh_worktree_gone "$worktree" || return 1 + ;; + esac + return 0 +} + +# Execute one planned row. $1=mode (apply|dry-run). +_drh_execute_row() { + local mode="$1" kind="$2" id="$3" name="$4" reason="$5" repo="$6" ref="$7" worktree="$8" + local detail blockers exclude + detail="${kind} ${name} ($(_drh_short "$id")) reason=${reason} repo=${repo:--} ref=${ref:--} worktree=${worktree:--}" + if [[ "$kind" != "$DRH_KIND_CONTAINER" ]]; then + exclude="$DRH_REMOVED_IDS" + [[ "$mode" == "$DRH_MODE_APPLY" ]] || exclude="$DRH_CANDIDATE_IDS" + blockers=$(_drh_blocking_attachments "$kind" "$id" "$exclude") || blockers="$DRH_STATE_UNKNOWN" + if [[ "$blockers" != "0" ]]; then + DRH_COUNT_KEPT=$((DRH_COUNT_KEPT + 1)) + _drh_log "kept ${detail} attached_containers=${blockers}" + return 0 + fi + fi + if [[ "$mode" != "$DRH_MODE_APPLY" ]]; then + _drh_log "would-remove ${detail}" + return 0 + fi + if ! _drh_still_eligible "$kind" "$id" "$reason" "$worktree"; then + DRH_COUNT_KEPT=$((DRH_COUNT_KEPT + 1)) + _drh_log "kept ${detail} recheck=changed" + return 0 + fi + if _drh_remove "$kind" "$id"; then + DRH_COUNT_REMOVED=$((DRH_COUNT_REMOVED + 1)) + DRH_REMOVED_IDS="${DRH_REMOVED_IDS}${id} " + _drh_log "removed ${detail}" + else + DRH_COUNT_FAILED=$((DRH_COUNT_FAILED + 1)) + fi + return 0 +} + +# Execute a plan file (kind id name reason repo ref worktree) in safe order. +_drh_execute_plan() { + local mode="$1" plan="$2" + local order kind id name reason repo ref worktree + for order in $DRH_KIND_CONTAINER network volume; do + while IFS="$DRH_US" read -r kind id name reason repo ref worktree; do + [[ "$kind" == "$order" ]] || continue + if ! _drh_budget_left; then + _drh_log "budget exhausted; remaining candidates retried next run" + return 0 + fi + _drh_execute_row "$mode" "$kind" "$id" "$name" "$reason" "$repo" "$ref" "$worktree" + done <"$plan" + done + return 0 +} + +_drh_add_candidate() { + local plan="$1" kind="$2" id="$3" name="$4" reason="$5" repo="$6" ref="$7" worktree="$8" + printf '%s\n' "${kind}${DRH_US}${id}${DRH_US}${name}${DRH_US}${reason}${DRH_US}${repo}${DRH_US}${ref}${DRH_US}${worktree}" >>"$plan" + DRH_COUNT_CANDIDATES=$((DRH_COUNT_CANDIDATES + 1)) + [[ "$kind" != "$DRH_KIND_CONTAINER" ]] || DRH_CANDIDATE_IDS="${DRH_CANDIDATE_IDS}${id} " + return 0 +} + +# --- teardown ---------------------------------------------------------------- + +cmd_teardown() { + local mode="$DRH_MODE_APPLY" matched=0 target="" + local records plan opt val kind id name owner repo ref worktree created ttl state + while [[ $# -gt 0 ]]; do + local opt="$1" val="${2:-}" + case "$opt" in + --worktree) + [[ $# -ge 2 ]] || { + printf 'Missing value for --worktree\n' >&2 + return 1 + } + target="$val" + shift + ;; + --dry-run) mode="$DRH_MODE_DRY_RUN" ;; + --force) DRH_FORCE=1 ;; + *) + printf 'Unknown option: %s\n' "$opt" >&2 + return 1 + ;; + esac + shift + done + while [[ "$target" == */ && "$target" != "/" ]]; do target="${target%/}"; done + case "$target" in + /?*) ;; + *) + printf 'teardown requires --worktree \n' >&2 + return 1 + ;; + esac + if [[ -e "$target" && "$DRH_FORCE" -ne 1 ]]; then + _drh_log "teardown: worktree still exists (${target}); pass --force to remove its resources anyway" + return 1 + fi + _drh_docker_ready || return 0 + _drh_init_tmp || return 1 + records="${DRH_TMP}/records" + plan="${DRH_TMP}/plan" + : >"$plan" + _drh_collect "$records" || { + _drh_log "teardown: docker listing failed; skipping" + return 0 + } + while IFS="$DRH_US" read -r kind id name owner repo ref worktree created ttl state; do + [[ "$owner" == "1" ]] || continue + while [[ "$worktree" == */ && "$worktree" != "/" ]]; do worktree="${worktree%/}"; done + [[ "$worktree" == "$target" ]] || continue + matched=$((matched + 1)) + _drh_add_candidate "$plan" "$kind" "$id" "$name" "worktree-teardown" "$repo" "$ref" "$worktree" + done <"$records" + [[ "$matched" -gt 0 ]] || return 0 + DRH_START=$SECONDS + _drh_execute_plan "$mode" "$plan" + _drh_log "teardown mode=${mode} worktree=${target} matched=${matched} removed=${DRH_COUNT_REMOVED} kept=${DRH_COUNT_KEPT} failed=${DRH_COUNT_FAILED}" + return 0 +} + +# --- reap -------------------------------------------------------------------- + +cmd_reap() { + local mode="$DRH_MODE_DRY_RUN" labelled=0 + local records plan opt val kind id name owner repo ref worktree created ttl state + while [[ $# -gt 0 ]]; do + local opt="$1" val="${2:-}" + case "$opt" in + --apply) mode="$DRH_MODE_APPLY" ;; + --dry-run) mode="$DRH_MODE_DRY_RUN" ;; + --verbose) DRH_VERBOSE=1 ;; + --max-seconds) + _drh_set_max_seconds "$val" || return 1 + shift + ;; + *) + printf 'Unknown option: %s\n' "$opt" >&2 + return 1 + ;; + esac + shift + done + DRH_START=$SECONDS + _drh_docker_ready || return 0 + _drh_init_tmp || return 1 + records="${DRH_TMP}/records" + plan="${DRH_TMP}/plan" + : >"$plan" + _drh_collect "$records" || { + _drh_log "reap: docker listing failed; skipping" + return 0 + } + DRH_NOW=$(date +%s) + while IFS="$DRH_US" read -r kind id name owner repo ref worktree created ttl state; do + [[ "$owner" == "1" ]] || continue + labelled=$((labelled + 1)) + _drh_budget_left || break + _drh_decide "$kind" "$id" "$owner" "$repo" "$ref" "$worktree" "$created" "$ttl" "$state" + if [[ "$DRH_DECISION" == "$DRH_DECISION_REMOVE" ]]; then + _drh_add_candidate "$plan" "$kind" "$id" "$name" "$DRH_REASON" "$repo" "$ref" "$worktree" + else + DRH_COUNT_KEPT=$((DRH_COUNT_KEPT + 1)) + [[ "$DRH_VERBOSE" -ne 1 ]] || _drh_log "kept ${kind} ${name} ($(_drh_short "$id")) reason=${DRH_REASON}" + fi + done <"$records" + _drh_execute_plan "$mode" "$plan" + _drh_log "reap mode=${mode} labelled=${labelled} candidates=${DRH_COUNT_CANDIDATES} removed=${DRH_COUNT_REMOVED} kept=${DRH_COUNT_KEPT} failed=${DRH_COUNT_FAILED} gh_lookups=${DRH_GH_LOOKUPS} elapsed_s=$((SECONDS - DRH_START)) budget_s=${DRH_MAX_SECONDS} budget_exhausted=${DRH_BUDGET_EXHAUSTED}" + return 0 +} + +# --- report ------------------------------------------------------------------ + +# Convert Docker's RFC 3339 UTC timestamp to epoch seconds (GNU or BSD date). +_drh_iso_to_epoch() { + local iso="$1" + local base + base="${iso%%.*}" + base="${base%Z}" + [[ "$base" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$ ]] || return 1 + [[ "$base" != 0001-* ]] || return 1 + date -u -d "${base}" +%s 2>/dev/null && return 0 + date -u -j -f '%Y-%m-%dT%H:%M:%S' "$base" +%s 2>/dev/null && return 0 + return 1 +} + +_drh_count_lines() { + local text="$1" + if [[ -z "$text" ]]; then + printf '0' + return 0 + fi + printf '%s\n' "$text" | awk 'NF { n++ } END { print n + 0 }' + return 0 +} + +_drh_report_ownership() { + local label="label=${DRH_PREFIX}.owner=1" n_all=0 n_owned=0 + local all owned + printf '\nOwnership (only labelled resources are ever removed by aidevops):\n' + all=$(_drh_docker ps -a -q --no-trunc) || all="" + owned=$(_drh_docker ps -a -q --no-trunc --filter "$label") || owned="" + n_all=$(_drh_count_lines "$all") + n_owned=$(_drh_count_lines "$owned") + printf ' containers: labelled=%s unlabelled=%s\n' "$n_owned" "$((n_all - n_owned))" + all=$(_drh_docker network ls -q --no-trunc --filter type=custom) || all="" + owned=$(_drh_docker network ls -q --no-trunc --filter type=custom --filter "$label") || owned="" + n_all=$(_drh_count_lines "$all") + n_owned=$(_drh_count_lines "$owned") + printf ' networks (custom): labelled=%s unlabelled=%s\n' "$n_owned" "$((n_all - n_owned))" + all=$(_drh_docker volume ls -q) || all="" + owned=$(_drh_docker volume ls -q --filter "$label") || owned="" + n_all=$(_drh_count_lines "$all") + n_owned=$(_drh_count_lines "$owned") + printf ' volumes: labelled=%s unlabelled=%s\n' "$n_owned" "$((n_all - n_owned))" + return 0 +} + +_drh_report_networks() { + local empty=0 shown=0 checked=0 + local networks line id name used + networks=$(_drh_docker network ls --no-trunc --filter type=custom --format "{{.ID}}${DRH_US}{{.Name}}") || networks="" + printf '\nCustom networks with no containers (each holds an address-pool subnet):\n' + while IFS= read -r line; do + [[ -n "$line" ]] || continue + _drh_budget_left || break + id="${line%%"$DRH_US"*}" + name="${line#*"$DRH_US"}" + checked=$((checked + 1)) + used=$(_drh_docker ps -a -q --no-trunc --filter "network=${id}") || continue + [[ -z "$used" ]] || continue + empty=$((empty + 1)) + if [[ "$shown" -lt 20 ]]; then + printf ' %s\n' "$name" + shown=$((shown + 1)) + fi + done <<<"$networks" + [[ "$empty" -le "$shown" ]] || printf ' ... and %s more\n' "$((empty - shown))" + printf ' total=%s (checked %s)\n' "$empty" "$checked" + printf ' reclaim: docker network prune # removes only networks with no containers\n' + return 0 +} + +_drh_report_stopped() { + local epoch=0 age=0 owned=0 other=0 shown=0 + local ttl_seconds=$((DRH_DEFAULT_TTL * 3600)) + local ids inspected cid name finished owner now + inspected="" + now=$(date +%s) + printf '\nContainers stopped longer than %sh:\n' "$DRH_DEFAULT_TTL" + ids=$(_drh_docker ps -a -q --no-trunc --filter status=exited) || ids="" + if [[ -n "$ids" ]]; then + # shellcheck disable=SC2086 # ids are docker container IDs (no spaces) + inspected=$(_drh_docker inspect --format \ + "{{.Id}}${DRH_US}{{.Name}}${DRH_US}{{.State.FinishedAt}}${DRH_US}{{index .Config.Labels \"${DRH_PREFIX}.owner\"}}" \ + $ids) || inspected="" + fi + while IFS="$DRH_US" read -r cid name finished owner; do + [[ -n "$cid" ]] || continue + epoch=$(_drh_iso_to_epoch "$finished") || continue + age=$((now - epoch)) + [[ "$age" -gt "$ttl_seconds" ]] || continue + if [[ "$owner" == "1" ]]; then owned=$((owned + 1)); else other=$((other + 1)); fi + if [[ "$shown" -lt 20 ]]; then + printf ' %s stopped_days=%s labelled=%s\n' "${name#/}" "$((age / 86400))" "$([[ "$owner" == "1" ]] && printf yes || printf no)" + shown=$((shown + 1)) + fi + done <<<"$inspected" + printf ' total=%s labelled=%s unlabelled=%s\n' "$((owned + other))" "$owned" "$other" + printf ' reclaim labelled: docker-resource-helper.sh reap --apply\n' + printf ' reclaim unlabelled (after confirming none are needed): docker container prune --filter "until=%sh"\n' "$DRH_DEFAULT_TTL" + return 0 +} + +cmd_report() { + local df dangling opt val + while [[ $# -gt 0 ]]; do + local opt="$1" val="${2:-}" + case "$opt" in + --max-seconds) + _drh_set_max_seconds "$val" || return 1 + shift + ;; + *) + printf 'Unknown option: %s\n' "$opt" >&2 + return 1 + ;; + esac + shift + done + DRH_START=$SECONDS + _drh_docker_ready || return 0 + printf '%s report (read-only; nothing is deleted)\n' "$DRH_TAG" + printf '\nDisk usage (docker system df):\n' + if df=$(_drh_docker system df --format "{{.Type}}${DRH_US}{{.TotalCount}}${DRH_US}{{.Active}}${DRH_US}{{.Size}}${DRH_US}{{.Reclaimable}}"); then + printf '%s\n' "$df" | awk -F"$DRH_US" 'NF >= 5 { printf " %-14s total=%s active=%s size=%s reclaimable=%s\n", $1, $2, $3, $4, $5 }' + else + printf ' unavailable (docker system df failed or exceeded %ss; set AIDEVOPS_DOCKER_CMD_TIMEOUT_SECONDS higher)\n' "$DRH_CMD_TIMEOUT" + fi + printf ' reclaim images: docker image prune (add -a for all unused images)\n' + printf ' reclaim build cache: docker builder prune --filter "until=%sh"\n' "$DRH_DEFAULT_TTL" + dangling=$(_drh_docker volume ls -q --filter dangling=true) || dangling="" + printf '\nUnused volumes (no container references them): %s\n' "$(_drh_count_lines "$dangling")" + printf ' inspect first: docker volume ls --filter dangling=true\n' + printf ' reclaim: docker volume prune (anonymous only; -a includes named volumes and deletes their data)\n' + _drh_report_ownership + _drh_report_stopped + _drh_report_networks + [[ "$DRH_BUDGET_EXHAUSTED" -eq 0 ]] || printf '\n(report budget of %ss exhausted; results are partial)\n' "$DRH_MAX_SECONDS" + printf '\nAddress-pool exhaustion and cleanup guide: tools/containers/disposable-resources.md\n' + return 0 +} + +usage() { + sed -n '8,16p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + return 0 +} + +main() { + local command="${1:-help}" + [[ $# -eq 0 ]] || shift + case "$command" in + labels) cmd_labels "$@" ;; + compose-labels) cmd_compose_labels "$@" ;; + inventory) cmd_inventory "$@" ;; + teardown) cmd_teardown "$@" ;; + reap) cmd_reap "$@" ;; + report) cmd_report "$@" ;; + help | --help | -h) usage ;; + *) + printf 'Unknown command: %s\n' "$command" >&2 + usage >&2 + return 1 + ;; + esac + return $? +} + +main "$@" diff --git a/.agents/scripts/pulse-cleanup.sh b/.agents/scripts/pulse-cleanup.sh index 2868f124fa..96e9c8fe85 100644 --- a/.agents/scripts/pulse-cleanup.sh +++ b/.agents/scripts/pulse-cleanup.sh @@ -1172,9 +1172,29 @@ cleanup_stale_opencode() { # stalled-worker, zombie-reap and ledger stages that follow. Bound the # per-cycle budget so the backlog drains incrementally across cycles. local _scratch_budget="${PULSE_SCRATCH_CLEANUP_MAX_SECONDS:-60}" + local _scratch_rc=0 [[ "$_scratch_budget" =~ ^[0-9]+$ ]] || _scratch_budget=60 if [[ -x "${_PULSE_CLEANUP_SCRIPT_DIR}/system-cleanup.sh" ]]; then - "${_PULSE_CLEANUP_SCRIPT_DIR}/system-cleanup.sh" --force --max-seconds "$_scratch_budget" >>"${LOGFILE:-/dev/null}" 2>&1 || return 1 + "${_PULSE_CLEANUP_SCRIPT_DIR}/system-cleanup.sh" --force --max-seconds "$_scratch_budget" >>"${LOGFILE:-/dev/null}" 2>&1 || _scratch_rc=1 + fi + _pulse_reap_docker_resources || true + return "$_scratch_rc" +} + +# GH#34285: reap aidevops-labelled (sh.aidevops.owner=1) Docker test stacks +# whose worktree is gone and issue/PR closed, or whose TTL expired. Report-only +# until AIDEVOPS_DOCKER_REAP=1. Unlabelled resources are never removed. The +# helper exits 0 within a short timeout when Docker is absent or stopped. +_pulse_reap_docker_resources() { + local _helper="${_PULSE_CLEANUP_SCRIPT_DIR}/docker-resource-helper.sh" + local _budget="${PULSE_DOCKER_CLEANUP_MAX_SECONDS:-30}" + [[ -x "$_helper" ]] || return 0 + command -v docker >/dev/null 2>&1 || return 0 + [[ "$_budget" =~ ^[0-9]+$ ]] || _budget=30 + if [[ "${AIDEVOPS_DOCKER_REAP:-0}" == "1" ]]; then + "$_helper" reap --apply --max-seconds "$_budget" >>"${LOGFILE:-/dev/null}" 2>&1 || true + else + "$_helper" reap --max-seconds "$_budget" >>"${LOGFILE:-/dev/null}" 2>&1 || true fi return 0 } diff --git a/.agents/scripts/tests/test-docker-resource-helper.sh b/.agents/scripts/tests/test-docker-resource-helper.sh new file mode 100644 index 0000000000..ef47e14045 --- /dev/null +++ b/.agents/scripts/tests/test-docker-resource-helper.sh @@ -0,0 +1,473 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2025-2026 Marcus Quinn +# +# test-docker-resource-helper.sh — GH#34285 regression coverage for +# docker-resource-helper.sh using a PATH-stubbed `docker` and `gh` (no real +# daemon). Covers owner labels, report-only reaping, the ownership rule and +# its negative cases (unlabelled, worktree present, ref open, ref lookup +# failure, running past TTL, fresh resources, shared networks), post-removal +# teardown, the pulse step and daemon-down/hung exits. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" || exit 1 +AGENTS_SCRIPTS_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" || exit 1 +HELPER="${AGENTS_SCRIPTS_DIR}/docker-resource-helper.sh" + +TEST_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/docker-resource-test.XXXXXX")" +STATE="${TEST_ROOT}/state" +STUB_BIN="${TEST_ROOT}/bin" +TESTS_RUN=0 +TESTS_FAILED=0 + +cleanup() { + rm -rf "$TEST_ROOT" + return 0 +} +trap cleanup EXIT + +print_result() { + local name="$1" status="$2" detail="${3:-}" + TESTS_RUN=$((TESTS_RUN + 1)) + if [[ "$status" -eq 0 ]]; then + printf 'PASS %s\n' "$name" + else + printf 'FAIL %s\n' "$name" + [[ -z "$detail" ]] || printf '%s\n' "$detail" | sed 's/^/ /' + TESTS_FAILED=$((TESTS_FAILED + 1)) + fi + return 0 +} + +write_stubs() { + mkdir -p "$STUB_BIN" + # Minimal docker emulator over pipe-separated fixture files: + # containers: id|name|owner|repo|ref|worktree|created|ttl|state|networks|volumes|finished + # networks: id|name|owner|repo|ref|worktree|created|ttl + # volumes: name|owner|repo|ref|worktree|created|ttl + cat >"${STUB_BIN}/docker" <<'STUB' +#!/usr/bin/env bash +set -u +S="${DOCKER_STUB_STATE:?}" +US=$'\x1f' +printf '%s\n' "$*" >>"${S}/calls.log" +sub="${1:-}" +[[ $# -eq 0 ]] || shift +case "${DOCKER_STUB_MODE:-up}" in +down) printf 'Cannot connect to the Docker daemon\n' >&2; exit 1 ;; +hang) sleep 10; exit 1 ;; +esac + +join_us() { + local IFS="$US" + printf '%s\n' "$*" + return 0 +} + +# Parse ls/ps style flags into globals. +parse_flags() { + ALL=0; QUIET=0; FILTERS="" + while [[ $# -gt 0 ]]; do + case "$1" in + -a | --all) ALL=1 ;; + -q | --quiet) QUIET=1 ;; + --no-trunc) ;; + --filter | -f) FILTERS="${FILTERS}${2}"$'\n'; shift ;; + --format) shift ;; + esac + shift + done + return 0 +} + +container_matches() { + local owner="$1" state="$2" networks="$3" volumes="$4" id="$5" f + [[ "$ALL" -eq 1 || "$state" == running ]] || return 1 + while IFS= read -r f; do + [[ -n "$f" ]] || continue + case "$f" in + label=sh.aidevops.owner=1) [[ "$owner" == 1 ]] || return 1 ;; + network=*) [[ ",${networks}," == *",${f#network=},"* ]] || return 1 ;; + volume=*) [[ ",${volumes}," == *",${f#volume=},"* ]] || return 1 ;; + status=*) [[ "$state" == "${f#status=}" ]] || return 1 ;; + id=*) [[ "$id" == "${f#id=}" ]] || return 1 ;; + esac + done <<<"$FILTERS" + return 0 +} + +do_ps() { + parse_flags "$@" + local id name owner repo ref wt created ttl state nets vols fin + while IFS='|' read -r id name owner repo ref wt created ttl state nets vols fin; do + [[ -n "$id" ]] || continue + container_matches "$owner" "$state" "$nets" "$vols" "$id" || continue + if [[ "$QUIET" -eq 1 ]]; then printf '%s\n' "$id"; else join_us "$id" "$name" "$owner" "$repo" "$ref" "$wt" "$created" "$ttl" "$state"; fi + done <"${S}/containers" + return 0 +} + +referenced() { + local kind="$1" key="$2" id name owner repo ref wt created ttl state nets vols fin + while IFS='|' read -r id name owner repo ref wt created ttl state nets vols fin; do + if [[ "$kind" == network && ",${nets}," == *",${key},"* ]]; then return 0; fi + if [[ "$kind" == volume && ",${vols}," == *",${key},"* ]]; then return 0; fi + done <"${S}/containers" + return 1 +} + +delete_row() { + local file="$1" key="$2" + awk -F'|' -v k="$key" '$1 != k' "$file" >"${file}.new" || return 1 + mv "${file}.new" "$file" || return 1 + return 0 +} + +has_row() { + local file="$1" key="$2" + awk -F'|' -v k="$key" '$1 == k { found = 1 } END { exit found ? 0 : 1 }' "$file" || return 1 + return 0 +} + +do_network() { + local action="$1" id name owner repo ref wt created ttl f + shift + case "$action" in + ls) + parse_flags "$@" + while IFS='|' read -r id name owner repo ref wt created ttl; do + [[ -n "$id" ]] || continue + if [[ "$FILTERS" == *"label=sh.aidevops.owner=1"* && "$owner" != 1 ]]; then continue; fi + if [[ "$QUIET" -eq 1 ]]; then printf '%s\n' "$id"; else join_us "$id" "$name" "$owner" "$repo" "$ref" "$wt" "$created" "$ttl"; fi + done <"${S}/networks" + ;; + rm) + f="$1" + has_row "${S}/networks" "$f" || { printf 'Error response from daemon: network %s not found\n' "$f" >&2; exit 1; } + referenced network "$f" && { printf 'Error response from daemon: network %s has active endpoints\n' "$f" >&2; exit 1; } + delete_row "${S}/networks" "$f" + printf 'network rm %s\n' "$f" >>"${S}/actions.log" + ;; + esac + return 0 +} + +do_volume() { + local action="$1" name owner repo ref wt created ttl f + shift + case "$action" in + ls) + parse_flags "$@" + while IFS='|' read -r name owner repo ref wt created ttl; do + [[ -n "$name" ]] || continue + if [[ "$FILTERS" == *"label=sh.aidevops.owner=1"* && "$owner" != 1 ]]; then continue; fi + if [[ "$FILTERS" == *"dangling=true"* ]] && referenced volume "$name"; then continue; fi + if [[ "$QUIET" -eq 1 ]]; then printf '%s\n' "$name"; else join_us "$name" "$name" "$owner" "$repo" "$ref" "$wt" "$created" "$ttl"; fi + done <"${S}/volumes" + ;; + rm) + f="$1" + has_row "${S}/volumes" "$f" || { printf 'Error: No such volume: %s\n' "$f" >&2; exit 1; } + referenced volume "$f" && { printf 'Error response from daemon: volume is in use\n' >&2; exit 1; } + delete_row "${S}/volumes" "$f" + printf 'volume rm %s\n' "$f" >>"${S}/actions.log" + ;; + esac + return 0 +} + +do_inspect() { + local format="" id row + [[ "${1:-}" == --format ]] && { format="$2"; shift 2; } + for id in "$@"; do + row=$(awk -F'|' -v k="$id" '$1 == k' "${S}/containers") + [[ -n "$row" ]] || { printf 'Error: No such object: %s\n' "$id" >&2; exit 1; } + IFS='|' read -r cid name owner repo ref wt created ttl state nets vols fin <<<"$row" + if [[ "$format" == *FinishedAt* ]]; then join_us "$cid" "/${name}" "$fin" "$owner"; else printf '%s\n' "$state"; fi + done + return 0 +} + +case "$sub" in +info) printf '27.0.0\n' ;; +ps) do_ps "$@" ;; +rm) + [[ "${1:-}" == -f ]] && shift + has_row "${S}/containers" "$1" || { printf 'Error: No such container: %s\n' "$1" >&2; exit 1; } + delete_row "${S}/containers" "$1" + printf 'rm %s\n' "$1" >>"${S}/actions.log" + ;; +inspect) do_inspect "$@" ;; +network) do_network "$@" ;; +volume) do_volume "$@" ;; +system) join_us Images 3 1 1.2GB "600MB (50%)"; join_us "Local Volumes" 4 1 2GB "1.5GB (75%)" ;; +*) exit 1 ;; +esac +exit 0 +STUB + cat >"${STUB_BIN}/gh" <<'STUB' +#!/usr/bin/env bash +S="${DOCKER_STUB_STATE:?}" +printf '%s\n' "$*" >>"${S}/gh.log" +path="${2:-}" +key="${path#repos/}" +key="${key/\/issues\//#}" +state=$(awk -F'|' -v k="$key" '$1 == k { print $2 }' "${S}/refs") +[[ -n "$state" && "$state" != fail ]] || exit 1 +printf '%s\n' "$state" +STUB + chmod +x "${STUB_BIN}/docker" "${STUB_BIN}/gh" + return 0 +} + +# Fixture: see the expectations in test_reap_* below. +write_fixture() { + local now old ancient fresh finished gone present td + now=$(date +%s) + old=$((now - 4 * 3600)) + ancient=$((now - 100 * 3600)) + fresh=$((now - 600)) + finished="2020-01-01T00:00:00.123456789Z" + gone="${TEST_ROOT}/gone-wt" + present="${TEST_ROOT}/present-wt" + td="${TEST_ROOT}/teardown-wt" + rm -rf "$STATE" "$gone" "$td" + mkdir -p "$STATE" "$present" + : >"${STATE}/calls.log" + : >"${STATE}/actions.log" + : >"${STATE}/gh.log" + cat >"${STATE}/refs" <"${STATE}/containers" <"${STATE}/networks" <"${STATE}/volumes" </dev/null 2>&1 && rc=1 + print_result "labels prints the six owner labels and a compose snippet; invalid ref rejected" "$rc" "$out" + return 0 +} + +test_inventory() { + local out rc=0 + write_fixture + out=$(run_helper inventory) || rc=1 + [[ "$out" == *"container"$'\t'"c1"$'\t'"closed-gone"* ]] || rc=1 + [[ "$out" == *"network"$'\t'"n4"* && "$out" == *"volume"$'\t'"v1"* ]] || rc=1 + [[ "$out" != *$'\tc2\t'* && "$out" != *$'\tc10\t'* ]] || rc=1 + [[ "$out" != *$'\tnU\t'* && "$out" != *$'\tvU\t'* ]] || rc=1 + print_result "inventory lists labelled containers, networks and volumes only" "$rc" "$out" + return 0 +} + +test_reap_report_only() { + local out rc=0 + write_fixture + out=$(run_helper reap) || rc=1 + [[ ! -s "${STATE}/actions.log" ]] || rc=1 + [[ "$out" == *"would-remove container closed-gone"* && "$out" == *"would-remove container ttl-stopped"* ]] || rc=1 + [[ "$out" == *"would-remove network net-closed-gone"* && "$out" == *"would-remove volume v1"* ]] || rc=1 + [[ "$out" == *"kept network net-shared-unlabelled"*"attached_containers=1"* ]] || rc=1 + [[ "$out" == *"reap mode=dry-run"* ]] || rc=1 + [[ "$(grep -c . "${STATE}/gh.log")" -eq 3 ]] || rc=1 + print_result "reap without --apply removes nothing, lists candidates, one gh lookup per ref" "$rc" "$out" + return 0 +} + +test_reap_apply() { + local out rc=0 id + write_fixture + out=$(run_helper reap --apply --verbose) || rc=1 + for id in c1 c7; do has_id containers "$id" && rc=1; done + has_id networks n1 && rc=1 + has_id volumes v1 && rc=1 + for id in c2 c3 c4 c5 c6 c8 c9 c10; do has_id containers "$id" || rc=1; done + for id in n3 n4 n5 nU; do has_id networks "$id" || rc=1; done + for id in v4 vU; do has_id volumes "$id" || rc=1; done + [[ "$out" == *"reason=worktree-present"* && "$out" == *"reason=ref-open"* ]] || rc=1 + [[ "$out" == *"reason=ref-unknown"* && "$out" == *"reason=ttl-expired-but-running"* ]] || rc=1 + [[ "$out" == *"reason=created-within-30m"* ]] || rc=1 + # Containers are removed before networks and volumes. + [[ "$(head -1 "${STATE}/actions.log")" == "rm c1" ]] || rc=1 + print_result "reap --apply removes only closed+gone or stopped TTL-expired labelled resources" "$rc" "$out" + return 0 +} + +test_reap_budget() { + local out rc=0 + write_fixture + out=$(run_helper reap --apply --max-seconds abc 2>&1) && rc=1 + out=$(run_helper reap --apply --max-seconds 0) || rc=1 + [[ "$out" == *"budget_s=0 budget_exhausted=0"* ]] || rc=1 + print_result "reap validates --max-seconds and reports its budget" "$rc" "$out" + return 0 +} + +test_teardown() { + local out rc=0 + write_fixture + out=$(run_helper teardown --worktree "${TEST_ROOT}/present-wt" 2>&1) && rc=1 + [[ ! -s "${STATE}/actions.log" ]] || rc=1 + out=$(run_helper teardown --worktree "${TEST_ROOT}/teardown-wt/") || rc=1 + has_id containers c9 && rc=1 + has_id networks n4 && rc=1 + has_id volumes v4 && rc=1 + has_id networks n5 || rc=1 + has_id containers c10 || rc=1 + has_id containers c1 || rc=1 + [[ "$out" == *"teardown mode=apply"*"matched=4 removed=3 kept=1"* ]] || rc=1 + out=$(run_helper teardown --worktree "${TEST_ROOT}/teardown-wt") || rc=1 + [[ -z "$out" || "$out" == *"matched=1"* ]] || rc=1 + print_result "teardown removes the removed worktree's labelled stack, keeps shared and existing-worktree resources" "$rc" "$out" + return 0 +} + +test_worktree_hook() { + local rc=0 + write_fixture + ( + export PATH="${STUB_BIN}:${PATH}" DOCKER_STUB_STATE="$STATE" + SCRIPT_DIR="$AGENTS_SCRIPTS_DIR" + # shellcheck source=../shared-constants.sh + source "${AGENTS_SCRIPTS_DIR}/shared-constants.sh" + # shellcheck source=../worktree-helper-integration.sh + source "${AGENTS_SCRIPTS_DIR}/worktree-helper-integration.sh" + AIDEVOPS_DOCKER_TEARDOWN=0 docker_resource_auto_teardown "${TEST_ROOT}/teardown-wt" + has_id containers c9 || exit 1 + docker_resource_auto_teardown "${TEST_ROOT}/teardown-wt" 2>/dev/null + ) >"${TEST_ROOT}/hook.out" || rc=1 + [[ ! -s "${TEST_ROOT}/hook.out" ]] || rc=1 + has_id containers c9 && rc=1 + has_id volumes v4 && rc=1 + print_result "worktree removal hook tears down labelled resources with empty stdout; opt-out honoured" "$rc" + return 0 +} + +test_pulse_step() { + local rc=0 dir="${TEST_ROOT}/pulse-scripts" + write_fixture + mkdir -p "$dir" + printf '#!/usr/bin/env bash\nexit 0\n' >"${dir}/system-cleanup.sh" + printf '#!/usr/bin/env bash\nprintf "%%s\\n" "$*" >>"%s/pulse-args.log"\nexec "%s" "$@"\n' "$STATE" "$HELPER" >"${dir}/docker-resource-helper.sh" + chmod +x "${dir}/system-cleanup.sh" "${dir}/docker-resource-helper.sh" + ( + export PATH="${STUB_BIN}:${PATH}" DOCKER_STUB_STATE="$STATE" HOME="${TEST_ROOT}/home" LOGFILE="${TEST_ROOT}/pulse.log" + mkdir -p "$HOME" + unset AIDEVOPS_DOCKER_REAP PULSE_DOCKER_CLEANUP_MAX_SECONDS + # shellcheck source=../shared-constants.sh + source "${AGENTS_SCRIPTS_DIR}/shared-constants.sh" + # shellcheck source=../worker-lifecycle-common.sh + source "${AGENTS_SCRIPTS_DIR}/worker-lifecycle-common.sh" + # shellcheck source=../pulse-cleanup.sh + source "${AGENTS_SCRIPTS_DIR}/pulse-cleanup.sh" + _PULSE_CLEANUP_SCRIPT_DIR="$dir" + cleanup_stale_opencode || exit 1 + [[ ! -s "${STATE}/actions.log" ]] || exit 1 + AIDEVOPS_DOCKER_REAP=1 cleanup_stale_opencode || exit 1 + ) || rc=1 + [[ "$(sed -n 1p "${STATE}/pulse-args.log")" == "reap --max-seconds 30" ]] || rc=1 + [[ "$(sed -n 2p "${STATE}/pulse-args.log")" == "reap --apply --max-seconds 30" ]] || rc=1 + has_id containers c1 && rc=1 + has_id containers c2 || rc=1 + grep -q 'reap mode=dry-run' "${TEST_ROOT}/pulse.log" || rc=1 + print_result "pulse step is report-only unless AIDEVOPS_DOCKER_REAP=1 and never touches unlabelled" "$rc" \ + "$(cat "${STATE}/pulse-args.log" 2>/dev/null)" + return 0 +} + +test_report() { + local out rc=0 + write_fixture + out=$(run_helper report) || rc=1 + [[ ! -s "${STATE}/actions.log" ]] || rc=1 + [[ "$out" == *"containers: labelled=8 unlabelled=2"* ]] || rc=1 + [[ "$out" == *"networks (custom): labelled=4 unlabelled=1"* ]] || rc=1 + [[ "$out" == *"volumes: labelled=2 unlabelled=1"* ]] || rc=1 + [[ "$out" == *"Unused volumes (no container references them): 1"* ]] || rc=1 + [[ "$out" == *"net-unlabelled"* && "$out" == *"docker network prune"* ]] || rc=1 + [[ "$out" == *"unlabelled-old stopped_days="* ]] || rc=1 + print_result "report counts labelled/unlabelled pressure and deletes nothing" "$rc" "$out" + return 0 +} + +test_daemon_unavailable() { + local rc=0 out start elapsed + write_fixture + out=$(DOCKER_STUB_MODE=down run_helper reap --apply) || rc=1 + [[ "$out" == *"docker daemon not reachable"* ]] || rc=1 + DOCKER_STUB_MODE=down run_helper report >/dev/null || rc=1 + DOCKER_STUB_MODE=down run_helper teardown --worktree "${TEST_ROOT}/teardown-wt" >/dev/null || rc=1 + start=$SECONDS + out=$(DOCKER_STUB_MODE=hang AIDEVOPS_DOCKER_TIMEOUT_SECONDS=1 run_helper reap --apply) || rc=1 + elapsed=$((SECONDS - start)) + [[ "$elapsed" -le 4 ]] || rc=1 + [[ ! -s "${STATE}/actions.log" ]] || rc=1 + print_result "daemon down or hung: entry points exit 0 within the short probe timeout" "$rc" "elapsed=${elapsed}s ${out}" + return 0 +} + +main() { + write_stubs + test_labels + test_inventory + test_reap_report_only + test_reap_apply + test_reap_budget + test_teardown + test_worktree_hook + test_pulse_step + test_report + test_daemon_unavailable + printf '\n%d/%d tests passed\n' "$((TESTS_RUN - TESTS_FAILED))" "$TESTS_RUN" + [[ "$TESTS_FAILED" -eq 0 ]] || return 1 + return 0 +} + +main "$@" diff --git a/.agents/scripts/worktree-clean-lib.sh b/.agents/scripts/worktree-clean-lib.sh index d58b56e7ea..ec65fd900c 100644 --- a/.agents/scripts/worktree-clean-lib.sh +++ b/.agents/scripts/worktree-clean-lib.sh @@ -1588,6 +1588,10 @@ _clean_remove_classified_worktree() { return 1 fi _clean_release_removal_lease "$worktree_path" || true + # GH#34285: remove aidevops-labelled Docker resources for this worktree. + if declare -F docker_resource_auto_teardown >/dev/null 2>&1; then + docker_resource_auto_teardown "$worktree_path" + fi if [[ "$preserve_branch" != "$_WT_CLEAN_BOOL_TRUE" ]]; then localdev_auto_branch_rm "$worktree_branch" if [[ -f "${SCRIPT_DIR}/local-branch-cleanup-helper.sh" ]]; then diff --git a/.agents/scripts/worktree-helper-cmds.sh b/.agents/scripts/worktree-helper-cmds.sh index 02e00681aa..e656c2f3b7 100644 --- a/.agents/scripts/worktree-helper-cmds.sh +++ b/.agents/scripts/worktree-helper-cmds.sh @@ -336,6 +336,10 @@ _remove_finalize_post_removal() { local cleanup_receipt="$2" local path_to_remove="$3" + # GH#34285: remove aidevops-labelled Docker resources for this worktree. + if declare -F docker_resource_auto_teardown >/dev/null 2>&1; then + docker_resource_auto_teardown "$path_to_remove" + fi if [[ -n "$removed_branch" ]]; then localdev_auto_branch_rm "$removed_branch" preview_proxy_auto_free "$removed_branch" diff --git a/.agents/scripts/worktree-helper-integration.sh b/.agents/scripts/worktree-helper-integration.sh index 29adcc2baf..2b5bb3351c 100644 --- a/.agents/scripts/worktree-helper-integration.sh +++ b/.agents/scripts/worktree-helper-integration.sh @@ -14,6 +14,9 @@ # via local proxy. On worktree add, allocate a port + register a proxy # route. On remove, free the port + deregister. Both best-effort, non-fatal. # +# Docker teardown (GH#34285): on removal, remove the aidevops-labelled Docker +# resources recorded for that worktree via docker-resource-helper.sh. +# # Usage: source "${SCRIPT_DIR}/worktree-helper-integration.sh" # # Dependencies: @@ -197,3 +200,34 @@ preview_proxy_auto_free() { "$PREVIEW_PROXY_HELPER" free "$repo_slug" "$branch" 2>/dev/null || true return 0 } + +# --- Docker disposable resources (GH#34285) --- + +# Tear down Docker containers, networks and volumes labelled +# sh.aidevops.worktree= once that linked worktree has been removed +# (post-merge cleanup, `worktree-helper.sh clean`, manual remove). Only +# resources carrying sh.aidevops.owner=1 are touched. Best-effort and +# non-fatal: skipped without docker or with AIDEVOPS_DOCKER_TEARDOWN=0, and +# the helper gives up after a short daemon probe. Output goes to stderr so +# `clean --auto` keeps an empty stdout. +# Args: $1=removed worktree path +docker_resource_auto_teardown() { + local worktree_path="${1:-}" + local helper="${SCRIPT_DIR}/docker-resource-helper.sh" + [[ -n "$worktree_path" ]] || return 0 + [[ "${AIDEVOPS_DOCKER_TEARDOWN:-1}" != "0" ]] || return 0 + command -v docker >/dev/null 2>&1 || return 0 + [[ -x "$helper" ]] || return 0 + # Probe the daemon once per process so a batch `clean` with Docker stopped + # pays the short timeout once, not once per removed worktree. + if [[ -z "${_WT_DOCKER_DAEMON_READY:-}" ]]; then + _WT_DOCKER_DAEMON_READY=1 + if declare -F timeout_sec >/dev/null 2>&1 && + ! timeout_sec "${AIDEVOPS_DOCKER_TIMEOUT_SECONDS:-5}" docker info --format '{{.ServerVersion}}' >/dev/null 2>&1; then + _WT_DOCKER_DAEMON_READY=0 + fi + fi + [[ "$_WT_DOCKER_DAEMON_READY" == "1" ]] || return 0 + "$helper" teardown --worktree "$worktree_path" 1>&2 || true + return 0 +} diff --git a/.agents/scripts/wp-plugin-release-helper.sh b/.agents/scripts/wp-plugin-release-helper.sh index 0dad754e1f..f949feed37 100755 --- a/.agents/scripts/wp-plugin-release-helper.sh +++ b/.agents/scripts/wp-plugin-release-helper.sh @@ -928,6 +928,23 @@ WPRH_PC_VOLUME="" WPRH_PC_DB_CONTAINER="" WPRH_PC_DB_PASS="" WPRH_PC_WP_ADMIN_PASS="" +# aidevops owner labels (GH#34285) so a stack orphaned by a killed run is +# found by docker-resource-helper.sh reap/teardown; wprh_pc_cleanup remains +# the primary cleanup. +WPRH_PC_LABELS=() + +# Fill WPRH_PC_LABELS with --label=sh.aidevops.*=... arguments. Labels are +# optional: a missing or failing helper leaves the array empty. +wprh_pc_load_labels() { + local helper="${WPRH_DIR}/docker-resource-helper.sh" line="" output="" + WPRH_PC_LABELS=() + [[ -x "$helper" ]] || return 0 + output="$("$helper" labels --ttl-hours 24 2>/dev/null)" || return 0 + while IFS= read -r line; do + [[ "$line" == --label=* ]] && WPRH_PC_LABELS+=("$line") + done <<<"$output" + return 0 +} wprh_pc_cleanup() { [[ -n "$WPRH_PC_DB_CONTAINER" ]] && docker rm -f "$WPRH_PC_DB_CONTAINER" >/dev/null 2>&1 || true @@ -1003,12 +1020,13 @@ wprh_pc_create_stack() { WPRH_PC_WP_ADMIN_PASS="$(aidevops_generate_execution_id wprhadmin)" log_info "creating disposable Docker network/volume/db for plugin-check (${run_id})" - docker network create "$WPRH_PC_NETWORK" >/dev/null - docker volume create "$WPRH_PC_VOLUME" >/dev/null - docker run --rm --user root -v "${WPRH_PC_VOLUME}:/var/www/html" wordpress:cli-php8.3 \ + wprh_pc_load_labels + docker network create ${WPRH_PC_LABELS[@]+"${WPRH_PC_LABELS[@]}"} "$WPRH_PC_NETWORK" >/dev/null + docker volume create ${WPRH_PC_LABELS[@]+"${WPRH_PC_LABELS[@]}"} "$WPRH_PC_VOLUME" >/dev/null + docker run --rm ${WPRH_PC_LABELS[@]+"${WPRH_PC_LABELS[@]}"} --user root -v "${WPRH_PC_VOLUME}:/var/www/html" wordpress:cli-php8.3 \ chown -R 33:33 /var/www/html >/dev/null - docker run -d --name "$WPRH_PC_DB_CONTAINER" --network "$WPRH_PC_NETWORK" \ + docker run -d ${WPRH_PC_LABELS[@]+"${WPRH_PC_LABELS[@]}"} --name "$WPRH_PC_DB_CONTAINER" --network "$WPRH_PC_NETWORK" \ -e "$(wprh_pc_kv MARIADB_ROOT_PASSWORD "$WPRH_PC_DB_PASS")" -e MARIADB_DATABASE=wordpress \ mariadb:10.6 >/dev/null @@ -1029,7 +1047,7 @@ wprh_pc_create_stack() { # array from a function). wprh_pc_wp_cli_args() { printf '%s\n' \ - --rm --network "$WPRH_PC_NETWORK" -v "${WPRH_PC_VOLUME}:/var/www/html" \ + --rm ${WPRH_PC_LABELS[@]+"${WPRH_PC_LABELS[@]}"} --network "$WPRH_PC_NETWORK" -v "${WPRH_PC_VOLUME}:/var/www/html" \ --user 33:33 -e "$(wprh_pc_kv WORDPRESS_DB_HOST "$WPRH_PC_DB_CONTAINER")" \ -e WORDPRESS_DB_USER=root -e "$(wprh_pc_kv WORDPRESS_DB_PASSWORD "$WPRH_PC_DB_PASS")" \ -e WORDPRESS_DB_NAME=wordpress \ @@ -1064,7 +1082,7 @@ wprh_pc_check_zip() { local zip_dir zip_name zip_dir="$(dirname "$zip")" zip_name="$(basename "$zip")" - docker run --rm --network "$WPRH_PC_NETWORK" \ + docker run --rm ${WPRH_PC_LABELS[@]+"${WPRH_PC_LABELS[@]}"} --network "$WPRH_PC_NETWORK" \ -v "${WPRH_PC_VOLUME}:/var/www/html" -v "${zip_dir}:/zips:ro" \ --user 33:33 -e "$(wprh_pc_kv WORDPRESS_DB_HOST "$WPRH_PC_DB_CONTAINER")" \ -e WORDPRESS_DB_USER=root -e "$(wprh_pc_kv WORDPRESS_DB_PASSWORD "$WPRH_PC_DB_PASS")" \ diff --git a/.agents/tools/containers/disposable-resources.md b/.agents/tools/containers/disposable-resources.md new file mode 100644 index 0000000000..14adbe4a31 --- /dev/null +++ b/.agents/tools/containers/disposable-resources.md @@ -0,0 +1,116 @@ +--- +description: Disposable Docker resources - owner labels, teardown, reaping, pressure report and address-pool fixes +mode: subagent +tools: + read: true + write: false + edit: false + bash: true + glob: true + grep: true + webfetch: false + task: false +--- + + + + +# Disposable Docker Resources + +## Quick Reference + +- **Helper**: `~/.aidevops/agents/scripts/docker-resource-helper.sh` (`labels`, `compose-labels`, `inventory`, `teardown`, `reap`, `report`) +- **Rule**: every container, network and volume a session starts for testing, plugin checks or review sites carries the six `sh.aidevops.*` owner labels +- **Teardown**: removing a linked worktree (`worktree-helper.sh remove`, post-merge cleanup, `worktree-helper.sh clean`) removes the labelled resources for that worktree +- **Reaping**: the pulse runs `reap`, which is report-only until `AIDEVOPS_DOCKER_REAP=1` +- **Never removed automatically**: resources without `sh.aidevops.owner=1`. They are only counted by `report`. Remove them only with the user's go-ahead. + +## Owner labels + +| Label | Value | +|-------|-------| +| `sh.aidevops.owner` | `1` | +| `sh.aidevops.repo` | `owner/repo` (default: `origin` remote) | +| `sh.aidevops.ref` | `GH#N` or `PR#N` (default: `WORKER_ISSUE_NUMBER`, else `ghN` in the branch name) | +| `sh.aidevops.worktree` | absolute worktree path (default: `git rev-parse --show-toplevel`) | +| `sh.aidevops.created` | epoch seconds at label time | +| `sh.aidevops.ttl-hours` | hours before the TTL rule applies (default `AIDEVOPS_DOCKER_TTL_HOURS`, else 72) | + +## Starting a labelled stack + +`labels` prints one `--label=key=value` per line. Read it into an array so worktree paths with spaces survive: + +```bash +helper=~/.aidevops/agents/scripts/docker-resource-helper.sh +labels=() +while IFS= read -r l; do labels+=("$l"); done < <("$helper" labels --ref GH#123 --ttl-hours 24) + +docker network create "${labels[@]}" site-gh123 +docker volume create "${labels[@]}" site-gh123-db +docker run -d "${labels[@]}" --name site-gh123-db --network site-gh123 \ + -v site-gh123-db:/var/lib/mysql -e MARIADB_ROOT_PASSWORD=[placeholder] mariadb:11 +"$helper" inventory # the three resources are listed +``` + +For Compose, paste the generated snippet under each service, network and volume (`--indent` matches the nesting): + +```bash +"$helper" compose-labels --ref PR#456 --indent 4 +``` + +`compose-labels` stamps the current time into `created`. Regenerate the snippet for each new stack, and don't commit it to a shared Compose file. + +`wp-plugin-release-helper.sh` labels its plugin-check network, volume and containers this way. It still removes them on exit through `wprh_pc_cleanup`. + +## Teardown, reap and report + +```bash +"$helper" teardown --worktree /path/to/worktree --dry-run # list what would go +"$helper" teardown --worktree /path/to/worktree # remove (worktree must be gone) +"$helper" teardown --worktree /path/to/worktree --force # remove even if it still exists +"$helper" reap # report-only candidates +"$helper" reap --apply --max-seconds 30 --verbose # remove candidates, explain keeps +"$helper" report # counts, sizes and reclaim commands; deletes nothing +``` + +`reap` removes a labelled resource only when all of these hold: + +1. It was created more than 30 minutes ago, so a stack another session is starting is safe. +2. Either: + - its worktree path is gone **and** its issue/PR is closed or merged (one `gh` lookup per ref per run, and a failed lookup means "keep"), or + - its TTL has expired and it is not running or used by a running container. +3. For networks and volumes: no container that is staying is still attached. + +Removal goes containers first, then networks, then volumes. "No such object" errors from a parallel run are ignored, so repeated runs converge. + +## Environment + +| Variable | Default | Effect | +|----------|---------|--------| +| `AIDEVOPS_DOCKER_REAP` | `0` | `1` lets the pulse run `reap --apply`; otherwise the pulse is report-only | +| `AIDEVOPS_DOCKER_TEARDOWN` | `1` | `0` disables teardown on worktree removal | +| `AIDEVOPS_DOCKER_TTL_HOURS` | `72` | default `ttl-hours` label and report threshold | +| `PULSE_DOCKER_CLEANUP_MAX_SECONDS` | `30` | time budget for the pulse reap step | +| `AIDEVOPS_DOCKER_TIMEOUT_SECONDS` | `5` | daemon probe timeout | +| `AIDEVOPS_DOCKER_CMD_TIMEOUT_SECONDS` | `30` | per-command timeout | + +If Docker is missing or the daemon doesn't answer within the probe timeout, every entry point logs one line and exits 0. Pulse and merge cleanup never fail because of Docker. + +## Address-pool exhaustion + +Symptom: `docker network create` (or `docker compose up`) fails with `all predefined address pools have been fully subnetted`. The default pools give about 30 bridge networks, and leftover test stacks use them up. + +1. Run `"$helper" report` and look at the "Custom networks with no containers" section. +2. Remove only those networks: `docker network prune` removes networks with no containers, or use `docker network rm ` for individual names from the report. Networks still attached to a container are kept. +3. Optional: to get about 256 `/24` networks instead of about 30, set `default-address-pools` in the daemon config. Use Docker Desktop/OrbStack settings, or `/etc/docker/daemon.json` on Linux: + + ```json + { "default-address-pools": [ { "base": "10.200.0.0/16", "size": 24 } ] } + ``` + + Restart the daemon afterwards. Choose a `base` that doesn't overlap your LAN or VPN ranges. Existing networks keep their subnets. + +## Related + +- `tools/containers/orbstack.md`: macOS Docker runtime and destructive-op cautions +- `scripts/system-cleanup.sh`: non-Docker workspace cleanup that the same pulse step runs diff --git a/.agents/tools/containers/orbstack.md b/.agents/tools/containers/orbstack.md index 3fcc1ac93f..d65e652f22 100644 --- a/.agents/tools/containers/orbstack.md +++ b/.agents/tools/containers/orbstack.md @@ -57,3 +57,5 @@ docker system df # Disk usage ``` **Destructive ops:** `docker system prune -a` removes all unused containers, images, networks, and build cache. Prefer `docker image prune` or `docker container prune` unless a full reset is intended. `orb reset` is a factory reset — last resort. + +**Leftover test stacks / `all predefined address pools have been fully subnetted`:** see `tools/containers/disposable-resources.md` (owner labels, `docker-resource-helper.sh report`, safe network cleanup, `default-address-pools`).