From f02d4fa62e1b393f3ca26b62d952d8a749157c58 Mon Sep 17 00:00:00 2001 From: WOPR Date: Tue, 17 Mar 2026 14:01:03 -0700 Subject: [PATCH 01/17] feat: add WOPR sidecar for managed hosting Adds /internal/health and /internal/provision endpoints so nemoclaw-platform can provision NemoClaw containers using the same contract as paperclip-platform. On provision: rewrites openclaw.json to route inference through WOPR's gateway instead of NVIDIA's, enabling metered billing via platform-core credit ledger. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/ci.yml | 38 ++++++++++ Dockerfile | 3 + scripts/nemoclaw-start.sh | 6 ++ wopr/package.json | 7 ++ wopr/sidecar.js | 149 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 203 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 wopr/package.json create mode 100644 wopr/sidecar.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..3631a852c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + build-and-push: + runs-on: [self-hosted, Linux, X64] + permissions: + contents: read + packages: write + + steps: + - uses: actions/checkout@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ github.ref == 'refs/heads/main' }} + tags: | + ghcr.io/${{ github.repository }}:latest + ghcr.io/${{ github.repository }}:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Dockerfile b/Dockerfile index 309ba1d8f..1451bd360 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,6 +69,9 @@ ENV NEMOCLAW_MODEL=${NEMOCLAW_MODEL} \ NEMOCLAW_INFERENCE_API=${NEMOCLAW_INFERENCE_API} \ NEMOCLAW_INFERENCE_COMPAT_B64=${NEMOCLAW_INFERENCE_COMPAT_B64} +# WOPR sidecar — provision + health endpoints for nemoclaw-platform +COPY wopr/ /opt/wopr/ + WORKDIR /sandbox USER sandbox diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index 6472459c6..ca46964ac 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -293,6 +293,12 @@ echo "[gateway] openclaw gateway launched as 'gateway' user (pid $GATEWAY_PID)" start_auto_pair print_dashboard_urls +# Start WOPR sidecar — /internal/health + /internal/provision for nemoclaw-platform +if [ -f /opt/wopr/sidecar.js ]; then + nohup node /opt/wopr/sidecar.js > /tmp/wopr-sidecar.log 2>&1 & + echo "[wopr-sidecar] launched (pid $!)" +fi + # Keep container running by waiting on the gateway process. # This script is PID 1 (ENTRYPOINT); if it exits, Docker kills all children. wait "$GATEWAY_PID" diff --git a/wopr/package.json b/wopr/package.json new file mode 100644 index 000000000..e539c40b8 --- /dev/null +++ b/wopr/package.json @@ -0,0 +1,7 @@ +{ + "name": "@wopr-network/nemoclaw-sidecar", + "version": "1.0.0", + "private": true, + "type": "module", + "description": "WOPR provision sidecar for NemoClaw managed hosting" +} diff --git a/wopr/sidecar.js b/wopr/sidecar.js new file mode 100644 index 000000000..f697482b1 --- /dev/null +++ b/wopr/sidecar.js @@ -0,0 +1,149 @@ +#!/usr/bin/env node +// WOPR NemoClaw sidecar — exposes /internal/health and /internal/provision +// so nemoclaw-platform can use the same provision contract as paperclip-platform. +// +// Env vars: +// WOPR_PROVISION_SECRET — shared secret for auth +// WOPR_GATEWAY_URL — WOPR inference gateway base URL (e.g. https://gateway.wopr.bot/v1) +// PORT — sidecar port (default: 3001) + +import http from "node:http"; +import fs from "node:fs"; +import path from "node:path"; +import os from "node:os"; + +const SECRET = process.env.WOPR_PROVISION_SECRET ?? ""; +const GATEWAY_URL = process.env.WOPR_GATEWAY_URL ?? ""; +const PORT = parseInt(process.env.WOPR_SIDECAR_PORT ?? "3001", 10); +const OPENCLAW_CONFIG_PATH = path.join(os.homedir(), ".openclaw", "openclaw.json"); + +function assertSecret(req) { + const auth = req.headers["authorization"] ?? ""; + if (!auth.startsWith("Bearer ")) return false; + return auth.slice("Bearer ".length).trim() === SECRET; +} + +function readJson(filePath) { + try { + return JSON.parse(fs.readFileSync(filePath, "utf8")); + } catch { + return {}; + } +} + +function writeJson(filePath, obj) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify(obj, null, 2), { mode: 0o600 }); +} + +function isGatewayUp() { + try { + const logPath = "/tmp/gateway.log"; + if (!fs.existsSync(logPath)) return false; + const tail = fs.readFileSync(logPath, "utf8").slice(-4096); + // openclaw gateway prints "Listening" or "Gateway running" when ready + return /listening|gateway running|started/i.test(tail); + } catch { + return false; + } +} + +function provision(body) { + const { tenantId, tenantName, gatewayUrl, apiKey, budgetCents } = body; + + if (!tenantId || !tenantName) { + throw new Error("Missing required fields: tenantId, tenantName"); + } + + const effectiveGateway = gatewayUrl || GATEWAY_URL; + if (!effectiveGateway) { + throw new Error("No gateway URL provided and WOPR_GATEWAY_URL not set"); + } + + // Point NemoClaw at WOPR's inference gateway instead of NVIDIA's + const cfg = readJson(OPENCLAW_CONFIG_PATH); + + cfg.agents ??= {}; + cfg.agents.defaults ??= {}; + cfg.agents.defaults.model ??= {}; + cfg.agents.defaults.model.primary = "nvidia/nemotron-3-super-120b-a12b"; + + cfg.models ??= {}; + cfg.models.mode = "merge"; + cfg.models.providers ??= {}; + cfg.models.providers["wopr-gateway"] = { + baseUrl: effectiveGateway, + apiKey: apiKey ?? "wopr-managed", + api: "openai-completions", + models: [ + { + id: "nvidia/nemotron-3-super-120b-a12b", + name: "Nemotron 3 Super 120B (via WOPR)", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 131072, + maxTokens: 4096, + }, + ], + }; + + // Set WOPR gateway as the active provider + cfg.agents.defaults.model.primary = "nvidia/nemotron-3-super-120b-a12b"; + cfg.gateway ??= {}; + cfg.gateway.inferenceProvider = "wopr-gateway"; + + // Store tenant metadata for reference + cfg._wopr = { tenantId, tenantName, budgetCents: budgetCents ?? 0, provisionedAt: new Date().toISOString() }; + + writeJson(OPENCLAW_CONFIG_PATH, cfg); + + // Derive a stable tenantEntityId and slug from tenantId + const tenantSlug = tenantName.toLowerCase().replace(/[^a-z0-9]/g, "-").replace(/-+/g, "-").slice(0, 32); + const tenantEntityId = `e:${tenantId}`; + + return { tenantEntityId, tenantSlug }; +} + +const server = http.createServer((req, res) => { + const url = new URL(req.url, `http://localhost:${PORT}`); + + // Health check — no auth required + if (req.method === "GET" && url.pathname === "/internal/health") { + const up = isGatewayUp(); + res.writeHead(up ? 200 : 503, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: up, gateway: up ? "running" : "starting" })); + return; + } + + // Provision — auth required + if (req.method === "POST" && url.pathname === "/internal/provision") { + if (!assertSecret(req)) { + res.writeHead(401, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: "Unauthorized" })); + return; + } + + let body = ""; + req.on("data", (chunk) => { body += chunk; }); + req.on("end", () => { + try { + const parsed = JSON.parse(body); + const result = provision(parsed); + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: true, ...result })); + } catch (err) { + res.writeHead(400, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: err.message })); + } + }); + return; + } + + res.writeHead(404, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: "Not found" })); +}); + +server.listen(PORT, "0.0.0.0", () => { + console.log(`[wopr-sidecar] listening on :${PORT}`); +}); From 705a5c66549ee2833e8cec423ccc3f229dcccf84 Mon Sep 17 00:00:00 2001 From: T Savo Date: Sun, 22 Mar 2026 15:24:40 -0700 Subject: [PATCH 02/17] fix: proper entrypoint, foreground sidecar, correct port (#2) - Dockerfile: change ENTRYPOINT from /bin/bash to /usr/local/bin/nemoclaw-start so the container actually runs the startup script instead of exiting immediately - nemoclaw-start.sh: run sidecar in foreground via exec (keeps container alive) with tail -f fallback if no sidecar present - sidecar.js: read PORT env var (set by fleet.ts to 3100) instead of WOPR_SIDECAR_PORT defaulting to 3001 Co-authored-by: WOPR Co-authored-by: Claude Opus 4.6 (1M context) --- Dockerfile | 3 +++ scripts/nemoclaw-start.sh | 7 ++++--- wopr/sidecar.js | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1451bd360..7548c4a59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -149,6 +149,9 @@ RUN sha256sum /sandbox/.openclaw/openclaw.json > /sandbox/.openclaw/.config-hash && chmod 444 /sandbox/.openclaw/.config-hash \ && chown root:root /sandbox/.openclaw/.config-hash +# Expose WOPR sidecar port +EXPOSE 3100 + # Entrypoint runs as root to start the gateway as the gateway user, # then drops to sandbox for agent commands. See nemoclaw-start.sh. ENTRYPOINT ["/usr/local/bin/nemoclaw-start"] diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index ca46964ac..934e10b18 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -293,10 +293,11 @@ echo "[gateway] openclaw gateway launched as 'gateway' user (pid $GATEWAY_PID)" start_auto_pair print_dashboard_urls -# Start WOPR sidecar — /internal/health + /internal/provision for nemoclaw-platform +# Start WOPR sidecar in foreground — keeps the container alive. +# Serves /internal/health + /internal/provision for nemoclaw-platform. if [ -f /opt/wopr/sidecar.js ]; then - nohup node /opt/wopr/sidecar.js > /tmp/wopr-sidecar.log 2>&1 & - echo "[wopr-sidecar] launched (pid $!)" + echo "[wopr-sidecar] starting in foreground (port ${PORT:-3100})" + exec node /opt/wopr/sidecar.js fi # Keep container running by waiting on the gateway process. diff --git a/wopr/sidecar.js b/wopr/sidecar.js index f697482b1..790c7d604 100644 --- a/wopr/sidecar.js +++ b/wopr/sidecar.js @@ -14,7 +14,7 @@ import os from "node:os"; const SECRET = process.env.WOPR_PROVISION_SECRET ?? ""; const GATEWAY_URL = process.env.WOPR_GATEWAY_URL ?? ""; -const PORT = parseInt(process.env.WOPR_SIDECAR_PORT ?? "3001", 10); +const PORT = parseInt(process.env.PORT ?? process.env.WOPR_SIDECAR_PORT ?? "3100", 10); const OPENCLAW_CONFIG_PATH = path.join(os.homedir(), ".openclaw", "openclaw.json"); function assertSecret(req) { From 6b2acf72af5ffd83b67473b0c1874120fab2c02d Mon Sep 17 00:00:00 2001 From: T Savo Date: Sun, 22 Mar 2026 15:40:41 -0700 Subject: [PATCH 03/17] fix: writable HOME for read-only rootfs containers (#3) FleetManager creates containers with ReadonlyRootfs=true. The startup script was writing to /sandbox/.openclaw/ which is in the read-only rootfs, causing OSError: Read-only file system. Fix: - Dockerfile: save build-time openclaw config to /opt/nemoclaw-defaults/, set HOME=/data so all writes go to the writable volume mount - nemoclaw-start.sh: copy default config from /opt/nemoclaw-defaults/ to $HOME/.openclaw/ on first boot before running openclaw commands Co-authored-by: WOPR Co-authored-by: Claude Opus 4.6 (1M context) --- Dockerfile | 16 +++++++++++++++- scripts/nemoclaw-start.sh | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7548c4a59..28a194dde 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,6 +75,12 @@ COPY wopr/ /opt/wopr/ WORKDIR /sandbox USER sandbox +# Pre-create OpenClaw directories and write default config. +# These are saved to /opt/nemoclaw-defaults/ (read-only at runtime). +# The startup script copies them to $HOME/.openclaw/ (writable volume). +RUN mkdir -p /sandbox/.openclaw/agents/main/agent \ + && chmod 700 /sandbox/.openclaw + # Write the COMPLETE openclaw.json including gateway config and auth token. # This file is immutable at runtime (Landlock read-only on /sandbox/.openclaw). # No runtime writes to openclaw.json are needed or possible. @@ -121,7 +127,6 @@ path = os.path.expanduser('~/.openclaw/openclaw.json'); \ json.dump(config, open(path, 'w'), indent=2); \ os.chmod(path, 0o600)" -# Install NemoClaw plugin into OpenClaw RUN openclaw doctor --fix > /dev/null 2>&1 || true \ && openclaw plugins install /opt/nemoclaw > /dev/null 2>&1 || true @@ -149,6 +154,15 @@ RUN sha256sum /sandbox/.openclaw/openclaw.json > /sandbox/.openclaw/.config-hash && chmod 444 /sandbox/.openclaw/.config-hash \ && chown root:root /sandbox/.openclaw/.config-hash +# Save build-time config as defaults — startup script copies to writable HOME +RUN cp -a /sandbox/.openclaw /opt/nemoclaw-defaults \ + && cp -a /sandbox/.nemoclaw /opt/nemoclaw-defaults/.nemoclaw +USER sandbox + +# At runtime, HOME=/data (writable volume mount from FleetManager). +# ReadonlyRootfs makes /sandbox read-only, so all writes go to /data. +ENV HOME=/data + # Expose WOPR sidecar port EXPOSE 3100 diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index 934e10b18..7a7661441 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -207,6 +207,20 @@ PYAUTOPAIR # ── Main ───────────────────────────────────────────────────────── +# Copy default config to writable HOME if not already present. +# FleetManager mounts a volume at /data and sets HOME=/data. +# The rootfs is read-only, so all writes must go to /data. +if [ -d /opt/nemoclaw-defaults ] && [ ! -f "${HOME}/.openclaw/openclaw.json" ]; then + echo "[init] Copying default config to ${HOME}/.openclaw/" + mkdir -p "${HOME}/.openclaw" "${HOME}/.nemoclaw" + cp -a /opt/nemoclaw-defaults/. "${HOME}/.openclaw/" + [ -d /opt/nemoclaw-defaults/.nemoclaw ] && cp -a /opt/nemoclaw-defaults/.nemoclaw/. "${HOME}/.nemoclaw/" +fi + +# Ensure writable dirs exist +mkdir -p "${HOME}/.openclaw/agents/main/agent" 2>/dev/null || true +touch /tmp/gateway.log 2>/dev/null || true + echo 'Setting up NemoClaw...' [ -f .env ] && chmod 600 .env From be8bffd2edf529c8f2471c231959d7b764c68d9b Mon Sep 17 00:00:00 2001 From: T Savo Date: Sun, 22 Mar 2026 16:19:17 -0700 Subject: [PATCH 04/17] fix: add provisioning field to sidecar health response (#4) The platform's checkHealth expects { ok: true, provisioning: true } but the sidecar only returned { ok, gateway }. Without the provisioning field, health checks always fail, blocking subdomain routing and container provision. Co-authored-by: WOPR Co-authored-by: Claude Opus 4.6 (1M context) --- wopr/sidecar.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wopr/sidecar.js b/wopr/sidecar.js index 790c7d604..58e6e5173 100644 --- a/wopr/sidecar.js +++ b/wopr/sidecar.js @@ -112,7 +112,7 @@ const server = http.createServer((req, res) => { if (req.method === "GET" && url.pathname === "/internal/health") { const up = isGatewayUp(); res.writeHead(up ? 200 : 503, { "content-type": "application/json" }); - res.end(JSON.stringify({ ok: up, gateway: up ? "running" : "starting" })); + res.end(JSON.stringify({ ok: up, provisioning: up, gateway: up ? "running" : "starting" })); return; } From 2099a690e95aa1ae5e3d02b0d8ff703bea44cf9d Mon Sep 17 00:00:00 2001 From: T Savo Date: Sun, 22 Mar 2026 16:36:07 -0700 Subject: [PATCH 05/17] fix: serve health on /internal/provision/health for provision-client (#5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit provision-client's checkHealth() hits /internal/provision/health, not /internal/health. The sidecar only served /internal/health, so health checks always got 404 → always unhealthy → no subdomain routing, no provisioning. Co-authored-by: WOPR Co-authored-by: Claude Opus 4.6 (1M context) --- wopr/sidecar.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wopr/sidecar.js b/wopr/sidecar.js index 58e6e5173..52f4d001a 100644 --- a/wopr/sidecar.js +++ b/wopr/sidecar.js @@ -108,8 +108,8 @@ function provision(body) { const server = http.createServer((req, res) => { const url = new URL(req.url, `http://localhost:${PORT}`); - // Health check — no auth required - if (req.method === "GET" && url.pathname === "/internal/health") { + // Health check — no auth required (provision-client checks /internal/provision/health) + if (req.method === "GET" && (url.pathname === "/internal/health" || url.pathname === "/internal/provision/health")) { const up = isGatewayUp(); res.writeHead(up ? 200 : 503, { "content-type": "application/json" }); res.end(JSON.stringify({ ok: up, provisioning: up, gateway: up ? "running" : "starting" })); From 2c4d391936bba7cf37155c3e95f5d9f755e236ec Mon Sep 17 00:00:00 2001 From: WOPR Date: Wed, 25 Mar 2026 20:25:25 -0700 Subject: [PATCH 06/17] =?UTF-8?q?feat:=20add=20upstream=20sync=20workflow?= =?UTF-8?q?=20=E2=80=94=20daily=20rebase=20from=20NVIDIA/NemoClaw?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent SDK-powered rebase that: 1. Fetches NVIDIA/NemoClaw upstream daily at 06:30 UTC 2. Rebases our 5 WOPR sidecar commits on top 3. Resolves conflicts via Claude Agent SDK 4. Verifies sidecar + Dockerfile integrity 5. Creates a PR (or force-pushes with --push flag) Uses extraheader auth to bypass self-hosted runner credential helpers. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/upstream-sync.yml | 94 +++++++ scripts/upstream-sync.mjs | 377 ++++++++++++++++++++++++++++ 2 files changed, 471 insertions(+) create mode 100644 .github/workflows/upstream-sync.yml create mode 100644 scripts/upstream-sync.mjs diff --git a/.github/workflows/upstream-sync.yml b/.github/workflows/upstream-sync.yml new file mode 100644 index 000000000..d89be9823 --- /dev/null +++ b/.github/workflows/upstream-sync.yml @@ -0,0 +1,94 @@ +name: Upstream Sync + +on: + schedule: + # Daily at 06:30 UTC (offset from paperclip's 06:00) + - cron: "30 6 * * *" + workflow_dispatch: + inputs: + mode: + description: "Sync mode" + required: true + default: "pr" + type: choice + options: + - pr + - push + - dry-run + +jobs: + sync: + runs-on: self-hosted + timeout-minutes: 20 + + steps: + - name: Checkout (full history for rebase) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GH_PAT }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + # Nuke stale credential helpers on self-hosted runner + git config --local --unset-all credential.helper 2>/dev/null || true + git config --global --unset-all credential.helper 2>/dev/null || true + + - name: Clean working tree (self-hosted runner may have leftovers) + run: | + git checkout main 2>/dev/null || true + git rebase --abort 2>/dev/null || true + git reset --hard HEAD + git clean -fd + + - name: Add upstream remote + run: | + git remote get-url upstream 2>/dev/null || \ + git remote add upstream https://github.com/NVIDIA/NemoClaw.git + git fetch upstream + + - name: Install Agent SDK + run: | + npm install -g @anthropic-ai/claude-code + npm install -g @anthropic-ai/claude-agent-sdk + + - name: Check for upstream changes + id: check + run: | + BEHIND=$(git rev-list HEAD..upstream/main --count) + echo "behind=$BEHIND" >> "$GITHUB_OUTPUT" + if [ "$BEHIND" -eq 0 ]; then + echo "Up to date with upstream. Skipping sync." + else + echo "Behind upstream by $BEHIND commits." + fi + + - name: Run upstream sync + if: steps.check.outputs.behind != '0' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + GH_TOKEN: ${{ secrets.GH_PAT }} + run: | + MODE="${{ github.event.inputs.mode || 'pr' }}" + case "$MODE" in + push) FLAG="--push" ;; + pr) FLAG="--pr" ;; + dry-run) FLAG="--dry-run" ;; + esac + node scripts/upstream-sync.mjs $FLAG + + - name: Upload agent event log + if: always() + uses: actions/upload-artifact@v4 + with: + name: agent-events-log + path: agent-events.log + if-no-files-found: ignore + retention-days: 14 diff --git a/scripts/upstream-sync.mjs b/scripts/upstream-sync.mjs new file mode 100644 index 000000000..d0d531393 --- /dev/null +++ b/scripts/upstream-sync.mjs @@ -0,0 +1,377 @@ +#!/usr/bin/env node +/** + * upstream-sync.mjs + * + * Keeps the wopr-network/nemoclaw fork rebased on NVIDIA/NemoClaw upstream. + * + * Our fork adds a WOPR sidecar (wopr/ directory) + Dockerfile/entrypoint + * tweaks for managed hosting. Upstream changes are always taken; our + * additions are rebased on top. + * + * 1. Fetches upstream and checks for new commits + * 2. Rebases our sidecar commits on top + * 3. Resolves any rebase conflicts (via Agent SDK) + * 4. Runs a build check + * 5. Pushes or creates a PR + * + * Usage: + * node scripts/upstream-sync.mjs [options] + * + * Options: + * --dry-run Report status but don't push + * --push Force-push main after sync + * --pr Create a PR instead of pushing (default for cron) + * + * Requires: + * - CLAUDE_CODE_OAUTH_TOKEN or ANTHROPIC_API_KEY env var + * - @anthropic-ai/claude-agent-sdk (npm install -g) + * - git remotes: origin (wopr-network), upstream (NVIDIA) + */ + +import { execSync } from "node:child_process"; +import { existsSync, appendFileSync, writeFileSync, copyFileSync } from "node:fs"; +import { join } from "node:path"; + +const CWD = process.cwd(); +const DRY_RUN = process.argv.includes("--dry-run"); +const AUTO_PUSH = process.argv.includes("--push"); +const CREATE_PR = process.argv.includes("--pr"); + +// Agent event log — saved as CI artifact +const AGENT_LOG_TMP = join("/tmp", `agent-events-${Date.now()}.log`); +const AGENT_LOG_PATH = join(CWD, "agent-events.log"); +writeFileSync(AGENT_LOG_TMP, `=== upstream-sync agent log — ${new Date().toISOString()} ===\n`); + +function logEvent(phase, event) { + const ts = new Date().toISOString(); + appendFileSync(AGENT_LOG_TMP, `[${ts}] [${phase}] ${JSON.stringify(event)}\n`); +} + +function flushLog() { + try { copyFileSync(AGENT_LOG_TMP, AGENT_LOG_PATH); } catch { /* best-effort */ } +} + +// --------------------------------------------------------------------------- +// Shell helpers +// --------------------------------------------------------------------------- + +function run(cmd) { + return execSync(cmd, { cwd: CWD, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim(); +} + +function tryRun(cmd) { + try { + return { ok: true, output: run(cmd) }; + } catch (e) { + return { ok: false, output: (e.stderr || e.message || "").trim() }; + } +} + +function log(msg) { console.log(`[upstream-sync] ${msg}`); } + +function die(msg) { + flushLog(); + console.error(`[upstream-sync] FATAL: ${msg}`); + process.exit(1); +} + +// --------------------------------------------------------------------------- +// Agent SDK wrapper +// --------------------------------------------------------------------------- + +let _query; + +async function loadSdk() { + if (_query) return; + const globalRoot = execSync("npm root -g", { encoding: "utf-8" }).trim(); + const candidates = [ + "@anthropic-ai/claude-agent-sdk", + `${globalRoot}/@anthropic-ai/claude-agent-sdk/sdk.mjs`, + ]; + for (const candidate of candidates) { + try { + const sdk = await import(candidate); + _query = sdk.query; + return; + } catch { /* try next */ } + } + die("@anthropic-ai/claude-agent-sdk not installed.\n npm install -g @anthropic-ai/claude-agent-sdk\n npm install -g @anthropic-ai/claude-code"); +} + +async function runAgent(prompt, opts = {}) { + await loadSdk(); + const phase = opts.phase ?? "unknown"; + const tools = opts.tools ?? ["Read", "Edit", "Write", "Bash", "Glob", "Grep"]; + let result = ""; + let turnCount = 0; + + log(`Agent [${phase}] starting (model: ${opts.model ?? "claude-sonnet-4-6"}, maxTurns: ${opts.maxTurns ?? 60})`); + logEvent(phase, { type: "agent_start", model: opts.model, maxTurns: opts.maxTurns ?? 60 }); + + for await (const message of _query({ + prompt, + options: { + cwd: CWD, + allowedTools: tools, + permissionMode: "bypassPermissions", + allowDangerouslySkipPermissions: true, + maxTurns: opts.maxTurns ?? 60, + model: opts.model ?? "claude-sonnet-4-6", + }, + })) { + if (message.type === "tool_use") { + turnCount++; + logEvent(phase, { type: "tool_use", turn: turnCount, tool: message.tool, input_preview: JSON.stringify(message.input).slice(0, 200) }); + } else if (message.type === "text") { + logEvent(phase, { type: "text", preview: (message.text || "").slice(0, 300) }); + } else if ("result" in message) { + result = message.result; + logEvent(phase, { type: "result", preview: result.slice(0, 500) }); + } else { + logEvent(phase, { type: message.type || "unknown", keys: Object.keys(message) }); + } + } + + log(`Agent [${phase}] finished after ${turnCount} tool calls`); + logEvent(phase, { type: "agent_done", turnCount }); + return result; +} + +// --------------------------------------------------------------------------- +// Fork context (shared across agent prompts) +// --------------------------------------------------------------------------- + +const FORK_CONTEXT = ` +## Context: WOPR NemoClaw Fork + +This is a fork of NVIDIA/NemoClaw maintained by wopr-network. +The fork adds a WOPR sidecar for managed hosting. Our additions: + +### Files we added (preserve these): +- \`wopr/sidecar.js\` — HTTP sidecar exposing /internal/health and /internal/provision +- \`wopr/package.json\` — sidecar dependencies +- Dockerfile modifications — adds sidecar setup and entrypoint changes +- Entrypoint tweaks — foreground sidecar, correct port, writable HOME + +### Conflict Resolution Rules: +1. TAKE all of upstream's changes (new features, bug fixes, security hardening) +2. REAPPLY our wopr/ additions on top +3. If upstream changed Dockerfile or entrypoint, adapt our additions to the new structure +4. Never drop upstream functionality — only add our sidecar layer +5. Keep wopr/ directory intact +`; + +// --------------------------------------------------------------------------- +// Rebase +// --------------------------------------------------------------------------- + +async function rebase() { + log("Fetching upstream..."); + run("git fetch upstream"); + + const behind = parseInt(run("git rev-list HEAD..upstream/main --count"), 10); + const ahead = parseInt(run("git rev-list upstream/main..HEAD --count"), 10); + + if (behind === 0) { + log("Already up to date with upstream."); + return { rebased: false, behind: 0, ahead }; + } + + log(`Behind upstream by ${behind} commits, ahead by ${ahead} commits.`); + + // Backup + const datestamp = new Date().toISOString().slice(0, 10); + const backupBranch = `backup/pre-sync-${datestamp}`; + tryRun(`git branch -D ${backupBranch}`); + run(`git branch ${backupBranch}`); + log(`Backup: ${backupBranch}`); + + // Attempt rebase + log("Rebasing onto upstream/main..."); + const rebaseResult = tryRun("git rebase upstream/main"); + + if (rebaseResult.ok) { + log("Rebase succeeded cleanly."); + return { rebased: true, behind, ahead }; + } + + // Conflicts — invoke agent + log("Rebase has conflicts. Invoking agent to resolve..."); + const conflicting = tryRun("git diff --name-only --diff-filter=U"); + const conflictFiles = conflicting.ok ? conflicting.output : "unknown"; + + await runAgent( + `You are resolving git rebase conflicts in a NemoClaw fork. + +${FORK_CONTEXT} + +## Current Conflicts + +These files have conflicts: +${conflictFiles} + +## Steps + +1. For each conflicting file, read it and find the conflict markers (<<<<<<< / ======= / >>>>>>>) +2. Resolve each conflict following the rules above +3. Run: git add +4. After ALL conflicts are resolved, run: git rebase --continue +5. If new conflicts appear, repeat +6. Continue until the rebase completes + +IMPORTANT: Do NOT use git rebase --abort. Resolve all conflicts.`, + { model: "claude-sonnet-4-6", maxTurns: 80, phase: "rebase-conflicts" }, + ); + + // Verify rebase completed + const status = tryRun("git rebase --show-current-patch"); + if (status.ok) { + die("Rebase still in progress after agent intervention. Manual resolution needed."); + } + + log("Rebase completed after conflict resolution."); + return { rebased: true, behind, ahead }; +} + +// --------------------------------------------------------------------------- +// Build check +// --------------------------------------------------------------------------- + +async function buildCheck() { + log("Running build check..."); + + // Check sidecar syntax + const sidecarCheck = tryRun("node --check wopr/sidecar.js"); + if (!sidecarCheck.ok) { + log("Sidecar syntax check failed. Invoking agent to fix..."); + await runAgent( + `The WOPR sidecar has a syntax error after upstream sync: + +\`\`\` +${sidecarCheck.output.slice(0, 2000)} +\`\`\` + +Fix the syntax error in wopr/sidecar.js. Do NOT remove sidecar functionality.`, + { model: "claude-sonnet-4-6", phase: "sidecar-fix" }, + ); + } + + // Check Dockerfile builds (syntax only — full build is too slow for CI sync) + if (existsSync(`${CWD}/Dockerfile`)) { + // docker build --check isn't universally available, so just verify Dockerfile parses + const hasFrom = tryRun("grep -q '^FROM' Dockerfile"); + if (!hasFrom.ok) { + log("Dockerfile appears broken (no FROM instruction)."); + return false; + } + } + + log("Build check passed."); + return true; +} + +// --------------------------------------------------------------------------- +// Push / PR +// --------------------------------------------------------------------------- + +function pushOrPr() { + if (DRY_RUN) { + log("Dry run — skipping push."); + return; + } + + // Configure git auth — use extraheader to bypass credential helpers + const ghToken = process.env.GH_TOKEN; + if (ghToken) { + tryRun("git config --local --unset-all credential.helper"); + tryRun("git config --global --unset-all credential.helper"); + const basicAuth = Buffer.from(`x-access-token:${ghToken}`).toString("base64"); + tryRun("git config --local --unset-all http.https://github.com/.extraheader"); + run(`git config --local http.https://github.com/.extraheader "AUTHORIZATION: basic ${basicAuth}"`); + log("Configured git auth via extraheader (bypasses credential helpers)."); + } + + if (AUTO_PUSH) { + log("Force-pushing to origin/main..."); + run("git push --force-with-lease origin main"); + log("Pushed successfully."); + } else if (CREATE_PR) { + const datestamp = new Date().toISOString().slice(0, 10); + const branch = `sync/upstream-${datestamp}`; + tryRun(`git branch -D ${branch}`); + run(`git checkout -b ${branch}`); + run(`git push -u origin ${branch} --force-with-lease`); + + const prBody = [ + "## Automated upstream sync", + "", + "Rebased our WOPR sidecar commits onto upstream/main (NVIDIA/NemoClaw).", + "", + "### What this does", + "- Pulls in latest upstream changes (security fixes, features, CI improvements)", + "- Resolves any rebase conflicts (preserving wopr/ sidecar)", + "- Verifies sidecar + Dockerfile integrity", + "", + "### Verify", + "- [ ] Build passes", + "- [ ] wopr/sidecar.js intact", + "- [ ] Dockerfile includes sidecar setup", + ].join("\n"); + + const pr = tryRun( + `gh pr create --title "sync: rebase on upstream (${datestamp})" --body "${prBody.replace(/"/g, '\\"')}" --base main`, + ); + if (pr.ok) { + log(`PR created: ${pr.output}`); + } else { + log(`PR creation failed: ${pr.output}`); + } + + run("git checkout main"); + } else { + log("Sync complete. Use --push to force-push or --pr to create a PR."); + } +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + const remotes = tryRun("git remote -v"); + if (!remotes.output.includes("nemoclaw")) { + die("Not in a nemoclaw repo."); + } + + if (!tryRun("git remote get-url upstream").ok) { + die("No 'upstream' remote. Add with: git remote add upstream https://github.com/NVIDIA/NemoClaw.git"); + } + + const status = run("git status --porcelain"); + if (status) { + die("Working tree is dirty. Commit or stash changes first."); + } + + const { rebased, behind } = await rebase(); + + if (!rebased && behind === 0) { + log("Up to date. Nothing to do."); + flushLog(); + return; + } + + const buildOk = await buildCheck(); + if (!buildOk) { + die("Build failed. Not pushing."); + } + + pushOrPr(); + flushLog(); + log("Done."); +} + +main().catch((err) => { + flushLog(); + console.error(err); + process.exit(1); +}); From ed63c2b949c97baf1b4f464fb33c7e7b7c78b984 Mon Sep 17 00:00:00 2001 From: WOPR Date: Wed, 25 Mar 2026 20:33:24 -0700 Subject: [PATCH 07/17] fix: nuclear auth override for self-hosted runner credential manager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit extraheader alone wasn't enough — system credential manager still intercepted. Now uses: token-embedded URL + extraheader + env overrides (GIT_ASKPASS=/bin/true, GCM_INTERACTIVE=never) + system-level credential.helper unset. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/upstream-sync.mjs | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/scripts/upstream-sync.mjs b/scripts/upstream-sync.mjs index d0d531393..5465896c8 100644 --- a/scripts/upstream-sync.mjs +++ b/scripts/upstream-sync.mjs @@ -280,27 +280,46 @@ function pushOrPr() { return; } - // Configure git auth — use extraheader to bypass credential helpers + // Configure git auth — set remote URL with token directly and kill all credential helpers const ghToken = process.env.GH_TOKEN; + const pushEnv = {}; if (ghToken) { + // Nuclear option: override remote URL with embedded token AND disable credential helpers + // via env vars (takes precedence over all config levels including system) + run(`git remote set-url origin https://x-access-token:${ghToken}@github.com/wopr-network/nemoclaw.git`); tryRun("git config --local --unset-all credential.helper"); tryRun("git config --global --unset-all credential.helper"); + tryRun("git config --system --unset-all credential.helper"); + // Also set extraheader as belt-and-suspenders const basicAuth = Buffer.from(`x-access-token:${ghToken}`).toString("base64"); tryRun("git config --local --unset-all http.https://github.com/.extraheader"); run(`git config --local http.https://github.com/.extraheader "AUTHORIZATION: basic ${basicAuth}"`); - log("Configured git auth via extraheader (bypasses credential helpers)."); + // Env-level overrides that bypass any credential manager + pushEnv.GIT_TERMINAL_PROMPT = "0"; + pushEnv.GIT_ASKPASS = "/bin/true"; + pushEnv.GCM_INTERACTIVE = "never"; + log("Configured git auth (token URL + extraheader + env overrides)."); + } + + function gitPush(cmd) { + return execSync(cmd, { + cwd: CWD, + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + env: { ...process.env, ...pushEnv }, + }).trim(); } if (AUTO_PUSH) { log("Force-pushing to origin/main..."); - run("git push --force-with-lease origin main"); + gitPush("git push --force-with-lease origin main"); log("Pushed successfully."); } else if (CREATE_PR) { const datestamp = new Date().toISOString().slice(0, 10); const branch = `sync/upstream-${datestamp}`; tryRun(`git branch -D ${branch}`); run(`git checkout -b ${branch}`); - run(`git push -u origin ${branch} --force-with-lease`); + gitPush(`git push -u origin ${branch} --force-with-lease`); const prBody = [ "## Automated upstream sync", From 6e5712c54032cea4b80745fb94f6435294fd2992 Mon Sep 17 00:00:00 2001 From: WOPR Date: Wed, 25 Mar 2026 20:41:22 -0700 Subject: [PATCH 08/17] =?UTF-8?q?fix:=20use=20inline=20git=20-c=20flags=20?= =?UTF-8?q?for=20auth=20=E2=80=94=20beats=20system=20credential=20manager?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous approaches (config unset, env vars, URL tokens) all failed because the self-hosted runner's system-level GCM intercepts before any of those take effect. Inline -c flags on the git command itself are the only override that takes absolute precedence. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/upstream-sync.mjs | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/scripts/upstream-sync.mjs b/scripts/upstream-sync.mjs index 5465896c8..479a38445 100644 --- a/scripts/upstream-sync.mjs +++ b/scripts/upstream-sync.mjs @@ -280,46 +280,27 @@ function pushOrPr() { return; } - // Configure git auth — set remote URL with token directly and kill all credential helpers + // Configure git auth — use inline -c flags to override ALL config levels (including system) + // This is the only approach that beats a system-level credential manager without sudo const ghToken = process.env.GH_TOKEN; - const pushEnv = {}; + let gitPushPrefix = "git"; if (ghToken) { - // Nuclear option: override remote URL with embedded token AND disable credential helpers - // via env vars (takes precedence over all config levels including system) - run(`git remote set-url origin https://x-access-token:${ghToken}@github.com/wopr-network/nemoclaw.git`); - tryRun("git config --local --unset-all credential.helper"); - tryRun("git config --global --unset-all credential.helper"); - tryRun("git config --system --unset-all credential.helper"); - // Also set extraheader as belt-and-suspenders const basicAuth = Buffer.from(`x-access-token:${ghToken}`).toString("base64"); - tryRun("git config --local --unset-all http.https://github.com/.extraheader"); - run(`git config --local http.https://github.com/.extraheader "AUTHORIZATION: basic ${basicAuth}"`); - // Env-level overrides that bypass any credential manager - pushEnv.GIT_TERMINAL_PROMPT = "0"; - pushEnv.GIT_ASKPASS = "/bin/true"; - pushEnv.GCM_INTERACTIVE = "never"; - log("Configured git auth (token URL + extraheader + env overrides)."); - } - - function gitPush(cmd) { - return execSync(cmd, { - cwd: CWD, - encoding: "utf-8", - stdio: ["pipe", "pipe", "pipe"], - env: { ...process.env, ...pushEnv }, - }).trim(); + // -c flags override system/global/local config for this command only + gitPushPrefix = `git -c credential.helper= -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${basicAuth}"`; + log("Will use inline -c auth overrides for push."); } if (AUTO_PUSH) { log("Force-pushing to origin/main..."); - gitPush("git push --force-with-lease origin main"); + run(`${gitPushPrefix} push --force-with-lease origin main`); log("Pushed successfully."); } else if (CREATE_PR) { const datestamp = new Date().toISOString().slice(0, 10); const branch = `sync/upstream-${datestamp}`; tryRun(`git branch -D ${branch}`); run(`git checkout -b ${branch}`); - gitPush(`git push -u origin ${branch} --force-with-lease`); + run(`${gitPushPrefix} push -u origin ${branch} --force-with-lease`); const prBody = [ "## Automated upstream sync", From 4fdeb9258b4d3ede70a72f83cde8de29227eb1ce Mon Sep 17 00:00:00 2001 From: WOPR Date: Wed, 25 Mar 2026 20:43:12 -0700 Subject: [PATCH 09/17] feat: add :staging tag and VPS deploy to CI Push :staging tag alongside :latest, SSH to staging VPS, copy prod DB, restart staging containers, verify health. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/ci.yml | 45 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3631a852c..d47927da0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,51 @@ jobs: push: ${{ github.ref == 'refs/heads/main' }} tags: | ghcr.io/${{ github.repository }}:latest + ghcr.io/${{ github.repository }}:staging ghcr.io/${{ github.repository }}:${{ github.sha }} cache-from: type=gha cache-to: type=gha,mode=max + + deploy-staging: + if: github.ref == 'refs/heads/main' + runs-on: [self-hosted, Linux, X64] + needs: build-and-push + steps: + - name: Deploy to staging VPS + uses: appleboy/ssh-action@v1 + with: + host: ${{ secrets.STAGING_HOST }} + username: root + key: ${{ secrets.PROD_SSH_KEY }} + script: | + set -euo pipefail + cd /opt/nemoclaw-platform + COMPOSE="docker compose -f docker-compose.yml -f docker-compose.staging.yml" + $COMPOSE stop staging-api 2>/dev/null || true + $COMPOSE pull staging-api + $COMPOSE up -d staging-postgres + sleep 3 + $COMPOSE exec -T staging-postgres psql -U nemoclaw -d nemoclaw_platform -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;" 2>/dev/null || true + docker compose exec -T postgres pg_dump -U nemoclaw --no-owner --no-acl nemoclaw_platform | \ + $COMPOSE exec -T staging-postgres psql -U nemoclaw -d nemoclaw_platform -q + $COMPOSE up -d staging-api staging-ui + docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile 2>/dev/null || docker compose restart caddy + echo "Staging deployed" + + - name: Verify staging health + uses: appleboy/ssh-action@v1 + with: + host: ${{ secrets.STAGING_HOST }} + username: root + key: ${{ secrets.PROD_SSH_KEY }} + script: | + for i in $(seq 1 30); do + if docker exec nemoclaw-platform-staging-api-1 curl -sf http://localhost:3100/health >/dev/null 2>&1; then + echo "Staging API is healthy" + exit 0 + fi + sleep 2 + done + echo "Health check timed out" + docker logs nemoclaw-platform-staging-api-1 --tail 20 + exit 1 From 54422a2e8529dab454ab157f48818d08870ad074 Mon Sep 17 00:00:00 2001 From: WOPR Date: Wed, 25 Mar 2026 20:44:39 -0700 Subject: [PATCH 10/17] fix: clean stale npm globals before installing Agent SDK Self-hosted runner had leftover @anthropic-ai/claude-code dir causing ENOTEMPTY on npm install -g. Remove old install before reinstalling. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/upstream-sync.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/upstream-sync.yml b/.github/workflows/upstream-sync.yml index d89be9823..7284107ba 100644 --- a/.github/workflows/upstream-sync.yml +++ b/.github/workflows/upstream-sync.yml @@ -56,6 +56,8 @@ jobs: - name: Install Agent SDK run: | + npm cache clean --force 2>/dev/null || true + rm -rf "$(npm root -g)/@anthropic-ai/claude-code" "$(npm root -g)/@anthropic-ai/claude-agent-sdk" 2>/dev/null || true npm install -g @anthropic-ai/claude-code npm install -g @anthropic-ai/claude-agent-sdk From 6a953ad4d9cd11023f617a7b9f1402e1a878381b Mon Sep 17 00:00:00 2001 From: WOPR Date: Wed, 25 Mar 2026 20:51:37 -0700 Subject: [PATCH 11/17] fix: use GIT_CONFIG_COUNT env vars for push auth -c flags had shell quoting issues with the AUTHORIZATION header space. GIT_CONFIG_COUNT/KEY/VALUE env vars set via execSync env option bypass all config levels cleanly without shell interpolation. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/upstream-sync.mjs | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/scripts/upstream-sync.mjs b/scripts/upstream-sync.mjs index 479a38445..5b7a42d24 100644 --- a/scripts/upstream-sync.mjs +++ b/scripts/upstream-sync.mjs @@ -280,27 +280,41 @@ function pushOrPr() { return; } - // Configure git auth — use inline -c flags to override ALL config levels (including system) - // This is the only approach that beats a system-level credential manager without sudo + // Configure git auth — use GIT_CONFIG_COUNT env vars to override ALL config levels + // This bypasses system credential managers without sudo and avoids shell quoting issues const ghToken = process.env.GH_TOKEN; - let gitPushPrefix = "git"; + const pushEnv = {}; if (ghToken) { const basicAuth = Buffer.from(`x-access-token:${ghToken}`).toString("base64"); - // -c flags override system/global/local config for this command only - gitPushPrefix = `git -c credential.helper= -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${basicAuth}"`; - log("Will use inline -c auth overrides for push."); + // GIT_CONFIG_COUNT/KEY/VALUE env vars override all config levels for the process + pushEnv.GIT_CONFIG_COUNT = "2"; + pushEnv.GIT_CONFIG_KEY_0 = "credential.helper"; + pushEnv.GIT_CONFIG_VALUE_0 = ""; + pushEnv.GIT_CONFIG_KEY_1 = "http.https://github.com/.extraheader"; + pushEnv.GIT_CONFIG_VALUE_1 = `AUTHORIZATION: basic ${basicAuth}`; + pushEnv.GIT_TERMINAL_PROMPT = "0"; + log("Will use GIT_CONFIG env overrides for push."); + } + + function gitPush(args) { + return execSync(`git ${args}`, { + cwd: CWD, + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + env: { ...process.env, ...pushEnv }, + }).trim(); } if (AUTO_PUSH) { log("Force-pushing to origin/main..."); - run(`${gitPushPrefix} push --force-with-lease origin main`); + gitPush("push --force-with-lease origin main"); log("Pushed successfully."); } else if (CREATE_PR) { const datestamp = new Date().toISOString().slice(0, 10); const branch = `sync/upstream-${datestamp}`; tryRun(`git branch -D ${branch}`); run(`git checkout -b ${branch}`); - run(`${gitPushPrefix} push -u origin ${branch} --force-with-lease`); + gitPush(`push -u origin ${branch} --force-with-lease`); const prBody = [ "## Automated upstream sync", From 4f864c0580e1a3b4d93972f9ee8a13e1c4b8d3b4 Mon Sep 17 00:00:00 2001 From: WOPR Date: Wed, 25 Mar 2026 20:58:50 -0700 Subject: [PATCH 12/17] fix: use credential helper script for push auth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Write a tiny shell script that echoes username/password, then override credential.helper via GIT_CONFIG_COUNT env. No base64, no extraheader, no quoting issues — git's native credential protocol handles it all. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/upstream-sync.mjs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/scripts/upstream-sync.mjs b/scripts/upstream-sync.mjs index 5b7a42d24..c783f613a 100644 --- a/scripts/upstream-sync.mjs +++ b/scripts/upstream-sync.mjs @@ -280,20 +280,21 @@ function pushOrPr() { return; } - // Configure git auth — use GIT_CONFIG_COUNT env vars to override ALL config levels - // This bypasses system credential managers without sudo and avoids shell quoting issues + // Configure git auth — write a tiny credential helper that returns the GH_TOKEN + // This is the simplest approach: no headers, no base64, no quoting issues const ghToken = process.env.GH_TOKEN; - const pushEnv = {}; + const pushEnv = { ...process.env, GIT_TERMINAL_PROMPT: "0" }; if (ghToken) { - const basicAuth = Buffer.from(`x-access-token:${ghToken}`).toString("base64"); - // GIT_CONFIG_COUNT/KEY/VALUE env vars override all config levels for the process + // Write a one-shot credential helper script + const helperPath = join("/tmp", `git-cred-helper-${Date.now()}.sh`); + writeFileSync(helperPath, `#!/bin/sh\necho "username=x-access-token"\necho "password=${ghToken}"\n`, { mode: 0o700 }); + // GIT_CONFIG_COUNT overrides all config levels including system pushEnv.GIT_CONFIG_COUNT = "2"; pushEnv.GIT_CONFIG_KEY_0 = "credential.helper"; - pushEnv.GIT_CONFIG_VALUE_0 = ""; - pushEnv.GIT_CONFIG_KEY_1 = "http.https://github.com/.extraheader"; - pushEnv.GIT_CONFIG_VALUE_1 = `AUTHORIZATION: basic ${basicAuth}`; - pushEnv.GIT_TERMINAL_PROMPT = "0"; - log("Will use GIT_CONFIG env overrides for push."); + pushEnv.GIT_CONFIG_VALUE_0 = helperPath; + pushEnv.GIT_CONFIG_KEY_1 = "credential.useHttpPath"; + pushEnv.GIT_CONFIG_VALUE_1 = "true"; + log("Will use credential helper script for push."); } function gitPush(args) { @@ -301,7 +302,7 @@ function pushOrPr() { cwd: CWD, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"], - env: { ...process.env, ...pushEnv }, + env: pushEnv, }).trim(); } From 5f4737a163093c435bd91952da8d6ec9459a67fe Mon Sep 17 00:00:00 2001 From: WOPR Date: Wed, 25 Mar 2026 21:08:31 -0700 Subject: [PATCH 13/17] fix: use temp remote with token URL for push auth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Credential managers intercept the 'origin' remote. Create a fresh temporary remote with token embedded in URL — credential managers have no cached creds for it. Clean up after push. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/upstream-sync.mjs | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/scripts/upstream-sync.mjs b/scripts/upstream-sync.mjs index c783f613a..d553d240c 100644 --- a/scripts/upstream-sync.mjs +++ b/scripts/upstream-sync.mjs @@ -280,30 +280,27 @@ function pushOrPr() { return; } - // Configure git auth — write a tiny credential helper that returns the GH_TOKEN - // This is the simplest approach: no headers, no base64, no quoting issues + // Push using GH_TOKEN embedded directly in the remote URL + // Use a fresh remote to avoid any cached credential interference const ghToken = process.env.GH_TOKEN; - const pushEnv = { ...process.env, GIT_TERMINAL_PROMPT: "0" }; - if (ghToken) { - // Write a one-shot credential helper script - const helperPath = join("/tmp", `git-cred-helper-${Date.now()}.sh`); - writeFileSync(helperPath, `#!/bin/sh\necho "username=x-access-token"\necho "password=${ghToken}"\n`, { mode: 0o700 }); - // GIT_CONFIG_COUNT overrides all config levels including system - pushEnv.GIT_CONFIG_COUNT = "2"; - pushEnv.GIT_CONFIG_KEY_0 = "credential.helper"; - pushEnv.GIT_CONFIG_VALUE_0 = helperPath; - pushEnv.GIT_CONFIG_KEY_1 = "credential.useHttpPath"; - pushEnv.GIT_CONFIG_VALUE_1 = "true"; - log("Will use credential helper script for push."); - } function gitPush(args) { - return execSync(`git ${args}`, { - cwd: CWD, - encoding: "utf-8", - stdio: ["pipe", "pipe", "pipe"], - env: pushEnv, - }).trim(); + if (!ghToken) return run(`git ${args}`); + // Add a temp remote with token in URL — bypasses all credential helpers entirely + const tmpRemote = `_push_${Date.now()}`; + const tokenUrl = `https://x-access-token:${ghToken}@github.com/wopr-network/nemoclaw.git`; + try { + run(`git remote add ${tmpRemote} ${tokenUrl}`); + const result = execSync(`git ${args.replace("origin", tmpRemote)}`, { + cwd: CWD, + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_ASKPASS: "/bin/echo" }, + }).trim(); + return result; + } finally { + tryRun(`git remote remove ${tmpRemote}`); + } } if (AUTO_PUSH) { From 9fc5d8c7d2d290c8a43258c2685ca3db437029b6 Mon Sep 17 00:00:00 2001 From: WOPR Date: Wed, 25 Mar 2026 21:13:03 -0700 Subject: [PATCH 14/17] Revert "feat: add :staging tag and VPS deploy to CI" This reverts commit 9d5a0ceb8e850107cdfcd27057f266866cd34ac9. --- .github/workflows/ci.yml | 45 ---------------------------------------- 1 file changed, 45 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d47927da0..3631a852c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,51 +33,6 @@ jobs: push: ${{ github.ref == 'refs/heads/main' }} tags: | ghcr.io/${{ github.repository }}:latest - ghcr.io/${{ github.repository }}:staging ghcr.io/${{ github.repository }}:${{ github.sha }} cache-from: type=gha cache-to: type=gha,mode=max - - deploy-staging: - if: github.ref == 'refs/heads/main' - runs-on: [self-hosted, Linux, X64] - needs: build-and-push - steps: - - name: Deploy to staging VPS - uses: appleboy/ssh-action@v1 - with: - host: ${{ secrets.STAGING_HOST }} - username: root - key: ${{ secrets.PROD_SSH_KEY }} - script: | - set -euo pipefail - cd /opt/nemoclaw-platform - COMPOSE="docker compose -f docker-compose.yml -f docker-compose.staging.yml" - $COMPOSE stop staging-api 2>/dev/null || true - $COMPOSE pull staging-api - $COMPOSE up -d staging-postgres - sleep 3 - $COMPOSE exec -T staging-postgres psql -U nemoclaw -d nemoclaw_platform -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;" 2>/dev/null || true - docker compose exec -T postgres pg_dump -U nemoclaw --no-owner --no-acl nemoclaw_platform | \ - $COMPOSE exec -T staging-postgres psql -U nemoclaw -d nemoclaw_platform -q - $COMPOSE up -d staging-api staging-ui - docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile 2>/dev/null || docker compose restart caddy - echo "Staging deployed" - - - name: Verify staging health - uses: appleboy/ssh-action@v1 - with: - host: ${{ secrets.STAGING_HOST }} - username: root - key: ${{ secrets.PROD_SSH_KEY }} - script: | - for i in $(seq 1 30); do - if docker exec nemoclaw-platform-staging-api-1 curl -sf http://localhost:3100/health >/dev/null 2>&1; then - echo "Staging API is healthy" - exit 0 - fi - sleep 2 - done - echo "Health check timed out" - docker logs nemoclaw-platform-staging-api-1 --tail 20 - exit 1 From 413f1812e1e84f153e5bfb9e44fc0a981fd0734c Mon Sep 17 00:00:00 2001 From: WOPR Date: Wed, 25 Mar 2026 21:14:53 -0700 Subject: [PATCH 15/17] debug: dump credential config + try GH_TOKEN env for gh credential helper Need to see what's intercepting git push on the self-hosted runner. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/upstream-sync.yml | 8 ++++---- scripts/upstream-sync.mjs | 30 ++++++++++++++--------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/upstream-sync.yml b/.github/workflows/upstream-sync.yml index 7284107ba..e7cd6c0e7 100644 --- a/.github/workflows/upstream-sync.yml +++ b/.github/workflows/upstream-sync.yml @@ -56,10 +56,10 @@ jobs: - name: Install Agent SDK run: | - npm cache clean --force 2>/dev/null || true - rm -rf "$(npm root -g)/@anthropic-ai/claude-code" "$(npm root -g)/@anthropic-ai/claude-agent-sdk" 2>/dev/null || true - npm install -g @anthropic-ai/claude-code - npm install -g @anthropic-ai/claude-agent-sdk + rm -rf "$(npm root -g)/@anthropic-ai/.claude-code-"* "$(npm root -g)/@anthropic-ai/.claude-agent-sdk-"* 2>/dev/null || true + npm install -g @anthropic-ai/claude-code || npm install -g @anthropic-ai/claude-code + npm install -g @anthropic-ai/claude-agent-sdk || npm install -g @anthropic-ai/claude-agent-sdk + node -e "require('@anthropic-ai/claude-agent-sdk')" 2>/dev/null || echo "SDK import check: will resolve at runtime via global root" - name: Check for upstream changes id: check diff --git a/scripts/upstream-sync.mjs b/scripts/upstream-sync.mjs index d553d240c..ba2146405 100644 --- a/scripts/upstream-sync.mjs +++ b/scripts/upstream-sync.mjs @@ -286,21 +286,21 @@ function pushOrPr() { function gitPush(args) { if (!ghToken) return run(`git ${args}`); - // Add a temp remote with token in URL — bypasses all credential helpers entirely - const tmpRemote = `_push_${Date.now()}`; - const tokenUrl = `https://x-access-token:${ghToken}@github.com/wopr-network/nemoclaw.git`; - try { - run(`git remote add ${tmpRemote} ${tokenUrl}`); - const result = execSync(`git ${args.replace("origin", tmpRemote)}`, { - cwd: CWD, - encoding: "utf-8", - stdio: ["pipe", "pipe", "pipe"], - env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_ASKPASS: "/bin/echo" }, - }).trim(); - return result; - } finally { - tryRun(`git remote remove ${tmpRemote}`); - } + // Debug: dump what credential helpers are active + log("DEBUG credential.helper: " + tryRun("git config --show-origin --get-all credential.helper").output); + log("DEBUG GH_PAT first 8: " + ghToken.slice(0, 8) + "..."); + log("DEBUG git credential-manager version: " + tryRun("git-credential-manager --version").output); + log("DEBUG gh auth status: " + tryRun("gh auth status").output); + // Use gh CLI's built-in git credential helper instead of fighting it + // gh auth setup-git registers gh as credential.helper for github.com + // So just make sure GH_TOKEN env is set — gh respects it for auth + const result = execSync(`git ${args}`, { + cwd: CWD, + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + env: { ...process.env, GH_TOKEN: ghToken, GITHUB_TOKEN: ghToken, GIT_TERMINAL_PROMPT: "0" }, + }).trim(); + return result; } if (AUTO_PUSH) { From 58512c5998e62706a754a03fca13ac577d6f7e08 Mon Sep 17 00:00:00 2001 From: WOPR Date: Wed, 25 Mar 2026 21:34:20 -0700 Subject: [PATCH 16/17] fix: target fork repo for PR, remove debug logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh pr create defaults to parent (NVIDIA/NemoClaw) on forks. Add --repo wopr-network/nemoclaw to target our fork. Also clean up debug output from credential investigation. Note: accidental PR #959 was opened on NVIDIA/NemoClaw — needs manual close. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/upstream-sync.mjs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/scripts/upstream-sync.mjs b/scripts/upstream-sync.mjs index ba2146405..76d05441f 100644 --- a/scripts/upstream-sync.mjs +++ b/scripts/upstream-sync.mjs @@ -286,14 +286,7 @@ function pushOrPr() { function gitPush(args) { if (!ghToken) return run(`git ${args}`); - // Debug: dump what credential helpers are active - log("DEBUG credential.helper: " + tryRun("git config --show-origin --get-all credential.helper").output); - log("DEBUG GH_PAT first 8: " + ghToken.slice(0, 8) + "..."); - log("DEBUG git credential-manager version: " + tryRun("git-credential-manager --version").output); - log("DEBUG gh auth status: " + tryRun("gh auth status").output); - // Use gh CLI's built-in git credential helper instead of fighting it - // gh auth setup-git registers gh as credential.helper for github.com - // So just make sure GH_TOKEN env is set — gh respects it for auth + // gh on the runner picks up GH_TOKEN env for auth — just pass it through const result = execSync(`git ${args}`, { cwd: CWD, encoding: "utf-8", @@ -331,7 +324,7 @@ function pushOrPr() { ].join("\n"); const pr = tryRun( - `gh pr create --title "sync: rebase on upstream (${datestamp})" --body "${prBody.replace(/"/g, '\\"')}" --base main`, + `gh pr create --repo wopr-network/nemoclaw --title "sync: rebase on upstream (${datestamp})" --body "${prBody.replace(/"/g, '\\"')}" --base main`, ); if (pr.ok) { log(`PR created: ${pr.output}`); From a86248cfa0b8fc4585ad101ce85a1402bbd34297 Mon Sep 17 00:00:00 2001 From: WOPR Date: Wed, 25 Mar 2026 22:00:12 -0700 Subject: [PATCH 17/17] fix: write Claude credentials file for Agent SDK auth Agent SDK reads ~/.claude/.credentials.json, not env vars. Write CLAUDE_CREDENTIALS_JSON org secret to disk before sync. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/upstream-sync.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/upstream-sync.yml b/.github/workflows/upstream-sync.yml index e7cd6c0e7..be8e6bd24 100644 --- a/.github/workflows/upstream-sync.yml +++ b/.github/workflows/upstream-sync.yml @@ -61,6 +61,11 @@ jobs: npm install -g @anthropic-ai/claude-agent-sdk || npm install -g @anthropic-ai/claude-agent-sdk node -e "require('@anthropic-ai/claude-agent-sdk')" 2>/dev/null || echo "SDK import check: will resolve at runtime via global root" + - name: Write Claude credentials + run: | + mkdir -p ~/.claude + echo '${{ secrets.CLAUDE_CREDENTIALS_JSON }}' > ~/.claude/.credentials.json + - name: Check for upstream changes id: check run: |