From 97fb3b28a95c303cb2afecf818843c7947cb5f80 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Mon, 17 Aug 2026 19:46:42 +0000
Subject: [PATCH] fix(hpc/sync): create remote project dir before parallel push
 rsyncs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

On the first push to a new remote project, push() launched the CODE_DIRS
rsyncs in parallel before anything had created ${HPC_BASE}/${PROJECT_NAME}.
rsync only creates the final path level, so all of them died with
"mkdir failed: No such file or directory". The later [root files] rsync
then created the base dir, so dataset/ synced and push exited 0 — the
failure stayed silent until sbatch could not find hpc/batch_gpu
(slope_hierarchy first push, job 330464 postmortem, 2026-07-16).

Two changes:

- ssh mkdir -p the remote project dir before the parallel rsyncs, matching
  what pull/pull_logs/push_data_init already do. Skipped under --dry-run so
  `status` stays side-effect free.
- Collect the background PIDs and wait on each one. Plain `wait` returns its
  own status, so a failed backgrounded rsync was swallowed even under
  `set -e`. push() now names the failing directory and returns 1, so
  push-submit aborts instead of submitting against an incomplete tree.

Verified against a local fake-HPC harness (real rsync, stubbed ssh):
first push previously exited 0 without hpc/batch_gpu and now transfers the
full tree; an injected transfer failure now exits 1 and stops push-submit
before sbatch; re-push, --no-data and status are unchanged.
---
 hpc/sync | 44 ++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 42 insertions(+), 2 deletions(-)

diff --git a/hpc/sync b/hpc/sync
index 1560e64..85ec334 100644
--- a/hpc/sync
+++ b/hpc/sync
@@ -102,8 +102,18 @@ EXCLUDES=(
 # ---------------------------------------------------------------------------
 push() {
     local dryrun="${1:-}"
+    local failed=0
+
+    # On a first push the remote project directory does not exist yet. rsync
+    # only creates the last path level, so every CODE_DIRS transfer below would
+    # fail with "mkdir failed: No such file or directory" — and because they run
+    # in parallel, the failures are invisible unless collected (see wait loop).
+    if [[ -z "$dryrun" ]]; then
+        ssh "${HPC_HOST}" "mkdir -p '${HPC_BASE}/${PROJECT_NAME}'"
+    fi
 
     echo "==> [push] code/config  →  ${REMOTE}"
+    local pids=() names=()
     for dir in "${CODE_DIRS[@]}"; do
         local src="${PROJECT_ROOT}/${dir}/"
         if [[ ! -d "$src" ]]; then
@@ -113,8 +123,19 @@ push() {
         echo "    ${dir}/"
         "${RSYNC[@]}" "${EXCLUDES[@]}" ${dryrun} \
             "${src}" "${REMOTE}/${dir}/" &
+        pids+=("$!")
+        names+=("${dir}")
     done
-    wait
+    # Plain `wait` reports only its own status, so a backgrounded rsync that
+    # failed would be swallowed even under `set -e`. Wait on each PID instead.
+    if (( ${#pids[@]} > 0 )); then
+        for i in "${!pids[@]}"; do
+            if ! wait "${pids[$i]}"; then
+                echo "    ERROR: rsync of ${names[$i]}/ failed" >&2
+                failed=1
+            fi
+        done
+    fi
 
     echo "    [root files]"
     local existing_root_files=()
@@ -129,6 +150,7 @@ push() {
     if [[ -z "${SKIP_DATA:-}" ]]; then
         echo ""
         echo "==> [push] data  →  ${REMOTE}  (--ignore-existing: skips files already on HPC)"
+        local data_pids=() data_names=()
         for dir in "${DATA_DIRS[@]}"; do
             local src="${PROJECT_ROOT}/${dir}/"
             if [[ ! -d "$src" ]]; then
@@ -138,12 +160,30 @@ push() {
             echo "    ${dir}/"
             "${RSYNC[@]}" "${EXCLUDES[@]}" --ignore-existing ${dryrun} \
                 "${src}" "${REMOTE}/${dir}/" &
+            data_pids+=("$!")
+            data_names+=("${dir}")
         done
-        wait
+        if (( ${#data_pids[@]} > 0 )); then
+            for i in "${!data_pids[@]}"; do
+                if ! wait "${data_pids[$i]}"; then
+                    echo "    ERROR: rsync of ${data_names[$i]}/ failed" >&2
+                    failed=1
+                fi
+            done
+        fi
     else
         echo ""
         echo "    [data skipped — --no-data]"
     fi
+
+    # Fail loudly rather than letting push-submit sbatch against a remote that
+    # is missing hpc/batch_gpu (job 330464 postmortem, 2026-07-16).
+    if (( failed )); then
+        echo "" >&2
+        echo "==> [push] FAILED — one or more transfers did not complete." >&2
+        echo "    The remote project tree is incomplete; do not submit jobs." >&2
+        return 1
+    fi
 }
 
 push_data_init() {