GH#34286: Explain tag-push release evidence failures

.agents/reference/ci-gate-policy.md

@@ -163,7 +163,10 @@ run those directly too, without `-k` (they do not provide unittest filtering).
     not rerun it, change code, file an issue or wait for it. Merge when the
     other checks pass, and note the skipped check in the PR body. A qlty
     failure with real findings still counts, and an out-of-credits failure on
-    a public personal-account repository is unexpected: report it.
+    a public personal-account repository is unexpected: report it. The free
+    minutes return monthly, so keep the app installed (it is not a reason to
+    remove Qlty) and never make "qlty check" a required check on a limited
+    repository. Setup guidance: `tools/code-review/qlty.md`.
 
 ## Private repositories and public launch
 

.agents/scripts/full-loop-helper-state-lifecycle-commands.sh

@@ -760,6 +760,29 @@ _full_loop_resolve_remote_release_tag_commit() {
 	return 0
 }
 
+_full_loop_report_release_run_events() {
+	local repo="$1"
+	local workflow_file="$2"
+	local merge_commit="$3"
+	local tag_name="$4"
+	local runs_json=""
+	local events=""
+	# Diagnostics only: never substitute these runs for publication evidence.
+	runs_json=$(gh api "repos/${repo}/actions/workflows/${workflow_file}/runs?head_sha=${merge_commit}&status=success&per_page=100" 2>/dev/null) || return 0
+	events=$(jq -r --arg sha "$merge_commit" '
+		[.workflow_runs[]? | select(.head_sha == $sha and .status == "completed" and .conclusion == "success")
+		| .event | select(type == "string")] | unique | join(", ")
+	' <<<"$runs_json" 2>/dev/null) || return 0
+	print_error "Successful events of workflow ${workflow_file} for ${merge_commit}: ${events:-none}"
+	if jq -e --arg sha "$merge_commit" --arg tag "$tag_name" '
+		any(.workflow_runs[]?; .head_sha == $sha and .head_branch == $tag and
+			.event == "push" and .status == "completed" and .conclusion == "success")
+	' <<<"$runs_json" >/dev/null 2>&1; then
+		print_error "A successful tag-push run exists for ${tag_name}; retry with --workflow ${workflow_file} --event push"
+	fi
+	return 0
+}
+
 _full_loop_verify_published_release() {
 	local repo="$1"
 	local tag_name="$2"
@@ -835,6 +858,9 @@ _full_loop_verify_published_release() {
 		)] | length > 0
 	' <<<"$workflow_runs_json" >/dev/null || {
 		print_error "No successful ${workflow_event} run${workflow_file:+ of workflow ${workflow_file}} found for ${merge_commit}"
+		if [[ -n "$workflow_file" && "$workflow_event" == "$_FULL_LOOP_WORKFLOW_EVENT_RELEASE" ]]; then
+			_full_loop_report_release_run_events "$repo" "$workflow_file" "$merge_commit" "$tag_name"
+		fi
 		return 1
 	}
 	return 0

.agents/scripts/pulse-dispatch-engine.sh

@@ -1185,13 +1185,157 @@ _pulse_run_blocker_refresh_catchup() {
 	return 0
 }
 
+#######################################
+# Return the post-dispatch housekeeping stage-progress state file path.
+#
+# Lives next to (not inside) the lock directory so a stale-lock reclaim, which
+# removes the lock directory, keeps the interrupted run's progress (GH#34287).
+#
+# Returns 0 always; emits path on stdout.
+#######################################
+_pulse_post_dispatch_housekeeping_statefile() {
+	printf '%s\n' "${HOME}/.aidevops/logs/pulse-post-dispatch-housekeeping.state"
+	return 0
+}
+
+#######################################
+# Return 0 when a housekeeping stage completed within the resume window.
+#
+# The state file holds `stage=epoch` lines written after each stage finishes
+# and is removed by a clean, complete run. Fresh entries therefore only exist
+# after an interrupted run (deploy reconciliation SIGTERM, crash, reboot).
+# AIDEVOPS_PULSE_HOUSEKEEPING_RESUME_WINDOW_S=0 disables resume.
+#
+# Args:
+#   $1 - stage name
+#   $2 - state file path
+# Returns 0 if recently completed, 1 otherwise; emits age seconds on stdout.
+#######################################
+_pulse_housekeeping_stage_recently_done() {
+	local stage_name="$1"
+	local state_file="$2"
+	local window="${AIDEVOPS_PULSE_HOUSEKEEPING_RESUME_WINDOW_S:-1800}"
+	[[ "$window" =~ ^[0-9]+$ ]] || window=1800
+	[[ "$window" -gt 0 && -f "$state_file" ]] || return 1
+
+	local line="" done_epoch=""
+	while IFS= read -r line || [[ -n "$line" ]]; do
+		if [[ "$line" == "${stage_name}="* ]]; then
+			done_epoch="${line#*=}"
+		fi
+	done <"$state_file"
+	[[ "$done_epoch" =~ ^[0-9]+$ ]] || return 1
+
+	local now=""
+	now=$(date +%s 2>/dev/null) || return 1
+	[[ "$now" =~ ^[0-9]+$ ]] || return 1
+	local age=$((now - done_epoch))
+	[[ "$age" -ge 0 && "$age" -lt "$window" ]] || return 1
+	printf '%s\n' "$age"
+	return 0
+}
+
+#######################################
+# Record a finished housekeeping stage in the state file (stage=epoch).
+#
+# Args:
+#   $1 - stage name
+#   $2 - state file path
+# Returns 0 always (progress persistence is best-effort).
+#######################################
+_pulse_housekeeping_mark_stage_done() {
+	local stage_name="$1"
+	local state_file="$2"
+	local now=""
+	now=$(date +%s 2>/dev/null) || return 0
+	[[ "$now" =~ ^[0-9]+$ ]] || return 0
+
+	local tmp_file="${state_file}.tmp.${BASHPID:-$$}"
+	local line=""
+	{
+		if [[ -f "$state_file" ]]; then
+			while IFS= read -r line || [[ -n "$line" ]]; do
+				[[ -n "$line" && "$line" != "${stage_name}="* ]] && printf '%s\n' "$line"
+			done <"$state_file"
+		fi
+		printf '%s=%s\n' "$stage_name" "$now"
+	} >"$tmp_file" 2>/dev/null || {
+		rm -f "$tmp_file" 2>/dev/null || true
+		return 0
+	}
+	mv -f "$tmp_file" "$state_file" 2>/dev/null || rm -f "$tmp_file" 2>/dev/null || true
+	return 0
+}
+
+#######################################
+# Run one resumable housekeeping stage.
+#
+# Skips the stage when an interrupted earlier run already completed it within
+# the resume window; otherwise records it as the current stage (for the
+# signal trap), runs it, and persists its completion.
+#
+# Args:
+#   $1 - stage name (state key)
+#   $2 - state file path
+#   $@ - stage command and arguments
+# Returns 0 always.
+#######################################
+_pulse_run_resumable_housekeeping_stage() {
+	local stage_name="$1"
+	local state_file="$2"
+	shift 2
+	local age=""
+	if age=$(_pulse_housekeeping_stage_recently_done "$stage_name" "$state_file"); then
+		echo "[pulse-wrapper] Async post-dispatch housekeeping: resume — skipping ${stage_name} (completed ${age}s ago by an interrupted run)" >>"$LOGFILE"
+		return 0
+	fi
+	_PULSE_HOUSEKEEPING_CURRENT_STAGE="$stage_name"
+	local stage_rc=0
+	"$@" || stage_rc=$?
+	# A stage child killed by a signal (e.g. the same deploy SIGTERM that is
+	# about to reach this subshell) did not finish: leave it for the resume.
+	if [[ "$stage_rc" -gt 128 ]]; then
+		echo "[pulse-wrapper] Async post-dispatch housekeeping: ${stage_name} ended by signal (rc=${stage_rc}) — not recorded as complete" >>"$LOGFILE"
+	else
+		_pulse_housekeeping_mark_stage_done "$stage_name" "$state_file"
+	fi
+	_PULSE_HOUSEKEEPING_CURRENT_STAGE="between_stages"
+	return 0
+}
+
+#######################################
+# Log and exit when the async housekeeping subshell receives a signal.
+#
+# Setup reconciliation SIGTERMs every Pulse runtime process on deploy
+# (intended: old-bundle code must be replaced). Logging the signal and the
+# active stage makes those deaths attributable from pulse.log (GH#34287).
+#
+# Args:
+#   $1 - signal name (TERM or HUP)
+# Exits 143 for TERM, 129 for HUP.
+#######################################
+_pulse_housekeeping_on_signal() {
+	local signal_name="$1"
+	trap - TERM HUP
+	echo "[pulse-wrapper] Async post-dispatch housekeeping: terminated by SIG${signal_name} during ${_PULSE_HOUSEKEEPING_CURRENT_STAGE:-unknown}" >>"$LOGFILE"
+	if [[ "$signal_name" == "HUP" ]]; then
+		exit 129
+	fi
+	exit 143
+}
+
 #######################################
 # Run non-dispatch post-dispatch housekeeping stages.
 #
 # These stages are intentionally after early dispatch and do not protect the
 # immediate worker claim/ledger safety boundary. They can therefore run under a
 # separate lock while the main pulse proceeds to prefetch + the next refill.
 #
+# GH#34287: deploy reconciliation kills long runs, so per-stage completion is
+# persisted and a following run resumes at the first incomplete stage. The
+# cheap terminal reconciles (preflight_ownership_reconcile) run before the
+# slow GH#33246 blocker-refresh catch-up so they are not starved behind it.
+#
 # Args:
 #   $1 - per-stage timeout seconds
 # Returns 0 always.
@@ -1200,22 +1344,34 @@ _pulse_run_post_dispatch_housekeeping_stages() {
 	local stage_timeout="${1:-${PREFLIGHT_GROUP_TIMEOUT:-${PRE_RUN_STAGE_TIMEOUT:-600}}}"
 	[[ "$stage_timeout" =~ ^[0-9]+$ ]] || stage_timeout=600
 
-	local lockdir
+	local lockdir="" state_file=""
 	lockdir="$(_pulse_post_dispatch_housekeeping_lockdir)"
+	state_file="$(_pulse_post_dispatch_housekeeping_statefile)"
 	if ! _pulse_acquire_post_dispatch_housekeeping_lock "$lockdir"; then
 		return 0
 	fi
 
+	_PULSE_HOUSEKEEPING_CURRENT_STAGE="startup"
 	echo "[pulse-wrapper] Async post-dispatch housekeeping: started (timeout=${stage_timeout}s)" >>"$LOGFILE"
-	_pulse_run_optional_stage_with_timeout "coderabbit_review" "$stage_timeout" run_daily_codebase_review || true
-	_pulse_run_optional_stage_with_timeout "post_merge_scanner" "$stage_timeout" _run_post_merge_review_scanner || true
-	_pulse_run_optional_stage_with_timeout "pr_review_thread_response" "$stage_timeout" _run_pr_review_thread_response_scanner || true
-	_pulse_run_optional_stage_with_timeout "auto_decomposer_scanner" "$stage_timeout" _run_auto_decomposer_scanner || true
-	_pulse_run_optional_stage_with_timeout "dedup_cleanup" "$stage_timeout" run_simplification_dedup_cleanup || true
-	_pulse_run_optional_stage_with_timeout "fast_fail_prune_expired" "$stage_timeout" fast_fail_prune_expired || true
-	_pulse_run_blocker_refresh_catchup "$stage_timeout" || true
-	run_stage_with_timeout "preflight_ownership_reconcile" "$stage_timeout" \
-		_preflight_ownership_reconcile "$stage_timeout" || true
+	_pulse_run_resumable_housekeeping_stage "coderabbit_review" "$state_file" \
+		_pulse_run_optional_stage_with_timeout "coderabbit_review" "$stage_timeout" run_daily_codebase_review
+	_pulse_run_resumable_housekeeping_stage "post_merge_scanner" "$state_file" \
+		_pulse_run_optional_stage_with_timeout "post_merge_scanner" "$stage_timeout" _run_post_merge_review_scanner
+	_pulse_run_resumable_housekeeping_stage "pr_review_thread_response" "$state_file" \
+		_pulse_run_optional_stage_with_timeout "pr_review_thread_response" "$stage_timeout" _run_pr_review_thread_response_scanner
+	_pulse_run_resumable_housekeeping_stage "auto_decomposer_scanner" "$state_file" \
+		_pulse_run_optional_stage_with_timeout "auto_decomposer_scanner" "$stage_timeout" _run_auto_decomposer_scanner
+	_pulse_run_resumable_housekeeping_stage "dedup_cleanup" "$state_file" \
+		_pulse_run_optional_stage_with_timeout "dedup_cleanup" "$stage_timeout" run_simplification_dedup_cleanup
+	_pulse_run_resumable_housekeeping_stage "fast_fail_prune_expired" "$state_file" \
+		_pulse_run_optional_stage_with_timeout "fast_fail_prune_expired" "$stage_timeout" fast_fail_prune_expired
+	_pulse_run_resumable_housekeeping_stage "preflight_ownership_reconcile" "$state_file" \
+		run_stage_with_timeout "preflight_ownership_reconcile" "$stage_timeout" \
+		_preflight_ownership_reconcile "$stage_timeout"
+	_pulse_run_resumable_housekeeping_stage "blocker_refresh_catchup" "$state_file" \
+		_pulse_run_blocker_refresh_catchup "$stage_timeout"
+	_PULSE_HOUSEKEEPING_CURRENT_STAGE="finishing"
+	rm -f "$state_file" 2>/dev/null || true
 	echo "[pulse-wrapper] Async post-dispatch housekeeping: complete" >>"$LOGFILE"
 
 	_pulse_release_post_dispatch_housekeeping_lock "$lockdir"
@@ -1251,7 +1407,11 @@ _pulse_start_post_dispatch_housekeeping() {
 	set -m 2>/dev/null || true
 	(
 		set +m 2>/dev/null || true
-		trap - EXIT INT TERM
+		trap - EXIT INT
+		# GH#34287: attribute deploy-reconciliation kills in pulse.log.
+		_PULSE_HOUSEKEEPING_CURRENT_STAGE="launch"
+		trap '_pulse_housekeeping_on_signal TERM' TERM
+		trap '_pulse_housekeeping_on_signal HUP' HUP
 		AIDEVOPS_PULSE_STAGE_CYCLE_CLAMP=0
 		_pulse_run_post_dispatch_housekeeping_stages "$stage_timeout"
 	) >>"$LOGFILE" 2>&1 &

.agents/scripts/pulse-issue-reconcile-actions.sh

@@ -800,6 +800,43 @@ _pir_reopen_unstable_parent_close() {
 	return 0
 }
 
+#######################################
+# Close a parent, verify the post-close snapshot, and compensate by reopening
+# when evidence changed. Args: slug parent_num live_body child_nums child_source
+# Returns: 0=closed and stable, 1=close failed or compensated
+#######################################
+_pir_close_parent_with_postcondition() {
+	local slug="$1" parent_num="$2" live_parent_body="$3" child_nums="$4" child_source="$5"
+	gh issue close "$parent_num" --repo "$slug" >/dev/null 2>&1 || return 1
+	if ! _pir_parent_close_postcondition_is_stable "$slug" "$parent_num" "$live_parent_body" \
+		"$child_nums" "$child_source"; then
+		if [[ "$_PIR_CPT_POST_CLOSE_REOPEN_ALLOWED" -eq 1 ]]; then
+			_pir_reopen_unstable_parent_close "$slug" "$parent_num" || \
+				echo "[pulse-wrapper] Reconcile parent-task: unable to compensate unstable close #${parent_num} in ${slug}" >>"${LOGFILE:-/dev/null}"
+		fi
+		return 1
+	fi
+	return 0
+}
+
+#######################################
+# Budget guard for multi-round-trip mutations (GH#34289). Reads the
+# single-pass locals _t2984_start_ts/_t2984_budget via dynamic scope. A parent
+# close spans several reads, the close, a stability check, an optional reopen
+# and a comment; starting one near the budget risks a mid-action stage kill.
+# Env: RECONCILE_MUTATION_RESERVE_SECS=reserve seconds (default 90)
+# Returns: 0=enough budget (or no budget active), 1=too little remaining
+#######################################
+_pir_budget_allows_mutation() {
+	local reserve="${RECONCILE_MUTATION_RESERVE_SECS:-90}"
+	[[ "$reserve" =~ ^[0-9]+$ ]] || reserve=90
+	local budget="${_t2984_budget:-0}" start="${_t2984_start_ts:-}"
+	[[ "$budget" =~ ^[0-9]+$ && "$start" =~ ^[0-9]+$ ]] || return 0
+	[[ "$budget" -gt 0 ]] || return 0
+	[[ $((budget - (SECONDS - start))) -ge "$reserve" ]] || return 1
+	return 0
+}
+
 #######################################
 # t2138 / t3544: extract per-parent close logic. Keeps
 # reconcile_completed_parent_tasks under the 100-line shell-complexity
@@ -813,6 +850,10 @@ _try_close_parent_tracker() {
 	local live_child_nums="" live_child_source=""
 	local live_parent_revision=""
 
+	if ! _pir_budget_allows_mutation; then
+		echo "[pulse-wrapper] Reconcile parent-task: deferred: insufficient reconcile budget for parent close #${parent_num} in ${slug}" >>"${LOGFILE:-/dev/null}"
+		return 1
+	fi
 	_pir_verify_parent_children_closed "$slug" "$child_nums" || return 1
 	child_count="$_PIR_CPT_VERIFIED_CHILD_COUNT"
 	child_summary="$_PIR_CPT_VERIFIED_CHILD_SUMMARY"
@@ -883,15 +924,8 @@ _try_close_parent_tracker() {
 		return 1
 	fi
 	local close_basis="$_PIR_PARENT_TRUST_BASIS"
-	gh issue close "$parent_num" --repo "$slug" >/dev/null 2>&1 || return 1
-	if ! _pir_parent_close_postcondition_is_stable "$slug" "$parent_num" "$live_parent_body" \
-		"$child_nums" "$child_source"; then
-		if [[ "$_PIR_CPT_POST_CLOSE_REOPEN_ALLOWED" -eq 1 ]]; then
-			_pir_reopen_unstable_parent_close "$slug" "$parent_num" || \
-				echo "[pulse-wrapper] Reconcile parent-task: unable to compensate unstable close #${parent_num} in ${slug}" >>"${LOGFILE:-/dev/null}"
-		fi
-		return 1
-	fi
+	_pir_close_parent_with_postcondition "$slug" "$parent_num" "$live_parent_body" \
+		"$child_nums" "$child_source" || return 1
 	gh_issue_comment "$parent_num" --repo "$slug" \
 		--body "## All declared child tasks completed — closing parent tracker
 

.agents/scripts/tests/test-full-loop-completion-evidence.sh

@@ -67,6 +67,20 @@ if [[ "$*" == *"/actions/runs?event=release"* || "$*" == *"/actions/workflows/re
 	esac
 	exit 0
 fi
+if [[ "$*" == *"/actions/workflows/release.yml/runs?event=release&"* ]]; then
+	printf '%s\n' '{"workflow_runs":[]}'
+	exit 0
+fi
+if [[ "$*" == *"/actions/workflows/release.yml/runs?head_sha="* ]]; then
+	case "$release_mode" in
+	diagnostic-unavailable) exit 70 ;;
+	diagnostic-wrong-sha) merge_sha="2222222222222222222222222222222222222222" ;;
+	esac
+	jq -cn --arg sha "$merge_sha" --arg mode "$release_mode" '{workflow_runs:[{event:"push",status:"completed",
+		conclusion:(if $mode == "diagnostic-failed" then "failure" else "success" end),
+		head_branch:(if $mode == "diagnostic-main" then "main" else "v3.0.0" end),head_sha:$sha}]}'
+	exit 0
+fi
 if [[ "$*" == *"repos/marcusquinn/aidevops/actions/workflows/release.yml/runs?event=push&status=success&per_page=100"* ]]; then
 	case "$release_mode" in
 	push-failed-workflow)
@@ -263,6 +277,24 @@ AIDEVOPS_FULL_LOOP_RECEIPT_DIR="$receipt_dir" AIDEVOPS_FULL_LOOP_CLEANUP_DIR="$c
 cmp -s "$published_cleanup_receipt" "${ROOT}/published-release-receipt-before.json"
 printf 'PASS verified manual release records published evidence and reconciles cleanup idempotently\n'
 
+for diagnostic_mode in valid diagnostic-wrong-sha diagnostic-failed diagnostic-main diagnostic-unavailable; do
+	if diagnostic_output=$(COMPLETION_RELEASE_MODE="$diagnostic_mode" AIDEVOPS_FULL_LOOP_RECEIPT_DIR="$receipt_dir" \
+		PATH="${ROOT}/bin:/opt/homebrew/bin:/usr/bin:/bin" \
+		bash "$published_record_runner" 60 v3.0.0 marcusquinn/aidevops --workflow release.yml 2>&1); then
+		printf 'FAIL diagnostic query recorded publication evidence\n'
+		exit 1
+	fi
+	[[ ! -e "${receipt_dir}/marcusquinn_aidevops-60.status" ]]
+	[[ "$diagnostic_output" == *"No successful release run of workflow release.yml"* ]]
+	if [[ "$diagnostic_mode" == "valid" ]]; then
+		[[ "$diagnostic_output" == *"Successful events of workflow release.yml"*": push"* ]]
+		[[ "$diagnostic_output" == *"--event push"* ]]
+	else
+		[[ "$diagnostic_output" != *"--event push"* ]]
+	fi
+done
+printf 'PASS missing release event suggests matching tag-push evidence without accepting mismatched or unavailable diagnostics\n'
+
 push_published_worktree="${ROOT}/push-published-release-worktree"
 mkdir -p "$push_published_worktree"
 push_published_receipt=$(full_loop_write_cleanup_deferred marcusquinn/aidevops 61 "$push_published_worktree" \