Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
246 changes: 246 additions & 0 deletions internal/branchsync/recover_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1494,6 +1494,252 @@ func (f *recoverFixture) localAnchorRef() string {
return "refs/no-mistakes/recover-local/" + f.run.ID
}

func newTerminalEqualTreeRewriteFixture(t *testing.T) (*recoverFixture, string) {
t.Helper()
f := newRebasedRecoverFixture(t, types.RunFailed)
pipeline := filepath.Join(filepath.Dir(f.local), "terminal-rewrite")
mustRun(t, filepath.Dir(f.local), "-c", "core.autocrlf=false", "clone", f.gate, pipeline)
configureIdentity(t, pipeline)
mustRun(t, pipeline, "checkout", "--detach", f.submitted)
mustWrite(t, filepath.Join(pipeline, "feature.txt"), "pipeline reviewed replacement\n")
mustRun(t, pipeline, "commit", "-am", "reviewed pipeline result")
recorded := mustRun(t, pipeline, "rev-parse", "HEAD")
tree := mustRun(t, pipeline, "rev-parse", recorded+"^{tree}")
live := mustRun(t, pipeline, "commit-tree", tree, "-p", "origin/main", "-m", "terminal equal-tree rewrite")
mustRun(t, pipeline, "push", "origin", recorded+":refs/no-mistakes/test-recorded")
mustRun(t, pipeline, "push", "--force", "origin", live+":refs/heads/feature/recover")
mustRun(t, f.gate, "update-ref", "-d", "refs/no-mistakes/test-recorded")
if err := f.db.UpdateRunHeadSHA(f.run.ID, recorded); err != nil {
t.Fatal(err)
}
if err := f.db.UpdateRunReviewApprovedHeadSHA(f.run.ID, recorded); err != nil {
t.Fatal(err)
}
if err := f.db.UpdateRunErrorStatus(f.run.ID, "terminal worker lost", types.RunFailed); err != nil {
t.Fatal(err)
}
f.preserved = live
f.run, _ = f.db.GetRun(f.run.ID)
return f, recorded
}

func TestRecoverTerminalUnverifiedEqualTreeRewriteAdoptsLiveGateHead(t *testing.T) {
t.Parallel()

f, recorded := newTerminalEqualTreeRewriteFixture(t)
if f.run.TerminalHeadVerifiedAt != nil {
t.Fatal("fixture terminal head is already verified")
}
if isAncestor(f.ctx, f.gate, recorded, f.preserved) || isAncestor(f.ctx, f.gate, f.preserved, recorded) {
t.Fatal("fixture heads are not non-ancestral")
}
if got, want := mustRun(t, f.gate, "rev-parse", recorded+"^{tree}"), mustRun(t, f.gate, "rev-parse", f.preserved+"^{tree}"); got != want {
t.Fatalf("fixture trees differ: %s != %s", got, want)
}
if !isAncestor(f.ctx, f.gate, f.submitted, recorded) {
t.Fatal("recorded reviewed head does not descend from submitted local head")
}
if preservedContainsLocalWork(f.ctx, f.gate, f.submitted, f.preserved) {
t.Fatal("fixture does not reproduce the direct live-head containment conflict")
}
if f.run.ReviewApprovedHeadSHA == nil || *f.run.ReviewApprovedHeadSHA != recorded {
t.Fatalf("review authority = %#v, want %s", f.run.ReviewApprovedHeadSHA, recorded)
}
runsBefore := len(mustRuns(t, f.db, f.repo.ID))
remoteBefore := mustRun(t, f.remote, "for-each-ref", "--format=%(refname) %(objectname)", "refs/heads")

state := f.service.Recover(f.ctx, false)
if !state.Recovered || !state.Changed || state.State != StateCustodyReturned {
t.Fatalf("equal-tree terminal recovery = %#v", state)
}
if got := mustRun(t, f.local, "rev-parse", "HEAD"); got != f.preserved {
t.Fatalf("HEAD = %s, want live gate head %s", got, f.preserved)
}
if got := mustRun(t, f.gate, "rev-parse", f.anchorRef()); got != f.preserved {
t.Fatalf("gate recovery anchor = %s, want %s", got, f.preserved)
}
if got := mustRun(t, f.local, "rev-parse", f.anchorRef()); got != f.preserved {
t.Fatalf("local recovery anchor = %s, want %s", got, f.preserved)
}
if got := mustRun(t, f.local, "rev-parse", f.localAnchorRef()); got != f.submitted {
t.Fatalf("local pre-recovery anchor = %s, want %s", got, f.submitted)
}
run, err := f.db.GetRun(f.run.ID)
if err != nil || run.HeadSHA != f.preserved || run.TerminalHeadVerifiedAt == nil || run.CustodyReturnedAt == nil || run.UpdatedAt < f.run.UpdatedAt {
t.Fatalf("recovered run = %#v, err %v", run, err)
}
if clean, reason := worktreeClean(f.ctx, f.local); !clean {
t.Fatalf("worktree not clean after recovery: %s", reason)
}
if second := f.service.Recover(f.ctx, false); !second.Recovered || second.Changed {
t.Fatalf("repeated recovery = %#v", second)
}
if got := len(mustRuns(t, f.db, f.repo.ID)); got != runsBefore {
t.Fatalf("recovery changed run count from %d to %d", runsBefore, got)
}
if got := mustRun(t, f.remote, "for-each-ref", "--format=%(refname) %(objectname)", "refs/heads"); got != remoteBefore {
t.Fatalf("recovery pushed upstream refs: %q != %q", got, remoteBefore)
}
}

func mustRuns(t *testing.T, database *db.DB, repoID string) []*db.Run {
t.Helper()
runs, err := database.GetRunsByRepo(repoID)
if err != nil {
t.Fatal(err)
}
return runs
}

func pushTerminalRewrite(t *testing.T, f *recoverFixture, mutate func(string)) string {
t.Helper()
pipeline := filepath.Join(t.TempDir(), "rewrite")
mustRun(t, filepath.Dir(pipeline), "-c", "core.autocrlf=false", "clone", f.gate, pipeline)
configureIdentity(t, pipeline)
mustRun(t, pipeline, "checkout", "feature/recover")
mutate(pipeline)
mustRun(t, pipeline, "add", "-A")
mustRun(t, pipeline, "commit", "-m", "terminal changed-tree rewrite")
live := mustRun(t, pipeline, "rev-parse", "HEAD")
mustRun(t, pipeline, "push", "--force", "origin", "HEAD:refs/heads/feature/recover")
f.preserved = live
return live
}

func assertUnverifiedRecoveryRefusalNoMutation(t *testing.T, f *recoverFixture) State {
t.Helper()
localRefs := mustRun(t, f.local, "for-each-ref", "--format=%(refname) %(objectname)", "refs/no-mistakes")
gateRefs := mustRun(t, f.gate, "for-each-ref", "--format=%(refname) %(objectname)", "refs/no-mistakes")
runBefore, err := f.db.GetRun(f.run.ID)
if err != nil {
t.Fatal(err)
}
state := f.service.Recover(f.ctx, false)
if state.Recovered || state.Changed {
t.Fatalf("unsafe recovery succeeded: %#v", state)
}
if got := mustRun(t, f.local, "for-each-ref", "--format=%(refname) %(objectname)", "refs/no-mistakes"); got != localRefs {
t.Fatalf("refusal changed local recovery refs: %q != %q", got, localRefs)
}
if got := mustRun(t, f.gate, "for-each-ref", "--format=%(refname) %(objectname)", "refs/no-mistakes"); got != gateRefs {
t.Fatalf("refusal changed gate recovery refs: %q != %q", got, gateRefs)
}
runAfter, err := f.db.GetRun(f.run.ID)
if err != nil || runAfter.HeadSHA != runBefore.HeadSHA || ptr(runAfter.ReviewApprovedHeadSHA) != ptr(runBefore.ReviewApprovedHeadSHA) || runAfter.TerminalHeadVerifiedAt != nil || runAfter.CustodyReturnedAt != nil || runAfter.UpdatedAt != runBefore.UpdatedAt {
t.Fatalf("refusal changed run: before %#v after %#v err %v", runBefore, runAfter, err)
}
return state
}

func TestRecoverTerminalUnverifiedRewriteNegativeControls(t *testing.T) {
t.Parallel()

t.Run("different tree", func(t *testing.T) {
f, _ := newTerminalEqualTreeRewriteFixture(t)
pushTerminalRewrite(t, f, func(dir string) { mustWrite(t, filepath.Join(dir, "pipeline.txt"), "different tree\n") })
if state := assertUnverifiedRecoveryRefusalNoMutation(t, f); state.Safety != "blocked_recover_unverified_head" {
t.Fatalf("different-tree refusal = %s", state.Safety)
}
if got := mustRun(t, f.local, "rev-parse", "HEAD"); got != f.submitted {
t.Fatalf("different-tree refusal moved HEAD to %s", got)
}
})
t.Run("drops local hunk", func(t *testing.T) {
f, _ := newTerminalEqualTreeRewriteFixture(t)
mustRun(t, f.gate, "update-ref", "refs/heads/feature/recover", mustRun(t, f.gate, "rev-parse", "refs/heads/main"))
if state := assertUnverifiedRecoveryRefusalNoMutation(t, f); state.Safety != "blocked_recover_unverified_head" {
t.Fatalf("dropped-work refusal = %s", state.Safety)
}
})
t.Run("rewrites operator line", func(t *testing.T) {
f, _ := newTerminalEqualTreeRewriteFixture(t)
pushTerminalRewrite(t, f, func(dir string) { mustWrite(t, filepath.Join(dir, "feature.txt"), "pipeline replacement\n") })
if state := assertUnverifiedRecoveryRefusalNoMutation(t, f); state.Safety != "blocked_recover_unverified_head" {
t.Fatalf("rewritten-line refusal = %s", state.Safety)
}
})
t.Run("conflicting create-only anchor", func(t *testing.T) {
f, recorded := newTerminalEqualTreeRewriteFixture(t)
mustRun(t, f.gate, "update-ref", f.anchorRef(), recorded)
if state := assertUnverifiedRecoveryRefusalNoMutation(t, f); state.Safety != "blocked_recover_anchor_mismatch" {
t.Fatalf("anchor-conflict refusal = %s", state.Safety)
}
})
t.Run("untrusted recorded head", func(t *testing.T) {
f, _ := newTerminalEqualTreeRewriteFixture(t)
if err := f.db.UpdateRunReviewApprovedHeadSHA(f.run.ID, f.submitted); err != nil {
t.Fatal(err)
}
if state := assertUnverifiedRecoveryRefusalNoMutation(t, f); state.Safety != "blocked_recover_unverified_head" {
t.Fatalf("untrusted-head refusal = %s", state.Safety)
}
})
t.Run("recorded head does not preserve local", func(t *testing.T) {
f, _ := newTerminalEqualTreeRewriteFixture(t)
pipeline := filepath.Join(t.TempDir(), "uncontained-recorded")
mustRun(t, filepath.Dir(pipeline), "-c", "core.autocrlf=false", "clone", f.gate, pipeline)
configureIdentity(t, pipeline)
mustRun(t, pipeline, "checkout", "--detach", "origin/main")
mustWrite(t, filepath.Join(pipeline, "feature.txt"), "recorded without local work\n")
mustRun(t, pipeline, "add", "feature.txt")
mustRun(t, pipeline, "commit", "-m", "uncontained recorded result")
recorded := mustRun(t, pipeline, "rev-parse", "HEAD")
tree := mustRun(t, pipeline, "rev-parse", "HEAD^{tree}")
live := mustRun(t, pipeline, "commit-tree", tree, "-p", "origin/main", "-m", "uncontained live rewrite")
mustRun(t, pipeline, "push", "origin", recorded+":refs/no-mistakes/test-recorded")
mustRun(t, pipeline, "push", "--force", "origin", live+":refs/heads/feature/recover")
mustRun(t, f.gate, "update-ref", "-d", "refs/no-mistakes/test-recorded")
if err := f.db.UpdateRunHeadSHA(f.run.ID, recorded); err != nil {
t.Fatal(err)
}
if err := f.db.UpdateRunReviewApprovedHeadSHA(f.run.ID, recorded); err != nil {
t.Fatal(err)
}
if state := assertUnverifiedRecoveryRefusalNoMutation(t, f); state.Safety != "blocked_recover_unverified_head" {
t.Fatalf("uncontained-recorded refusal = %s", state.Safety)
}
})
t.Run("symbolic live branch", func(t *testing.T) {
f, _ := newTerminalEqualTreeRewriteFixture(t)
mustRun(t, f.gate, "update-ref", "refs/no-mistakes/symbolic-live", f.preserved)
mustRun(t, f.gate, "symbolic-ref", "refs/heads/feature/recover", "refs/no-mistakes/symbolic-live")
if state := assertUnverifiedRecoveryRefusalNoMutation(t, f); state.Safety != "blocked_recover_unverified_head" {
t.Fatalf("symbolic-live refusal = %s", state.Safety)
}
})
t.Run("active run", func(t *testing.T) {
f, _ := newTerminalEqualTreeRewriteFixture(t)
if err := f.db.UpdateRunStatus(f.run.ID, types.RunRunning); err != nil {
t.Fatal(err)
}
if state := assertUnverifiedRecoveryRefusalNoMutation(t, f); state.Safety != "blocked_recover_run_active" {
t.Fatalf("active-run refusal = %s", state.Safety)
}
})
t.Run("dirty worktree", func(t *testing.T) {
f, _ := newTerminalEqualTreeRewriteFixture(t)
mustWrite(t, filepath.Join(f.local, "feature.txt"), "dirty local edit\n")
if state := assertUnverifiedRecoveryRefusalNoMutation(t, f); state.Safety != "blocked_recover_unverified_head" {
t.Fatalf("dirty-worktree refusal = %s", state.Safety)
}
if got := readOptional(t, filepath.Join(f.local, "feature.txt")); got != "dirty local edit\n" {
t.Fatalf("dirty worktree changed: %q", got)
}
})
t.Run("racing local branch", func(t *testing.T) {
f, _ := newTerminalEqualTreeRewriteFixture(t)
f.service.beforeRecoverTerminalHeadPreserve = func() {
mustRun(t, f.local, "checkout", "-b", "racing-branch", f.submitted)
}
if state := assertUnverifiedRecoveryRefusalNoMutation(t, f); state.Safety != "blocked_recover_assumptions_changed" {
t.Fatalf("branch-race refusal = %s", state.Safety)
}
if got := mustRun(t, f.local, "branch", "--show-current"); got != "racing-branch" {
t.Fatalf("racing branch was changed to %s", got)
}
})
}

// TestRecoverRebasedPreservedHeadAdoptsWithoutEscalating is the regression for
// the over-escalating custody return: a cancelled validation whose preserved
// pipeline head is the operator's own work rebased onto a newer base loses
Expand Down
Loading
Loading