fix: establish direct AWS checkpoint captures before non-waiting success

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Fixed
 
+- Checkpoints: establish every direct AWS backing snapshot before non-waiting success, expose provider-owned `captureEstablished` attestations while preserving other providers' success semantics, and reduce native availability and AWS public-IP polling slack. [PR 2834](https://github.com/openclaw/crabbox/pull/2834).
 - Testbox: announce readiness only after setup succeeds, so failed preparation does not keep an unprepared runner in its idle loop. [PR 2831](https://github.com/openclaw/crabbox/pull/2831).
 - Release verification: check online notarization on each binary's native macOS host while retaining foreign-architecture static checks and a manual online opt-in. [PR 2829](https://github.com/openclaw/crabbox/pull/2829).
 - Release verification: report bounded, attributable macOS notarization diagnostics without changing failed checks or executing rejected binaries. [PR 2823](https://github.com/openclaw/crabbox/pull/2823).

docs/commands/checkpoint.md

@@ -106,7 +106,7 @@ crabbox checkpoint create --id swift-crab --workdir /work/cbx_123/my-app
 
 # Return the complete checkpoint record for orchestration
 crabbox checkpoint create --id swift-crab --mode native --json
-# {"id":"chk_abc123","kind":"aws-ebs-snapshot","leaseId":"cbx_abcdef123456","workdir":"/work/cbx_abcdef123456/my-app","native":{"imageId":"snap-0123456789abcdef0","state":"available"}}
+# {"captureEstablished":true,"id":"chk_abc123","kind":"aws-ebs-snapshot","leaseId":"cbx_abcdef123456","workdir":"/work/cbx_abcdef123456/my-app","native":{"imageId":"snap-0123456789abcdef0","state":"available"}}
 ```
 
 **Flags**
@@ -140,7 +140,35 @@ crabbox checkpoint create --id swift-crab --mode native --json
 --discard-failed            Explicitly discard a verified failed capture and retire.
 ```
 
-On success, `--json` prints the checkpoint record. A native checkpoint operation
+On success, `--json` prints the checkpoint record. The optional top-level field
+`"captureEstablished": true` attests that the provider's capture contract has
+established the point-in-time
+capture of every included volume (or the completed archive's contents) is fixed:
+the source may keep running and writing without changing that capture. It does
+not mean the image is available to fork, nor certify application consistency or
+capture of memory, mounted external storage, or writes after the capture point.
+
+`--wait=false` skips waiting for native availability. Direct AWS AMIs still wait
+for evidence that capture is established, polling about once per second for the
+accepted image's exact backing snapshots, matching every source EBS device and
+requiring each snapshot to be `pending` or `completed`. `--wait-timeout` and
+cancellation bound this barrier. Permission errors, terminal failures, and
+timeouts report capture as uncertain and retain the accepted image identity;
+inspect that checkpoint rather than submitting a replacement. Other provider
+paths retain their existing success and wait semantics. They attest capture at
+their own established completion boundary, or omit the field if their existing
+contract supplies no evidence yet; an accepted pending image alone is insufficient.
+Native availability polling uses about three seconds; uncertain coordinator
+capture recovery retains its existing fifteen-second cadence.
+
+`--wait` results also include `captureEstablished: true` once capture is attested.
+Metadata-only recipes, retirement admission receipts, unresolved retirement
+phases, and failures omit the field; absence is never evidence that capture is
+established. A retirement record may attest capture after verified image
+readiness, while `capture.phase` remains the authority for source retirement.
+Historical records may omit the field.
+
+A native checkpoint operation
 that can prove it never attempted image submission may instead return this failure
 object, with a nonzero exit status, after the exact local reservation is removed
 and its absence is verified:

docs/features/provider-authoring.md

@@ -248,6 +248,18 @@ capabilities as any other provider that can checkpoint and fork workspace state;
 its Firecracker, Kubernetes, or image identifiers belong inside the adapter and
 checkpoint metadata.
 
+Native adapters retain their own create success and wait semantics. Set
+`NativeCheckpointCreateResult.CaptureEstablished` only where the existing
+provider contract establishes the exact capture (for example, Daytona's active
+snapshot or a draft image whose underlying snapshot is ready). Keep it false
+when the existing operation has only accepted a pending capture; this omits the
+assertion without rejecting normal success or introducing another wait.
+This internal attestation is not serialized into provider results; the CLI emits
+the single top-level `captureEstablished` field in its checkpoint record.
+Core does not reinterpret provider state names as capture barriers.
+An accepted request or allocated image ID alone is insufficient. Preserve accepted
+identity through errors and persist it before any observation or readiness wait.
+
 Run evidence and agent attachments follow the same rule. Declare
 `FeatureRunProof`, `FeatureRunArtifacts`, `FeatureRunDownloads`,
 `FeatureURLBridge`, `FeatureRunSession`, and `FeatureMCP` only when the provider

internal/cli/aws.go

@@ -910,6 +910,7 @@ func mapMarket(spot bool) string {
 func (c *AWSClient) waitForServerIP(ctx context.Context, id string) (Server, error) {
 	ctx, cancel := context.WithTimeoutCause(ctx, 10*time.Minute, Exit(5, "timed out waiting for AWS instance public IP"))
 	defer cancel()
+	interval := 250 * time.Millisecond
 	for {
 		server, err := c.getServerForAcquire(ctx, id)
 		if ctx.Err() != nil {
@@ -922,9 +923,10 @@ func (c *AWSClient) waitForServerIP(ctx context.Context, id string) (Server, err
 		if err == nil && server.PublicNet.IPv4.IP != "" {
 			return server, nil
 		}
-		if err := sleepContext(ctx, 5*time.Second); err != nil {
+		if err := sleepContext(ctx, interval); err != nil {
 			return Server{}, context.Cause(ctx)
 		}
+		interval = min(interval*2, 5*time.Second)
 	}
 }
 

internal/cli/aws_checkpoint_capture.go

@@ -0,0 +1,154 @@
+package cli
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/ec2"
+	"github.com/aws/aws-sdk-go-v2/service/ec2/types"
+)
+
+const awsImageCapturePollInterval = time.Second
+
+func (c *AWSClient) imageCheckpointSourceDevices(ctx context.Context, instanceID string) (map[string]string, error) {
+	out, err := c.ec2.DescribeInstances(ctx, &ec2.DescribeInstancesInput{InstanceIds: []string{instanceID}})
+	if err != nil {
+		return nil, err
+	}
+	for _, reservation := range out.Reservations {
+		for _, instance := range reservation.Instances {
+			if aws.ToString(instance.InstanceId) != instanceID {
+				continue
+			}
+			devices := make(map[string]string)
+			for _, mapping := range instance.BlockDeviceMappings {
+				if mapping.Ebs == nil {
+					continue
+				}
+				device, volume := aws.ToString(mapping.DeviceName), aws.ToString(mapping.Ebs.VolumeId)
+				if device == "" || volume == "" {
+					return nil, Exit(5, "AWS checkpoint source has an incomplete EBS mapping")
+				}
+				devices[device] = volume
+			}
+			if len(devices) == 0 {
+				return nil, Exit(5, "AWS checkpoint source has no EBS devices to capture")
+			}
+			return devices, nil
+		}
+	}
+	return nil, Exit(4, "aws instance not found: %s", instanceID)
+}
+
+type awsImageCaptureReader interface {
+	DescribeImages(context.Context, *ec2.DescribeImagesInput, ...func(*ec2.Options)) (*ec2.DescribeImagesOutput, error)
+	DescribeSnapshots(context.Context, *ec2.DescribeSnapshotsInput, ...func(*ec2.Options)) (*ec2.DescribeSnapshotsOutput, error)
+}
+
+func (c *AWSClient) waitForImageCapture(ctx context.Context, image CoordinatorImage, devices map[string]string, timeout time.Duration) (CoordinatorImage, error) {
+	return waitForAWSImageCapture(ctx, c.ec2, image, devices, timeout)
+}
+
+func waitForAWSImageCapture(ctx context.Context, reader awsImageCaptureReader, image CoordinatorImage, devices map[string]string, timeout time.Duration) (result CoordinatorImage, err error) {
+	result = image
+	waitCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("capture of AWS image %s is uncertain; retain its exact identity and inspect the checkpoint before continuing source writes: %w", image.ID, err)
+			err = errors.Join(Exit(5, "%s", err), err)
+		}
+	}()
+	if image.ID == "" || image.AccountID == "" || len(devices) == 0 {
+		return result, Exit(5, "capture observation requires exact image, account, and source EBS identities")
+	}
+	for {
+		if err := waitCtx.Err(); err != nil {
+			return result, err
+		}
+		out, readErr := reader.DescribeImages(waitCtx, &ec2.DescribeImagesInput{ImageIds: []string{image.ID}})
+		if readErr != nil && awsAPIErrorCode(readErr) != "InvalidAMIID.NotFound" {
+			return result, readErr
+		}
+		if readErr == nil && len(out.Images) > 0 {
+			if len(out.Images) != 1 || aws.ToString(out.Images[0].ImageId) != image.ID || aws.ToString(out.Images[0].OwnerId) != image.AccountID {
+				return result, Exit(5, "AWS image identity or owner changed during capture observation")
+			}
+			observed := out.Images[0]
+			result.State = string(observed.State)
+			result.SnapshotIDs = awsImageSnapshotIDs(observed)
+			result.Architecture = string(observed.Architecture)
+			if observed.State == types.ImageStateFailed || observed.State == types.ImageStateInvalid || observed.State == types.ImageStateDeregistered {
+				return result, Exit(5, "AWS image %s failed in state %s", image.ID, observed.State)
+			}
+			complete, observeErr := awsImageSnapshotsEstablished(waitCtx, reader, observed, devices, image.AccountID)
+			if observeErr != nil {
+				return result, observeErr
+			}
+			if complete {
+				if err := waitCtx.Err(); err != nil {
+					return result, err
+				}
+				result.captureEstablished = true
+				return result, nil
+			}
+		}
+		if err := sleepContext(waitCtx, awsImageCapturePollInterval); err != nil {
+			return result, err
+		}
+	}
+}
+
+func awsImageSnapshotsEstablished(ctx context.Context, reader awsImageCaptureReader, image types.Image, devices map[string]string, accountID string) (bool, error) {
+	snapshotVolumes := make(map[string]string)
+	seenDevices := make(map[string]bool)
+	for _, mapping := range image.BlockDeviceMappings {
+		if mapping.Ebs == nil {
+			continue
+		}
+		device, snapshotID := aws.ToString(mapping.DeviceName), aws.ToString(mapping.Ebs.SnapshotId)
+		volumeID := devices[device]
+		if volumeID == "" || seenDevices[device] {
+			return false, Exit(5, "AWS image EBS device mapping differs from the checkpoint source")
+		}
+		seenDevices[device] = true
+		if snapshotID == "" {
+			return false, nil
+		}
+		if _, exists := snapshotVolumes[snapshotID]; exists {
+			return false, Exit(5, "AWS image reuses a backing snapshot for multiple source devices")
+		}
+		snapshotVolumes[snapshotID] = volumeID
+	}
+	// DescribeImages can publish only part of the mapping while CreateImage is
+	// initiating snapshots. Every source EBS device must have its own snapshot.
+	if len(seenDevices) != len(devices) {
+		return false, nil
+	}
+	out, err := reader.DescribeSnapshots(ctx, &ec2.DescribeSnapshotsInput{SnapshotIds: awsImageSnapshotIDs(image)})
+	if err != nil {
+		if awsAPIErrorCode(err) == "InvalidSnapshot.NotFound" {
+			return false, nil
+		}
+		return false, err
+	}
+	seenSnapshots := make(map[string]bool)
+	for _, snapshot := range out.Snapshots {
+		id := aws.ToString(snapshot.SnapshotId)
+		if snapshotVolumes[id] == "" || seenSnapshots[id] || aws.ToString(snapshot.OwnerId) != accountID || aws.ToString(snapshot.VolumeId) != snapshotVolumes[id] {
+			return false, Exit(5, "AWS backing snapshot identity, owner, or source volume changed")
+		}
+		seenSnapshots[id] = true
+		switch snapshot.State {
+		case types.SnapshotStatePending, types.SnapshotStateCompleted:
+		case types.SnapshotStateError:
+			return false, Exit(5, "AWS backing snapshot %s failed in state error", id)
+		default:
+			return false, nil
+		}
+	}
+	return len(seenSnapshots) == len(snapshotVolumes), nil
+}

internal/cli/aws_checkpoint_capture_test.go

@@ -0,0 +1,144 @@
+package cli
+
+import (
+	"context"
+	"errors"
+	"slices"
+	"strings"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/ec2"
+	"github.com/aws/aws-sdk-go-v2/service/ec2/types"
+	"github.com/aws/smithy-go"
+)
+
+type fakeAWSImageCaptureReader struct {
+	images    func(context.Context, *ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error)
+	snapshots func(context.Context, *ec2.DescribeSnapshotsInput) (*ec2.DescribeSnapshotsOutput, error)
+}
+
+func (f fakeAWSImageCaptureReader) DescribeImages(ctx context.Context, in *ec2.DescribeImagesInput, _ ...func(*ec2.Options)) (*ec2.DescribeImagesOutput, error) {
+	return f.images(ctx, in)
+}
+func (f fakeAWSImageCaptureReader) DescribeSnapshots(ctx context.Context, in *ec2.DescribeSnapshotsInput, _ ...func(*ec2.Options)) (*ec2.DescribeSnapshotsOutput, error) {
+	return f.snapshots(ctx, in)
+}
+
+func TestAWSImageCaptureEstablishedBarrier(t *testing.T) {
+	for _, scenario := range []string{"image not found", "empty image", "missing snapshot ID", "partial mapping", "snapshot not found", "missing snapshot", "snapshot error", "timeout", "in-flight timeout", "cancellation", "permission", "wrong image", "wrong owner", "wrong snapshot", "wrong volume", "image failed", "unknown snapshot state"} {
+		t.Run(scenario, func(t *testing.T) {
+			synctest.Test(t, func(t *testing.T) {
+				const imageID, account = "ami-exact", "123456789012"
+				imageReads, snapshotReads := 0, 0
+				start := time.Now()
+				ctx, cancel := context.WithCancel(t.Context())
+				defer cancel()
+				reader := fakeAWSImageCaptureReader{
+					images: func(ctx context.Context, in *ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error) {
+						imageReads++
+						if !slices.Equal(in.ImageIds, []string{imageID}) {
+							t.Fatalf("wrong image IDs: %v", in.ImageIds)
+						}
+						if scenario == "in-flight timeout" {
+							<-ctx.Done()
+							return nil, ctx.Err()
+						}
+						if scenario == "cancellation" {
+							cancel()
+							return nil, ctx.Err()
+						}
+						if scenario == "permission" {
+							return nil, &smithy.GenericAPIError{Code: "UnauthorizedOperation"}
+						}
+						if scenario == "image not found" && imageReads == 1 {
+							return nil, &smithy.GenericAPIError{Code: "InvalidAMIID.NotFound"}
+						}
+						if scenario == "empty image" && imageReads == 1 {
+							return &ec2.DescribeImagesOutput{}, nil
+						}
+						observed := types.Image{ImageId: aws.String(imageID), OwnerId: aws.String(account), State: types.ImageStatePending,
+							BlockDeviceMappings: []types.BlockDeviceMapping{
+								{DeviceName: aws.String("/dev/sda1"), Ebs: &types.EbsBlockDevice{SnapshotId: aws.String("snap-root")}},
+								{DeviceName: aws.String("/dev/sdf"), Ebs: &types.EbsBlockDevice{SnapshotId: aws.String("snap-data")}},
+							},
+						}
+						if scenario == "wrong image" {
+							observed.ImageId = aws.String("ami-other")
+						}
+						if scenario == "wrong owner" {
+							observed.OwnerId = aws.String("999999999999")
+						}
+						if scenario == "image failed" {
+							observed.State = types.ImageStateFailed
+						}
+						if scenario == "timeout" || scenario == "missing snapshot ID" && imageReads == 1 {
+							observed.BlockDeviceMappings[1].Ebs.SnapshotId = nil
+						}
+						if scenario == "partial mapping" && imageReads == 1 {
+							observed.BlockDeviceMappings = observed.BlockDeviceMappings[:1]
+						}
+						return &ec2.DescribeImagesOutput{Images: []types.Image{observed}}, nil
+					},
+					snapshots: func(_ context.Context, in *ec2.DescribeSnapshotsInput) (*ec2.DescribeSnapshotsOutput, error) {
+						snapshotReads++
+						if !slices.Equal(in.SnapshotIds, []string{"snap-root", "snap-data"}) {
+							t.Fatalf("wrong snapshot IDs: %v", in.SnapshotIds)
+						}
+						if scenario == "snapshot not found" && snapshotReads == 1 {
+							return nil, &smithy.GenericAPIError{Code: "InvalidSnapshot.NotFound"}
+						}
+						out := &ec2.DescribeSnapshotsOutput{Snapshots: []types.Snapshot{
+							{SnapshotId: aws.String("snap-root"), OwnerId: aws.String(account), VolumeId: aws.String("vol-root"), State: types.SnapshotStateCompleted},
+							{SnapshotId: aws.String("snap-data"), OwnerId: aws.String(account), VolumeId: aws.String("vol-data"), State: types.SnapshotStatePending},
+						}}
+						if scenario == "snapshot error" {
+							out.Snapshots[1].State = types.SnapshotStateError
+						}
+						if scenario == "wrong snapshot" {
+							out.Snapshots[1].SnapshotId = aws.String("snap-other")
+						}
+						if scenario == "wrong volume" {
+							out.Snapshots[1].VolumeId = aws.String("vol-other")
+						}
+						if scenario == "unknown snapshot state" {
+							out.Snapshots[1].State = types.SnapshotState("recovering")
+						}
+						if scenario == "missing snapshot" && snapshotReads == 1 {
+							out.Snapshots = out.Snapshots[:1]
+						}
+						return out, nil
+					},
+				}
+				image := CoordinatorImage{ID: imageID, AccountID: account, State: "pending", Provider: "aws", Direct: true}
+				result, err := waitForAWSImageCapture(ctx, reader, image, map[string]string{"/dev/sda1": "vol-root", "/dev/sdf": "vol-data"}, 2500*time.Millisecond)
+				success := slices.Contains([]string{"image not found", "empty image", "missing snapshot ID", "partial mapping", "snapshot not found", "missing snapshot"}, scenario)
+				if result.ID != imageID || result.AccountID != account {
+					t.Fatalf("accepted identity lost: %+v", result)
+				}
+				if success {
+					if err != nil || !result.captureEstablished || len(result.SnapshotIDs) != 2 || result.State != "pending" || time.Since(start) != time.Second {
+						t.Fatalf("barrier=%+v err=%v elapsed=%s", result, err, time.Since(start))
+					}
+				} else {
+					if result.captureEstablished {
+						t.Fatal("uncertain AWS observation attests capture")
+					}
+					if err == nil || !strings.Contains(err.Error(), "uncertain") || strings.Contains(err.Error(), "established") {
+						t.Fatalf("false capture attestation: %v", err)
+					}
+					if strings.Contains(scenario, "timeout") || scenario == "unknown snapshot state" {
+						if !errors.Is(err, context.DeadlineExceeded) || time.Since(start) != 2500*time.Millisecond {
+							t.Fatalf("deadline not bounded: %v elapsed=%s", err, time.Since(start))
+						}
+					}
+					if scenario == "cancellation" && !errors.Is(err, context.Canceled) {
+						t.Fatalf("lost cancellation: %v", err)
+					}
+				}
+			})
+		})
+	}
+}