diff --git a/conformance/README.md b/conformance/README.md index ee0a017..64610ab 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -52,6 +52,25 @@ Some collections grow with an account's whole history — terminated VMs stay in `exact` rejects members outside `required` as well as missing ones, because sparse bodies and over-full ones are both real divergences — an emulator returning half the members of a summary is the commonest of all. `minItems` guards against a target that returns an empty list and satisfies every member check vacuously. +## Side probes in the middle of a scenario + +A failing step halts the rest of its scenario, because whatever the later steps assume about the target's state is no longer trustworthy. An unimplemented one halted it too, which was wrong for a step nothing downstream depends on. + +`CreateMicrovmAuthToken` sits between `suspend` and `resume` in `vm-suspend-resume` for a good reason — that is the only place a live recording can observe a token issued against a suspended VM — but while it was unimplemented it took `ResumeMicrovm` off the coverage report with it, and the operation had nothing wrong with it. + +`"optional": true` on a step exempts it from halting the scenario **when the target answers 501**: + +```json +{ + "name": "auth-token-while-suspended", + "operation": "CreateMicrovmAuthToken", + "optional": true, + ... +} +``` + +It exempts nothing else. A step that genuinely fails still halts its scenario however it is marked. Do not mark a step that carries `capture`: the vars it would have set go missing and the failure resurfaces several steps later, a long way from its cause. + ## Rejected fixtures A fixture renamed to `.json.rejected-` is a recording the suite refuses to treat as truth, because it captured the recording *account* rather than the service. They are kept rather than deleted: each one is evidence, and each cost a live session. diff --git a/conformance/cases/40-vm-suspend-resume.json b/conformance/cases/40-vm-suspend-resume.json index 4b64a75..238342e 100644 --- a/conformance/cases/40-vm-suspend-resume.json +++ b/conformance/cases/40-vm-suspend-resume.json @@ -109,6 +109,7 @@ { "name": "auth-token-while-suspended", "operation": "CreateMicrovmAuthToken", + "optional": true, "method": "POST", "path": "/2025-09-09/microvms/${microvmId}/auth-token", "body": { diff --git a/conformance/runner/runner.go b/conformance/runner/runner.go index b3dea04..bb05c57 100644 --- a/conformance/runner/runner.go +++ b/conformance/runner/runner.go @@ -42,6 +42,20 @@ type Step struct { Expect Expect `json:"expect"` Capture map[string]string `json:"capture,omitempty"` Until *Until `json:"until,omitempty"` + + // Optional marks a step whose operation is incidental to the scenario, so + // a 501 from it does not halt the rest. Without it a side probe wedged + // mid-scenario hides every step behind it: CreateMicrovmAuthToken sits + // between suspend and resume in vm-suspend-resume purely because that is + // where a live recording could observe it, and while it was unimplemented + // it took ResumeMicrovm's coverage down with it. + // + // It exempts Unimplemented only. A step that genuinely fails still halts + // the scenario however it is marked, because a wrong answer means the + // state the later steps assume is no longer trustworthy. A step carrying + // Capture should not be optional: the vars it would have set go missing + // and the failure resurfaces later, further from its cause. + Optional bool `json:"optional,omitempty"` } // Until turns a step into a poll: the request repeats until the dot-path in @@ -241,7 +255,7 @@ func (r *Runner) runScenario(s Scenario) []StepResult { } outcome, detail, fixture := r.runStep(s, st, vars) res.Outcome, res.Detail, res.Fixture = outcome, detail, fixture - if outcome != Pass { + if outcome != Pass && !(st.Optional && outcome == Unimplemented) { failed = true } results = append(results, res) diff --git a/conformance/runner/runner_test.go b/conformance/runner/runner_test.go index 4b19e1d..e573189 100644 --- a/conformance/runner/runner_test.go +++ b/conformance/runner/runner_test.go @@ -141,6 +141,97 @@ func TestRunAgainstStub(t *testing.T) { } } +// An optional step's 501 must not take the rest of the scenario with it — +// the case vm-suspend-resume is in, where CreateMicrovmAuthToken sat between +// suspend and resume and hid ResumeMicrovm's coverage behind it. +func TestOptionalUnimplementedStepDoesNotHaltScenario(t *testing.T) { + srv := stub() + defer srv.Close() + dir := t.TempDir() + writeScenario(t, dir, "30-optional.json", Scenario{ + ID: "optional-probe", + Tags: []string{"documented-only"}, + Steps: []Step{ + { + Name: "create", Operation: "CreateMicrovmImage", + Method: "POST", Path: "/2025-09-09/microvm-images", + Body: json.RawMessage(`{"name":"img1"}`), + Expect: Expect{Status: 201}, + Capture: map[string]string{"imageName": "name"}, + }, + { + Name: "side-probe", Operation: "CreateMicrovmAuthToken", + Method: "POST", Path: "/2025-09-09/microvms/x/auth-token", + Optional: true, + Expect: Expect{}, + }, + { + Name: "after-probe", Operation: "GetMicrovmImage", + Method: "GET", Path: "/2025-09-09/microvm-images/${imageName}", + Expect: Expect{Status: 200}, + }, + }, + }) + + r := newTestRunner(t, srv.URL, dir, t.TempDir(), false, nil) + scenarios, err := r.LoadScenarios() + if err != nil { + t.Fatal(err) + } + got := outcomes(r.Run(scenarios)) + + want := map[string]Outcome{ + "optional-probe/create": Pass, + "optional-probe/side-probe": Unimplemented, + "optional-probe/after-probe": Pass, + } + for k, w := range want { + if got[k] != w { + t.Errorf("%s: got %s, want %s", k, got[k], w) + } + } +} + +// Optional exempts 501 and nothing else. A wrong answer means the state the +// later steps assume is no longer trustworthy, so the scenario still halts. +func TestOptionalStepStillHaltsOnFailure(t *testing.T) { + srv := stub() + defer srv.Close() + dir := t.TempDir() + writeScenario(t, dir, "31-optional-fail.json", Scenario{ + ID: "optional-fail", + Tags: []string{"documented-only"}, + Steps: []Step{ + { + Name: "wrong-answer", Operation: "GetMicrovmImage", + Method: "GET", Path: "/2025-09-09/microvm-images/img1", + Optional: true, + // The stub answers 200 CREATED; demanding 404 is a real fail. + Expect: Expect{Status: 404}, + }, + { + Name: "after", Operation: "GetMicrovmImage", + Method: "GET", Path: "/2025-09-09/microvm-images/img1", + Expect: Expect{Status: 200}, + }, + }, + }) + + r := newTestRunner(t, srv.URL, dir, t.TempDir(), false, nil) + scenarios, err := r.LoadScenarios() + if err != nil { + t.Fatal(err) + } + got := outcomes(r.Run(scenarios)) + + if got["optional-fail/wrong-answer"] != Fail { + t.Errorf("wrong-answer: got %s, want fail", got["optional-fail/wrong-answer"]) + } + if got["optional-fail/after"] != Skipped { + t.Errorf("after: got %s, want skipped — an optional step that fails still halts", got["optional-fail/after"]) + } +} + func TestTagFilter(t *testing.T) { srv := stub() defer srv.Close() diff --git a/internal/vms/handlers.go b/internal/vms/handlers.go index fb51ee8..77c2e26 100644 --- a/internal/vms/handlers.go +++ b/internal/vms/handlers.go @@ -25,6 +25,8 @@ func Register(srv *api.Server, svc *Service, images ImageResolver) { srv.Register("RunMicrovm", h.run) srv.Register("GetMicrovm", h.get) srv.Register("ListMicrovms", h.list) + srv.Register("SuspendMicrovm", h.suspend) + srv.Register("ResumeMicrovm", h.resume) srv.Register("TerminateMicrovm", h.terminate) } @@ -95,7 +97,7 @@ func (h *handlers) run(w http.ResponseWriter, r *http.Request) { } vm := h.svc.Run(region, arn, version, idle) - api.WriteJSON(w, http.StatusOK, detail(vm)) + api.WriteJSON(w, http.StatusOK, detail(h.svc.Snapshot(vm))) } func (h *handlers) lookup(w http.ResponseWriter, r *http.Request) (*VM, bool) { @@ -118,38 +120,76 @@ func (h *handlers) get(w http.ResponseWriter, r *http.Request) { if !ok { return } - api.WriteJSON(w, http.StatusOK, detail(vm)) + api.WriteJSON(w, http.StatusOK, detail(h.svc.Snapshot(vm))) } func (h *handlers) list(w http.ResponseWriter, r *http.Request) { region := api.RegionFromRequest(r) items := make([]any, 0) - for _, vm := range h.svc.List(region) { + for _, vm := range h.svc.Snapshots(region) { items = append(items, listItem(vm)) } api.WriteJSON(w, http.StatusOK, map[string]any{"items": items, "nextToken": nil}) } -func (h *handlers) terminate(w http.ResponseWriter, r *http.Request) { +// mutable resolves the VM and rejects the one case the recording pinned down: +// any state change on a terminated VM is a plain 400 ValidationException, not +// either modeled conflict type. Suspend, resume and terminate share it. +func (h *handlers) mutable(w http.ResponseWriter, r *http.Request) (*VM, bool) { vm, ok := h.lookup(w, r) if !ok { - return + return nil, false } - if vm.Terminal() { - // The recorded shape for mutating a terminated VM: a plain 400 - // ValidationException, not either modeled conflict type. + if snap := h.svc.Snapshot(vm); snap.Terminal() { api.WriteError(w, http.StatusBadRequest, "ValidationException", map[string]any{ "message": "The MicroVM " + vm.ID + " has been terminated and its state cannot be changed.", }) + return nil, false + } + return vm, true +} + +// Recorded: 200 with an empty object, not the VM. All three mutations answer +// the same way. +func writeAccepted(w http.ResponseWriter) { + api.WriteJSON(w, http.StatusOK, map[string]any{}) +} + +func (h *handlers) suspend(w http.ResponseWriter, r *http.Request) { + vm, ok := h.mutable(w, r) + if !ok { + return + } + h.svc.Suspend(vm) + writeAccepted(w) +} + +func (h *handlers) resume(w http.ResponseWriter, r *http.Request) { + vm, ok := h.mutable(w, r) + if !ok { + return + } + h.svc.Resume(vm) + writeAccepted(w) +} + +func (h *handlers) terminate(w http.ResponseWriter, r *http.Request) { + vm, ok := h.mutable(w, r) + if !ok { return } h.svc.Terminate(vm) - // Recorded: 200 with an empty object, not the VM. - api.WriteJSON(w, http.StatusOK, map[string]any{}) + writeAccepted(w) } -// detail is the full VM shape, returned identically by Run and Get. -func detail(vm *VM) map[string]any { +// detail is the full VM shape, returned identically by Run and Get. It takes +// a snapshot rather than the live VM so a transition cannot change state +// halfway through building the body. +// +// The state marker is deliberately absent: it is m80's own instrumentation, +// read through the per-VM endpoint stub (#12), and putting it on a modeled +// response would be an invented member on the wire. +func detail(vm VM) map[string]any { body := map[string]any{ "egressNetworkConnectors": []any{managedConnector(vm.Region, "INTERNET_EGRESS")}, "endpoint": vm.Endpoint, @@ -173,7 +213,7 @@ func detail(vm *VM) map[string]any { } // listItem is a summary: five members, not the full detail. -func listItem(vm *VM) map[string]any { +func listItem(vm VM) map[string]any { return map[string]any{ "imageArn": vm.ImageArn, "imageVersion": vm.ImageVersion, diff --git a/internal/vms/vms.go b/internal/vms/vms.go index 1a996ab..1036099 100644 --- a/internal/vms/vms.go +++ b/internal/vms/vms.go @@ -1,22 +1,28 @@ -// Package vms implements the MicroVM resource: run, get, list, terminate. -// -// Suspend, resume and the idle timers are #11; this package owns the states -// either side of them and the storage they share. +// Package vms implements the MicroVM resource: run, get, list, suspend, +// resume, terminate, and the three timers that bound a VM's life. // // Two recorded facts shape everything here. VM ids are microvm-, not // the mv-… the docs guessed, and every VM carries managed default connectors // on both directions — INTERNET_EGRESS out and HTTP_INGRESS in — neither of // which appears in the model's NetworkConnectorType enum. +// +// Every mutable field of every VM is guarded by the service mutex. The +// transitions run on clock callbacks, which under clock.Real are separate +// goroutines, while handlers read the same fields to build a response. Tests +// use clock.Test, whose callbacks run on the test goroutine, so -race cannot +// see that collision; it is real in the shipped binary regardless. package vms import ( + "sync" "time" "github.com/intentius/m80/internal/clock" "github.com/intentius/m80/internal/store" ) -// VM states, from MicrovmState. +// VM states, from MicrovmState. There is no RESUMING: recorded, a resume goes +// SUSPENDED straight to RUNNING with nothing sampled between. const ( StatePending = "PENDING" StateRunning = "RUNNING" @@ -26,12 +32,19 @@ const ( StateTerminated = "TERMINATED" ) -// MaximumDurationSeconds is the eight-hour session cap every VM reports. +// MaximumDurationSeconds is the eight-hour session cap every VM reports, and +// the deadline the service enforces against it whatever state the VM is in. const MaximumDurationSeconds = 28800 +// MaximumDuration is MaximumDurationSeconds as a duration. +const MaximumDuration = MaximumDurationSeconds * time.Second + // IdlePolicy is echoed back as given. autoResumeEnabled is required whenever // the policy is present at all — recorded, and the model marks no member of // it required. +// +// The policy is written once, at Run, and read thereafter; nothing mutates it +// in place, which is why a VM snapshot can share the pointer. type IdlePolicy struct { AutoResumeEnabled bool `json:"autoResumeEnabled"` MaxIdleDurationSeconds *int `json:"maxIdleDurationSeconds,omitempty"` @@ -49,11 +62,30 @@ type VM struct { StateReason *string IdlePolicy *IdlePolicy Endpoint string + + // LastActivity is when the endpoint last saw traffic. The idle timer + // measures from here, not from when it happened to be armed. + LastActivity time.Time + + // Marker is the state marker: a monotonic counter bumped by every + // endpoint request and never reset, so a client that reads it through the + // endpoint stub (#12) across a suspend and resume can prove the VM's + // state survived rather than being rebuilt. + Marker uint64 + + // stateSeq is bumped on every state change. A timer captures it when + // armed and does nothing if it no longer matches, which is how a stale + // idle or suspend-cap timer from an earlier RUNNING or SUSPENDED period + // stays harmless — clock.Clock has no cancel, by design. + stateSeq uint64 } // Terminal reports whether the VM can still change state. Mutating a // terminated VM is a 400 ValidationException, recorded — not either of the // conflict types the model offers. +// +// Callers outside this package read it off a Snapshot; inside, it is only +// safe under the service mutex. func (v *VM) Terminal() bool { return v.State == StateTerminated } @@ -63,8 +95,13 @@ type Service struct { store *store.Store // Transition is one hop of a VM state machine: PENDING to RUNNING, - // TERMINATING to TERMINATED. + // SUSPENDING to SUSPENDED, TERMINATING to TERMINATED. Transition time.Duration + + // mu guards every mutable field of every VM this service owns. One lock + // rather than one per VM: an emulator has no contention worth splitting, + // and a single lock is one fewer ordering rule to get wrong. + mu sync.Mutex } func NewService(c clock.Clock, s *store.Store, transition time.Duration) *Service { @@ -79,6 +116,15 @@ func (s *Service) Get(region, id string) (*VM, bool) { return s.collection(region).Get(id) } +// Snapshot copies a VM's mutable state under the lock, so a handler renders +// one consistent view rather than reading fields a transition is midway +// through changing. +func (s *Service) Snapshot(vm *VM) VM { + s.mu.Lock() + defer s.mu.Unlock() + return *vm +} + // List returns VMs sorted by id so responses are stable. Terminated VMs stay // listed — recorded, and the reason a recorded ListMicrovms fixture can never // match a fresh emulator. @@ -95,60 +141,236 @@ func (s *Service) List(region string) []*VM { return out } -// Run creates a VM in PENDING and schedules it into RUNNING. +// Snapshots is List with every VM copied under one acquisition of the lock, +// so a list response cannot show two VMs from different instants. +func (s *Service) Snapshots(region string) []VM { + vms := s.List(region) + s.mu.Lock() + defer s.mu.Unlock() + out := make([]VM, 0, len(vms)) + for _, vm := range vms { + out = append(out, *vm) + } + return out +} + +// Run creates a VM in PENDING and schedules it into RUNNING. The eight-hour +// session cap is armed here and never re-armed: it bounds total life from +// launch, regardless of how the VM spends it. func (s *Service) Run(region, imageArn, imageVersion string, idle *IdlePolicy) *VM { id := "microvm-" + newUUID() + now := s.clock.Now() vm := &VM{ ID: id, Region: region, ImageArn: imageArn, ImageVersion: imageVersion, State: StatePending, - StartedAt: s.clock.Now(), + StartedAt: now, + LastActivity: now, IdlePolicy: idle, // The endpoint hostname is a bare UUID, not the microvm- prefixed id. Endpoint: newUUID() + ".lambda-microvm." + region + ".on.aws", } s.collection(region).Put(id, vm) + + seq := vm.stateSeq s.clock.After(s.Transition, func() { - if vm.State == StatePending { - vm.State = StateRunning + s.mu.Lock() + defer s.mu.Unlock() + if vm.stateSeq != seq || vm.State != StatePending { + return } + s.enterRunningLocked(vm) }) + s.armSessionCap(vm) return vm } +// Suspend walks a VM to SUSPENDED through SUSPENDING. +// +// Suspending an already suspending or suspended VM is a no-op answered 200, +// and so is suspending one on its way to TERMINATED. Neither was recorded, and +// between inventing an error type and being idempotent, idempotent is the +// safer guess for a consumer whose reconcile loop may re-issue the call. +// PENDING is allowed through for the same reason. +func (s *Service) Suspend(vm *VM) { + s.mu.Lock() + defer s.mu.Unlock() + switch vm.State { + case StatePending, StateRunning: + s.beginSuspendLocked(vm) + } +} + +// Resume returns a suspended VM to RUNNING with no state in between. +// +// Recorded 2026-07-30: a five-second poll across a full cycle saw SUSPENDED +// then RUNNING and nothing else, while the same poll did catch PENDING on the +// initial launch — so the two paths genuinely differ and this is not a +// sampling artifact. The enum has no RESUMING to occupy anyway. +// +// A VM still in SUSPENDING resumes too: its pending transition finds a +// changed stateSeq and does nothing. +func (s *Service) Resume(vm *VM) { + s.mu.Lock() + defer s.mu.Unlock() + switch vm.State { + case StateSuspending, StateSuspended: + s.enterRunningLocked(vm) + } +} + +// Touch records endpoint traffic: it bumps the state marker and resets the +// idle timer's reference point. #12's endpoint stub is the caller; it returns +// the marker so the stub can serve it back. +func (s *Service) Touch(vm *VM) uint64 { + s.mu.Lock() + defer s.mu.Unlock() + vm.LastActivity = s.clock.Now() + vm.Marker++ + return vm.Marker +} + // Terminate walks the VM to TERMINATED through TERMINATING. The recording // never sampled TERMINATING at a five-second poll, but it is in the enum and // a faster client can see it, so m80 goes through it rather than jumping. func (s *Service) Terminate(vm *VM) { - if vm.Terminal() || vm.State == StateTerminating { + s.mu.Lock() + defer s.mu.Unlock() + s.terminateLocked(vm) +} + +// HasRunningVMs implements the images package's VMChecker: an image cannot be +// deleted while anything is running off it. +func (s *Service) HasRunningVMs(region, imageArn string) bool { + vms := s.List(region) + s.mu.Lock() + defer s.mu.Unlock() + for _, vm := range vms { + if vm.ImageArn != imageArn { + continue + } + if !vm.Terminal() { + return true + } + } + return false +} + +func (s *Service) setStateLocked(vm *VM, state string) { + vm.State = state + vm.stateSeq++ +} + +// enterRunningLocked is the one way into RUNNING, from launch or from resume, +// so the idle timer is armed identically on both paths. +func (s *Service) enterRunningLocked(vm *VM) { + s.setStateLocked(vm, StateRunning) + vm.LastActivity = s.clock.Now() + s.armIdleLocked(vm) +} + +func (s *Service) beginSuspendLocked(vm *VM) { + s.setStateLocked(vm, StateSuspending) + seq := vm.stateSeq + s.clock.After(s.Transition, func() { + s.mu.Lock() + defer s.mu.Unlock() + if vm.stateSeq != seq { + return + } + s.setStateLocked(vm, StateSuspended) + s.armSuspendCapLocked(vm) + }) +} + +func (s *Service) terminateLocked(vm *VM) { + if vm.State == StateTerminated || vm.State == StateTerminating { return } - vm.State = StateTerminating + s.setStateLocked(vm, StateTerminating) + seq := vm.stateSeq s.clock.After(s.Transition, func() { - vm.State = StateTerminated + s.mu.Lock() + defer s.mu.Unlock() + if vm.stateSeq != seq { + return + } + s.setStateLocked(vm, StateTerminated) now := s.clock.Now() vm.TerminatedAt = &now // Recorded: a cleanly terminated VM reports exactly this, trailing - // period included. + // period included. A VM the suspend cap or the session cap ended + // reports it too — the service's wording for those paths was never + // recorded, and guessing a different string would put an invented + // value on the wire. reason := "Success." vm.StateReason = &reason }) } -// HasRunningVMs implements the images package's VMChecker: an image cannot be -// deleted while anything is running off it. -func (s *Service) HasRunningVMs(region, imageArn string) bool { - for _, vm := range s.List(region) { - if vm.ImageArn != imageArn { - continue +// armIdleLocked starts the idle countdown for the VM's current RUNNING +// period. No policy, or no maxIdleDurationSeconds in it, means no idle +// suspend at all. +func (s *Service) armIdleLocked(vm *VM) { + if vm.IdlePolicy == nil || vm.IdlePolicy.MaxIdleDurationSeconds == nil { + return + } + window := time.Duration(*vm.IdlePolicy.MaxIdleDurationSeconds) * time.Second + s.armIdleAfterLocked(vm, window, vm.stateSeq) +} + +// armIdleAfterLocked schedules the idle check. Because the clock has no +// cancel, traffic does not reset the timer; the timer fires, finds activity +// newer than it expected, and re-arms for the remainder. Same behavior, one +// less thing for the clock to model. +func (s *Service) armIdleAfterLocked(vm *VM, d time.Duration, seq uint64) { + s.clock.After(d, func() { + s.mu.Lock() + defer s.mu.Unlock() + if vm.stateSeq != seq || vm.State != StateRunning { + return } - if !vm.Terminal() { - return true + if vm.IdlePolicy == nil || vm.IdlePolicy.MaxIdleDurationSeconds == nil { + return } + window := time.Duration(*vm.IdlePolicy.MaxIdleDurationSeconds) * time.Second + if idle := s.clock.Now().Sub(vm.LastActivity); idle < window { + s.armIdleAfterLocked(vm, window-idle, seq) + return + } + s.beginSuspendLocked(vm) + }) +} + +// armSuspendCapLocked bounds how long a VM may sit in SUSPENDED before the +// service reclaims it. +func (s *Service) armSuspendCapLocked(vm *VM) { + if vm.IdlePolicy == nil || vm.IdlePolicy.SuspendedDurationSeconds == nil { + return } - return false + window := time.Duration(*vm.IdlePolicy.SuspendedDurationSeconds) * time.Second + seq := vm.stateSeq + s.clock.After(window, func() { + s.mu.Lock() + defer s.mu.Unlock() + if vm.stateSeq != seq || vm.State != StateSuspended { + return + } + s.terminateLocked(vm) + }) +} + +// armSessionCap bounds total session life at eight hours. It carries no +// stateSeq guard: unlike the other two it is not scoped to a state, and a VM +// that suspended and resumed six times still dies at the same wall time. +func (s *Service) armSessionCap(vm *VM) { + s.clock.After(MaximumDuration, func() { + s.mu.Lock() + defer s.mu.Unlock() + s.terminateLocked(vm) + }) } func sortStrings(s []string) { diff --git a/internal/vms/vms_test.go b/internal/vms/vms_test.go index 1ece7e3..621c38b 100644 --- a/internal/vms/vms_test.go +++ b/internal/vms/vms_test.go @@ -5,6 +5,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "testing" "time" @@ -75,6 +76,47 @@ func (h *harness) run(t *testing.T) (string, map[string]any) { return doc["microvmId"].(string), doc } +// runIdle launches a VM carrying an idle policy and settles it into RUNNING, +// which is where every timer test starts. +func (h *harness) runIdle(t *testing.T, maxIdleSec, suspendedSec int) string { + t.Helper() + policy := map[string]any{"autoResumeEnabled": false} + if maxIdleSec > 0 { + policy["maxIdleDurationSeconds"] = maxIdleSec + } + if suspendedSec > 0 { + policy["suspendedDurationSeconds"] = suspendedSec + } + rec, doc := h.do("POST", "/2025-09-09/microvms", map[string]any{ + "imageIdentifier": imgArn, + "idlePolicy": policy, + }) + if rec.Code != http.StatusOK { + t.Fatalf("run: status %d (%s)", rec.Code, rec.Body.String()) + } + h.clk.Advance(hop) + return doc["microvmId"].(string) +} + +func (h *harness) state(t *testing.T, id string) string { + t.Helper() + rec, doc := h.do("GET", "/2025-09-09/microvms/"+id, nil) + if rec.Code != http.StatusOK { + t.Fatalf("get %s: status %d", id, rec.Code) + } + s, _ := doc["state"].(string) + return s +} + +func (h *harness) vm(t *testing.T, id string) *VM { + t.Helper() + vm, ok := h.svc.Get(region, id) + if !ok { + t.Fatalf("VM %s not in the store", id) + } + return vm +} + // VM ids are microvm-; the mv-… in the early docs was a guess, and the // wrong shape made the live API gateway answer with an HTML 502. func TestVMIdShape(t *testing.T) { @@ -301,3 +343,337 @@ func TestMissingVMIs404(t *testing.T) { t.Errorf("status %d, want 404", rec.Code) } } + +// SUSPENDING was never sampled live at a five-second poll, but it is in the +// enum and a faster client can see it, so m80 goes through it. +func TestSuspendWalksThroughSuspending(t *testing.T) { + h := newHarness(t, stubImages{runnable: true}) + id := h.runIdle(t, 0, 0) + + rec, doc := h.do("POST", "/2025-09-09/microvms/"+id+"/suspend", nil) + if rec.Code != http.StatusOK { + t.Fatalf("suspend: status %d (%s)", rec.Code, rec.Body.String()) + } + if len(doc) != 0 { + t.Errorf("suspend body %v, want {}", doc) + } + if got := h.state(t, id); got != StateSuspending { + t.Fatalf("state %v, want SUSPENDING", got) + } + h.clk.Advance(hop) + if got := h.state(t, id); got != StateSuspended { + t.Fatalf("state %v, want SUSPENDED", got) + } +} + +// The recorded asymmetry: the same five-second poll that caught PENDING on +// the initial launch saw nothing at all between SUSPENDED and RUNNING. There +// is no RESUMING in the enum, and resume does not go back through PENDING. +func TestResumeGoesStraightToRunningWithoutPending(t *testing.T) { + h := newHarness(t, stubImages{runnable: true}) + id := h.runIdle(t, 0, 0) + h.do("POST", "/2025-09-09/microvms/"+id+"/suspend", nil) + h.clk.Advance(hop) + + rec, doc := h.do("POST", "/2025-09-09/microvms/"+id+"/resume", nil) + if rec.Code != http.StatusOK { + t.Fatalf("resume: status %d (%s)", rec.Code, rec.Body.String()) + } + if len(doc) != 0 { + t.Errorf("resume body %v, want {}", doc) + } + // RUNNING on the very next read, with no transition hop advanced. + if got := h.state(t, id); got != StateRunning { + t.Fatalf("state %v immediately after resume, want RUNNING", got) + } +} + +// A resume that arrives while the suspend is still settling wins: the pending +// transition finds a changed generation and does nothing. +func TestResumeDuringSuspendingCancelsTheSuspend(t *testing.T) { + h := newHarness(t, stubImages{runnable: true}) + id := h.runIdle(t, 0, 0) + h.do("POST", "/2025-09-09/microvms/"+id+"/suspend", nil) + if got := h.state(t, id); got != StateSuspending { + t.Fatalf("state %v, want SUSPENDING", got) + } + + h.do("POST", "/2025-09-09/microvms/"+id+"/resume", nil) + if got := h.state(t, id); got != StateRunning { + t.Fatalf("state %v, want RUNNING", got) + } + // The suspend's transition is still scheduled; it must not land. + h.clk.Advance(hop * 4) + if got := h.state(t, id); got != StateRunning { + t.Fatalf("state %v after the stale transition came due, want RUNNING", got) + } +} + +// Suspending something already suspended is a no-op answered 200. Unrecorded, +// and idempotence is the safer guess than an invented error for a reconciler +// that may re-issue the call. +func TestSuspendIsIdempotent(t *testing.T) { + h := newHarness(t, stubImages{runnable: true}) + id := h.runIdle(t, 0, 0) + h.do("POST", "/2025-09-09/microvms/"+id+"/suspend", nil) + h.clk.Advance(hop) + + rec, _ := h.do("POST", "/2025-09-09/microvms/"+id+"/suspend", nil) + if rec.Code != http.StatusOK { + t.Fatalf("status %d, want 200", rec.Code) + } + if got := h.state(t, id); got != StateSuspended { + t.Errorf("state %v, want SUSPENDED", got) + } +} + +func TestIdleTimerSuspendsAfterMaxIdle(t *testing.T) { + h := newHarness(t, stubImages{runnable: true}) + id := h.runIdle(t, 900, 0) + + h.clk.Advance(899 * time.Second) + if got := h.state(t, id); got != StateRunning { + t.Fatalf("state %v one second short of the idle window, want RUNNING", got) + } + h.clk.Advance(time.Second) + if got := h.state(t, id); got != StateSuspending { + t.Fatalf("state %v at the idle window, want SUSPENDING", got) + } + h.clk.Advance(hop) + if got := h.state(t, id); got != StateSuspended { + t.Fatalf("state %v, want SUSPENDED", got) + } +} + +// Endpoint traffic resets the countdown. The clock has no cancel, so the +// armed timer fires, finds newer activity than it expected and re-arms for +// the remainder — the VM must not suspend on that first firing. +func TestEndpointTrafficDefersIdleSuspend(t *testing.T) { + h := newHarness(t, stubImages{runnable: true}) + id := h.runIdle(t, 900, 0) + vm := h.vm(t, id) + + h.clk.Advance(400 * time.Second) + h.svc.Touch(vm) + + // The original timer comes due here and must decline to act. + h.clk.Advance(500 * time.Second) + if got := h.state(t, id); got != StateRunning { + t.Fatalf("state %v after traffic reset the window, want RUNNING", got) + } + // 900s after the touch, not after the arming. + h.clk.Advance(400 * time.Second) + if got := h.state(t, id); got != StateSuspending { + t.Fatalf("state %v 900s after the last traffic, want SUSPENDING", got) + } +} + +func TestNoIdlePolicyMeansNoIdleSuspend(t *testing.T) { + h := newHarness(t, stubImages{runnable: true}) + id, _ := h.run(t) + h.clk.Advance(hop) + + h.clk.Advance(4 * time.Hour) + if got := h.state(t, id); got != StateRunning { + t.Errorf("state %v with no idlePolicy, want RUNNING", got) + } +} + +func TestSuspendCapTerminatesSuspendedVM(t *testing.T) { + h := newHarness(t, stubImages{runnable: true}) + id := h.runIdle(t, 900, 1800) + h.do("POST", "/2025-09-09/microvms/"+id+"/suspend", nil) + h.clk.Advance(hop) + if got := h.state(t, id); got != StateSuspended { + t.Fatalf("state %v, want SUSPENDED", got) + } + + h.clk.Advance(1799 * time.Second) + if got := h.state(t, id); got != StateSuspended { + t.Fatalf("state %v one second short of the suspend cap, want SUSPENDED", got) + } + h.clk.Advance(2*time.Second + hop) + if got := h.state(t, id); got != StateTerminated { + t.Fatalf("state %v past the suspend cap, want TERMINATED", got) + } +} + +// A resume restarts the suspend cap. The first suspension's timer is stale +// and must not reclaim a VM that has since suspended a second time. +func TestSuspendCapDoesNotFireOnAStaleSuspension(t *testing.T) { + h := newHarness(t, stubImages{runnable: true}) + id := h.runIdle(t, 0, 1800) + + h.do("POST", "/2025-09-09/microvms/"+id+"/suspend", nil) + h.clk.Advance(hop) + h.clk.Advance(1000 * time.Second) + h.do("POST", "/2025-09-09/microvms/"+id+"/resume", nil) + + // Suspend again; the first suspension's cap comes due 800s from now and + // must be inert, because this suspension has its own full 1800s. + h.do("POST", "/2025-09-09/microvms/"+id+"/suspend", nil) + h.clk.Advance(hop) + h.clk.Advance(1000 * time.Second) + if got := h.state(t, id); got != StateSuspended { + t.Fatalf("state %v: a stale suspend cap reclaimed the VM early", got) + } +} + +// Eight hours bounds total session life regardless of how the VM spent it. +func TestSessionCapTerminatesRegardlessOfState(t *testing.T) { + h := newHarness(t, stubImages{runnable: true}) + id := h.runIdle(t, 0, 0) + h.do("POST", "/2025-09-09/microvms/"+id+"/suspend", nil) + h.clk.Advance(hop) + + h.clk.Advance(MaximumDuration - 10*time.Second) + if got := h.state(t, id); got != StateSuspended { + t.Fatalf("state %v short of the session cap, want SUSPENDED", got) + } + h.clk.Advance(10*time.Second + hop) + if got := h.state(t, id); got != StateTerminated { + t.Fatalf("state %v at the eight-hour cap, want TERMINATED", got) + } +} + +// The point of the marker: state that survives a suspend and resume, which is +// what #12's endpoint stub serves back to prove the VM was not rebuilt. +func TestMarkerSurvivesSuspendResume(t *testing.T) { + h := newHarness(t, stubImages{runnable: true}) + id := h.runIdle(t, 900, 1800) + vm := h.vm(t, id) + + h.svc.Touch(vm) + h.svc.Touch(vm) + if got := h.svc.Snapshot(vm).Marker; got != 2 { + t.Fatalf("marker %d after two requests, want 2", got) + } + + h.do("POST", "/2025-09-09/microvms/"+id+"/suspend", nil) + h.clk.Advance(hop) + h.do("POST", "/2025-09-09/microvms/"+id+"/resume", nil) + + if got := h.svc.Snapshot(vm).Marker; got != 2 { + t.Fatalf("marker %d across suspend and resume, want it preserved at 2", got) + } + if got := h.svc.Touch(vm); got != 3 { + t.Errorf("marker %d on the next request, want it to keep counting at 3", got) + } +} + +// The marker is m80's own instrumentation and must not leak onto a modeled +// response, where it would be an invented member. +func TestMarkerIsNotOnTheWire(t *testing.T) { + h := newHarness(t, stubImages{runnable: true}) + id := h.runIdle(t, 0, 0) + h.svc.Touch(h.vm(t, id)) + + _, doc := h.do("GET", "/2025-09-09/microvms/"+id, nil) + for _, member := range []string{"marker", "Marker", "stateMarker", "lastActivity"} { + if _, has := doc[member]; has { + t.Errorf("GetMicrovm response carries %q", member) + } + } +} + +// Case 82: suspending a terminated VM is a plain 400 ValidationException, not +// either conflict type the model offers. Resume takes the same path. +func TestSuspendAndResumeOnTerminatedVMAre400(t *testing.T) { + h := newHarness(t, stubImages{runnable: true}) + id := h.runIdle(t, 0, 0) + h.do("DELETE", "/2025-09-09/microvms/"+id, nil) + h.clk.Advance(hop) + if got := h.state(t, id); got != StateTerminated { + t.Fatalf("state %v, want TERMINATED", got) + } + + for _, action := range []string{"suspend", "resume"} { + rec, doc := h.do("POST", "/2025-09-09/microvms/"+id+"/"+action, nil) + if rec.Code != http.StatusBadRequest { + t.Errorf("%s: status %d, want 400", action, rec.Code) + } + if got := rec.Header().Get("X-Amzn-Errortype"); got != "ValidationException" { + t.Errorf("%s: error type %q, want ValidationException", action, got) + } + if msg, _ := doc["message"].(string); !strings.Contains(msg, "has been terminated") { + t.Errorf("%s: message %q", action, msg) + } + } +} + +func TestSuspendOnMissingVMIs404(t *testing.T) { + h := newHarness(t, stubImages{runnable: true}) + rec, _ := h.do("POST", "/2025-09-09/microvms/microvm-00000000-0000-0000-0000-000000000000/suspend", nil) + if rec.Code != http.StatusNotFound { + t.Errorf("status %d, want 404", rec.Code) + } +} + +// Every other test here drives clock.Test, whose callbacks run on the test +// goroutine, so -race never sees the transitions and handlers touching a VM +// at once. This one runs the real clock and reads while they fire, which is +// the arrangement the shipped binary is actually in. +func TestTransitionsAndHandlersDoNotRace(t *testing.T) { + st := store.New() + clk := clock.Real{} + srv := api.NewServer(clk, st, "test") + svc := NewService(clk, st, time.Millisecond) + Register(srv, svc, stubImages{runnable: true}) + + get := func(path string) { + r := httptest.NewRequest("GET", path, nil) + r.Header.Set("Authorization", + "AWS4-HMAC-SHA256 Credential=AKID/20260730/"+region+"/lambda/aws4_request, SignedHeaders=host, Signature=x") + srv.ServeHTTP(httptest.NewRecorder(), r) + } + post := func(path string, body any) { + raw, _ := json.Marshal(body) + r := httptest.NewRequest("POST", path, strings.NewReader(string(raw))) + r.Header.Set("Authorization", + "AWS4-HMAC-SHA256 Credential=AKID/20260730/"+region+"/lambda/aws4_request, SignedHeaders=host, Signature=x") + srv.ServeHTTP(httptest.NewRecorder(), r) + } + + idle := 1 + vm := svc.Run(region, imgArn, "1.0", &IdlePolicy{ + AutoResumeEnabled: false, + MaxIdleDurationSeconds: &idle, + }) + + var wg sync.WaitGroup + deadline := time.Now().Add(150 * time.Millisecond) + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for time.Now().Before(deadline) { + get("/2025-09-09/microvms/" + vm.ID) + get("/2025-09-09/microvms") + svc.Touch(vm) + svc.HasRunningVMs(region, imgArn) + } + }() + } + wg.Add(1) + go func() { + defer wg.Done() + for time.Now().Before(deadline) { + post("/2025-09-09/microvms/"+vm.ID+"/suspend", nil) + post("/2025-09-09/microvms/"+vm.ID+"/resume", nil) + } + }() + wg.Wait() +} + +// A suspended VM still blocks its image's deletion; only a terminal one frees +// it. +func TestSuspendedVMStillBlocksImageDeletion(t *testing.T) { + h := newHarness(t, stubImages{runnable: true}) + id := h.runIdle(t, 0, 0) + h.do("POST", "/2025-09-09/microvms/"+id+"/suspend", nil) + h.clk.Advance(hop) + + if !h.svc.HasRunningVMs(region, imgArn) { + t.Error("a SUSPENDED VM did not block image deletion") + } +}