diff --git a/pkg/vm/explicit_frame_test.go b/pkg/vm/explicit_frame_test.go deleted file mode 100644 index ed8cdbcd9..000000000 --- a/pkg/vm/explicit_frame_test.go +++ /dev/null @@ -1,383 +0,0 @@ -package vm - -import ( - "strings" - "testing" -) - -func testBytecodeFnReturningArg(consts *Consts, arity int) *Func { - chunk := NewCodeChunk(consts) - chunk.Append(OP_LOAD_ARG) - chunk.Append32(0) - chunk.Append(OP_RETURN) - chunk.SetMaxStack(4) - return MakeFunc(arity, false, chunk) -} - -func testInvokeZeroChunk(consts *Consts, fn Fn) *CodeChunk { - chunk := NewCodeChunk(consts) - chunk.Append(OP_LOAD_CONST) - chunk.Append32(consts.Intern(fn)) - chunk.Append(OP_INVOKE) - chunk.Append32(0) - chunk.Append(OP_RETURN) - chunk.SetMaxStack(4) - return chunk -} - -func testTailCallZeroChunk(consts *Consts, fn Fn) *CodeChunk { - chunk := NewCodeChunk(consts) - chunk.Append(OP_LOAD_CONST) - chunk.Append32(consts.Intern(fn)) - chunk.Append(OP_TAIL_CALL) - chunk.Append32(0) - chunk.Append(OP_RETURN) - chunk.SetMaxStack(4) - return chunk -} - -func TestResolveBytecodeCallSelectsMultiArityAndPreservesCaptures(t *testing.T) { - consts := NewConsts() - oneArg := testBytecodeFnReturningArg(consts, 1) - twoArg := testBytecodeFnReturningArg(consts, 2) - multi, err := MakeMultiArity([]Value{oneArg, twoArg}) - if err != nil { - t.Fatal(err) - } - - target, direct, err := resolveBytecodeCall(NewMetaFn(multi, NIL), []Value{Int(7)}) - if err != nil { - t.Fatal(err) - } - if !direct || target.fn.chunk != oneArg.chunk { - t.Fatal("metadata-wrapped multi-arity call did not select its bytecode variant") - } - - captures := []Value{Int(99)} - closure := &Closure{fn: multi, closedOvers: captures} - target, direct, err = resolveBytecodeCall(closure, []Value{Int(7), Int(8)}) - if err != nil { - t.Fatal(err) - } - if !direct || target.fn.chunk != twoArg.chunk { - t.Fatal("closure over multi-arity function did not select its bytecode variant") - } - if len(target.closedOvers) != 1 || target.closedOvers[0] != captures[0] { - t.Fatalf("closure captures were not preserved: got %v, want %v", target.closedOvers, captures) - } -} - -func TestResolveBytecodeCallVariadicPackingDoesNotMutateBorrowedArgs(t *testing.T) { - consts := NewConsts() - variadic := MakeFunc(2, true, testBytecodeFnReturningArg(consts, 2).chunk) - borrowed := []Value{Int(7), Int(11), Int(13)} - - target, direct, err := resolveBytecodeCall(variadic, borrowed) - if err != nil { - t.Fatal(err) - } - if !direct { - t.Fatal("variadic bytecode function was not resolved") - } - if borrowed[0] != Value(Int(7)) || borrowed[1] != Value(Int(11)) || borrowed[2] != Value(Int(13)) { - t.Fatalf("resolver mutated borrowed args: got %v", borrowed) - } - if len(target.args) != 2 || target.args[0] != Value(Int(7)) { - t.Fatalf("unexpected packed args: %v", target.args) - } - if target.args[1].String() != "(11 13)" { - t.Fatalf("unexpected packed rest args: %v", target.args[1]) - } -} - -func TestInstallBytecodeCallOwnsBorrowedArgs(t *testing.T) { - consts := NewConsts() - targetFn := testBytecodeFnReturningArg(consts, 2) - frame := NewFrame(testBytecodeFnReturningArg(consts, 0).chunk, nil) - frame.stack[0] = Int(7) - frame.stack[1] = Int(11) - frame.sp = 2 - - target, direct, err := resolveBytecodeCall(targetFn, frame.stack[:2]) - if err != nil { - t.Fatal(err) - } - if !direct { - t.Fatal("fixed-arity bytecode function was not resolved") - } - installBytecodeCall(frame, target) - frame.stack[0] = Int(99) - if frame.args[0] != Value(Int(7)) || frame.args[1] != Value(Int(11)) { - t.Fatalf("tail transition retained operand-stack-backed args: %v", frame.args) - } - ReleaseFrame(frame) -} - -func TestSuspendedCallArityValidatesContinuation(t *testing.T) { - valid := NewCodeChunk(NewConsts()) - valid.Append(OP_INVOKE) - valid.Append32(1) - valid.SetMaxStack(4) - frame := NewFrame(valid, nil) - frame.sp = 2 - if arity, err := suspendedCallArity(frame); err != nil || arity != 1 { - t.Fatalf("valid continuation: arity=%d err=%v", arity, err) - } - - tests := []struct { - name string - code []int32 - ip int - sp int - match string - }{ - {name: "negative ip", code: []int32{OP_INVOKE, 0}, ip: -1, sp: 1, match: "out of bounds"}, - {name: "ip past end", code: []int32{OP_INVOKE, 0}, ip: 2, sp: 1, match: "out of bounds"}, - {name: "missing arity", code: []int32{OP_INVOKE}, ip: 0, sp: 1, match: "missing arity"}, - {name: "wrong opcode", code: []int32{OP_NOOP, 0}, ip: 0, sp: 1, match: "NOOP"}, - {name: "invalid stack depth", code: []int32{OP_INVOKE, 0}, ip: 0, sp: 5, match: "stack depth 5 is out of bounds"}, - {name: "negative arity", code: []int32{OP_INVOKE, -1}, ip: 0, sp: 1, match: "arity -1"}, - {name: "arity exceeds stack", code: []int32{OP_TAIL_CALL, 2}, ip: 0, sp: 2, match: "exceeds stack depth"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - chunk := NewCodeChunk(NewConsts()) - chunk.Append(tt.code...) - chunk.SetMaxStack(4) - candidate := NewFrame(chunk, nil) - candidate.ip = tt.ip - candidate.sp = tt.sp - _, err := suspendedCallArity(candidate) - if err == nil || !strings.Contains(err.Error(), tt.match) { - t.Fatalf("got %v, want error containing %q", err, tt.match) - } - ReleaseFrame(candidate) - }) - } - ReleaseFrame(frame) -} - -func TestTailCallClosureArityErrorMatchesInvokeAndIsCatchable(t *testing.T) { - consts := NewConsts() - closure := &Closure{fn: testBytecodeFnReturningArg(consts, 1)} - - run := func(chunk *CodeChunk) error { - frame := NewFrame(chunk, nil) - frame.ec = RootExecContext - _, err := frame.Run() - ReleaseFrame(frame) - return err - } - invokeErr := run(testInvokeZeroChunk(consts, closure)) - tailErr := run(testTailCallZeroChunk(consts, closure)) - if invokeErr == nil || tailErr == nil { - t.Fatalf("expected arity errors: invoke=%v tail=%v", invokeErr, tailErr) - } - if got, want := innermostMessage(tailErr), innermostMessage(invokeErr); got != want { - t.Fatalf("tail-call arity error differs from invoke: got %q, want %q", got, want) - } - - catching := NewCodeChunk(consts) - catching.Append(OP_TRY_PUSH) - catching.Append32(9) // catch at the final OP_RETURN - catching.Append32(0) - catching.Append(OP_LOAD_CONST) - catching.Append32(consts.Intern(closure)) - catching.Append(OP_TAIL_CALL) - catching.Append32(0) - catching.Append(OP_TRY_POP) - catching.Append(OP_RETURN) - catching.Append(OP_RETURN) - catching.SetMaxStack(4) - frame := NewFrame(catching, nil) - frame.ec = RootExecContext - result, err := frame.Run() - ReleaseFrame(frame) - if err != nil { - t.Fatalf("caught tail-call arity error escaped: %v", err) - } - ex, ok := result.(*ExInfo) - if !ok || !strings.Contains(ex.Message(), "expected 1 args, got 0") { - t.Fatalf("unexpected caught value: %v", result) - } -} - -func TestDescendedFrameLifecycle(t *testing.T) { - oldAllocAttr := allocAttrEnabled - allocAttrEnabled = true - attrMu.Lock() - oldAttrStack := attrStack - attrStack = nil - attrMu.Unlock() - t.Cleanup(func() { - attrMu.Lock() - attrStack = oldAttrStack - attrMu.Unlock() - allocAttrEnabled = oldAllocAttr - }) - - consts := NewConsts() - var parentChunk, childChunk *CodeChunk - checkerValue, err := NativeFnType.Wrap(func(_ []Value) (Value, error) { - attrMu.Lock() - defer attrMu.Unlock() - if len(attrStack) != 2 { - return NIL, NewExecutionError("allocation attribution stack did not include parent and child") - } - if attrStack[0].code != parentChunk || attrStack[1].code != childChunk { - return NIL, NewExecutionError("allocation attribution stack has the wrong frame order") - } - return Int(1), nil - }) - if err != nil { - t.Fatal(err) - } - checker := checkerValue.(Fn) - childChunk = testInvokeZeroChunk(consts, checker) - child := MakeFunc(0, false, childChunk) - parentChunk = testInvokeZeroChunk(consts, child) - - root := NewFrame(parentChunk, nil) - root.ec = RootExecContext - if _, err := root.Run(); err != nil { - t.Fatal(err) - } - ReleaseFrame(root) - - attrMu.Lock() - defer attrMu.Unlock() - if len(attrStack) != 0 { - t.Fatalf("allocation attribution stack leaked %d frame(s)", len(attrStack)) - } -} - -func TestZeroArityClosureTailCallDescendsInDispatchLoop(t *testing.T) { - oldAllocAttr := allocAttrEnabled - allocAttrEnabled = true - attrMu.Lock() - oldAttrStack := attrStack - attrStack = nil - attrMu.Unlock() - t.Cleanup(func() { - attrMu.Lock() - attrStack = oldAttrStack - attrMu.Unlock() - allocAttrEnabled = oldAllocAttr - }) - - consts := NewConsts() - checkerValue, err := NativeFnType.Wrap(func(_ []Value) (Value, error) { - attrMu.Lock() - defer attrMu.Unlock() - if len(attrStack) != 2 { - return NIL, NewExecutionError("zero-arity tail call did not enter exactly one child frame") - } - if attrStack[1].parent != attrStack[0] { - return NIL, NewExecutionError("zero-arity closure re-entered Frame.Run instead of linking its parent") - } - return Int(1), nil - }) - if err != nil { - t.Fatal(err) - } - childChunk := testInvokeZeroChunk(consts, checkerValue.(Fn)) - closure := &Closure{fn: MakeFunc(0, false, childChunk)} - root := NewFrame(testTailCallZeroChunk(consts, closure), nil) - root.ec = RootExecContext - if _, err := root.Run(); err != nil { - t.Fatal(err) - } - ReleaseFrame(root) - - attrMu.Lock() - defer attrMu.Unlock() - if len(attrStack) != 0 { - t.Fatalf("allocation attribution stack leaked %d frame(s)", len(attrStack)) - } -} - -func TestOpcodeProfileSeparatesDescendedFrames(t *testing.T) { - ResetProfile() - ProfilingEnabled.Store(true) - t.Cleanup(func() { - ProfilingEnabled.Store(false) - ResetProfile() - }) - - consts := NewConsts() - childChunk := NewCodeChunk(consts) - childChunk.Append(OP_LOAD_CONST) - childChunk.Append32(consts.Intern(NIL)) - childChunk.Append(OP_RETURN) - childChunk.SetMaxStack(4) - parentChunk := testInvokeZeroChunk(consts, MakeFunc(0, false, childChunk)) - - root := NewFrame(parentChunk, nil) - root.ec = RootExecContext - if _, err := root.Run(); err != nil { - t.Fatal(err) - } - ReleaseFrame(root) - - var initialLoads, crossFrameLoads, invokeReturns uint64 - for _, pair := range PairSnapshot() { - switch { - case pair.Prev == 0 && pair.Curr == uint8(OP_LOAD_CONST): - initialLoads = pair.Count - case pair.Prev == uint8(OP_INVOKE) && pair.Curr == uint8(OP_LOAD_CONST): - crossFrameLoads = pair.Count - case pair.Prev == uint8(OP_INVOKE) && pair.Curr == uint8(OP_RETURN): - invokeReturns = pair.Count - } - } - if initialLoads != 2 { - t.Fatalf("first opcode was not reset for both frames: got %d initial LOAD_CONST pairs", initialLoads) - } - if crossFrameLoads != 0 { - t.Fatalf("profiler joined parent OP_INVOKE to child LOAD_CONST %d time(s)", crossFrameLoads) - } - if invokeReturns != 1 { - t.Fatalf("parent profiler state was not restored: got %d INVOKE→RETURN pairs", invokeReturns) - } -} - -func TestDescendedErrorReleasesChildFrame(t *testing.T) { - framePoolMu.Lock() - oldPool := framePoolStack - framePoolStack = nil - framePoolMu.Unlock() - t.Cleanup(func() { - framePoolMu.Lock() - clear(framePoolStack) - framePoolStack = oldPool - framePoolMu.Unlock() - }) - - consts := NewConsts() - badChunk := NewCodeChunk(consts) - badChunk.Append(OP_LOAD_ARG) - badChunk.Append32(0) - badChunk.Append(OP_RETURN) - badChunk.SetMaxStack(4) - parentChunk := testInvokeZeroChunk(consts, MakeFunc(0, false, badChunk)) - - root := NewFrame(parentChunk, nil) - root.ec = RootExecContext - if _, err := root.Run(); err == nil { - t.Fatal("expected descended child to fail") - } - if root.parent != nil { - t.Fatal("root retained a parent link after failed descent") - } - - framePoolMu.Lock() - pooled := append([]*Frame(nil), framePoolStack...) - framePoolMu.Unlock() - if len(pooled) != 1 { - t.Fatalf("failed child was not returned exactly once: pool has %d frames", len(pooled)) - } - if pooled[0].parent != nil || pooled[0].code != nil || pooled[0].args != nil { - t.Fatal("failed child retained execution references after pooling") - } - ReleaseFrame(root) -} diff --git a/pkg/vm/func.go b/pkg/vm/func.go index 2a9c98ea3..639d982e7 100644 --- a/pkg/vm/func.go +++ b/pkg/vm/func.go @@ -97,76 +97,6 @@ func boxRest(rest []Value) (Value, error) { return ListType.Box(rest) } -// bytecodeCallTarget is the resolved, frame-independent description of a -// directly executable bytecode call. Frame allocation and same-frame tail -// replacement are deliberately separate transitions so this resolver can be -// shared with the constant-space tail-call work in #620. -type bytecodeCallTarget struct { - fn *Func - args []Value - closedOvers []Value -} - -// resolveBytecodeCall unwraps metadata, selects multi-arity variants, -// preserves closure captures, validates arity, and packs variadic arguments. -// It returns false for callables that must continue through ExecContext.Invoke. -// -// Fixed-arity args may still borrow the caller's operand stack. A transition -// that reuses that same frame must copy them into frame-owned storage before -// resetting the operand stack. Variadic packing always returns a fresh slice. -func resolveBytecodeCall(fn Fn, args []Value) (bytecodeCallTarget, bool, error) { - display := fn - var closedOvers []Value - for { - switch t := fn.(type) { - case *MetaFn: - fn = t.Wrapped() - case *MultiArityFn: - variant, err := t.variantFor(display, len(args)) - if err != nil { - return bytecodeCallTarget{}, false, err - } - fn = variant - case *Closure: - closedOvers = t.closedOvers - fn = t.fn - case *Func: - prepared := args - if t.isVariadric { - if len(prepared) < t.arity-1 { - return bytecodeCallTarget{}, false, NewExecutionError(fmt.Sprintf("function %s expected at least %d args, got %d", display, t.arity-1, len(prepared))) - } - restlist, boxErr := boxRest(prepared[t.arity-1:]) - if boxErr != nil { - return bytecodeCallTarget{}, false, boxErr - } - // Never append into prepared: it may be a window into the - // caller's operand stack or a host-owned argument slice. - packed := make([]Value, t.arity) - copy(packed, prepared[:t.arity-1]) - packed[t.arity-1] = restlist - prepared = packed - } else if len(prepared) != t.arity { - return bytecodeCallTarget{}, false, NewExecutionError(fmt.Sprintf("function %s expected %d args, got %d", display, t.arity, len(prepared))) - } - return bytecodeCallTarget{ - fn: t, - args: prepared, - closedOvers: closedOvers, - }, true, nil - default: - return bytecodeCallTarget{}, false, nil - } - } -} - -func runBytecodeCallTarget(ec *ExecContext, target bytecodeCallTarget) (Value, error) { - f := newFrameForBytecodeCall(target, ec) - result, err := f.Run() - ReleaseFrame(f) - return result, err -} - func (l *Func) Invoke(pargs []Value) (result Value, err error) { return l.invokeIn(RootExecContext, pargs) } @@ -175,14 +105,36 @@ func (l *Func) Invoke(pargs []Value) (result Value, err error) { // so dynamic bindings propagate into the call. Invoke is invokeIn against the // root context. func (l *Func) invokeIn(ec *ExecContext, pargs []Value) (result Value, err error) { - target, ok, err := resolveBytecodeCall(l, pargs) - if err != nil { - return NIL, err - } - if !ok { - return NIL, NewExecutionError("unsupported function type") + args := pargs + if l.isVariadric { + if len(args) < l.arity-1 { + return NIL, NewExecutionError(fmt.Sprintf("function %s expected at least %d args, got %d", l, l.arity-1, len(args))) + } + rest := args[l.arity-1:] + restlist, boxErr := boxRest(rest) + if boxErr != nil { + return NIL, boxErr + } + // Build a FRESH slice; do not append into args' backing array. + // `append(args[0:l.arity-1], restlist)` reuses the caller's array + // (the reslice keeps its capacity), so the packed rest-list is + // written over the caller's element l.arity-1 — for a plain + // (fn [& as]) that is args[0]. A Go caller reusing one []Value + // across invocations then sees its arguments silently replaced by + // the previous call's rest-list: correct on the first call, garbage + // on the second. + packed := make([]Value, l.arity) + copy(packed, args[:l.arity-1]) + packed[l.arity-1] = restlist + args = packed + } else if len(args) != l.arity { + return NIL, NewExecutionError(fmt.Sprintf("function %s expected %d args, got %d", l, l.arity, len(args))) } - return runBytecodeCallTarget(ec, target) + f := NewFrame(l.chunk, args) + f.ec = ec + result, err = f.Run() + ReleaseFrame(f) + return result, err } func (l *Func) String() string { @@ -264,18 +216,51 @@ func (l *Closure) Invoke(pargs []Value) (result Value, err error) { // so dynamic bindings propagate into the call. Invoke delegates to invokeIn // against the root context. func (l *Closure) invokeIn(ec *ExecContext, pargs []Value) (result Value, err error) { - target, ok, err := resolveBytecodeCall(l, pargs) - if err != nil { - return NIL, err - } - if ok { - return runBytecodeCallTarget(ec, target) + if f, ok := l.fn.(*Func); ok { + args := pargs + if f.isVariadric { + if len(args) < f.arity-1 { + return NIL, NewExecutionError(fmt.Sprintf("function %s expected at least %d args, got %d", l, f.arity-1, len(args))) + } + rest := args[f.arity-1:] + restlist, boxErr := boxRest(rest) + if boxErr != nil { + return NIL, boxErr + } + // Fresh slice — see Func.invokeIn for why appending into the + // caller's backing array corrupts the caller's arguments. + packed := make([]Value, f.arity) + copy(packed, args[:f.arity-1]) + packed[f.arity-1] = restlist + args = packed + } else if len(args) != f.arity { + return NIL, NewExecutionError(fmt.Sprintf("function %s expected %d args, got %d", l, f.arity, len(args))) + } + frame := NewFrame(f.chunk, args) + frame.closedOvers = l.closedOvers + frame.ec = ec + result, err = frame.Run() + ReleaseFrame(frame) + return result, err } if mfn, ok := l.fn.(*MultiArityFn); ok { - variant, variantErr := mfn.variantFor(l, len(pargs)) - if variantErr != nil { - return NIL, variantErr + le := len(pargs) + var variant Fn + if f, ok := mfn.fns[le]; ok { + variant = f + } else if mfn.rest != nil && le >= mfn.rest.Arity() { + variant = mfn.rest + } else { + return NIL, NewExecutionError(fmt.Sprintf("function %s doesn't have a %d-arity variant", l, le)) + } + + if f, ok := variant.(*Func); ok { + subClosure := &Closure{ + closedOvers: l.closedOvers, + fn: f, + } + return subClosure.invokeIn(ec, pargs) } return ec.Invoke(variant, pargs) } @@ -339,25 +324,18 @@ func (l *MultiArityFn) Invoke(pargs []Value) (Value, error) { return l.invokeIn(RootExecContext, pargs) } -func (l *MultiArityFn) variantFor(display Fn, arity int) (Fn, error) { - if f, ok := l.fns[arity]; ok { - return f, nil - } - if l.rest != nil && arity >= l.rest.Arity() { - return l.rest, nil - } - return nil, NewExecutionError(fmt.Sprintf("function %s doesn't have a %d-arity variant", display, arity)) -} - // invokeIn runs the multi-arity function with the given ExecContext active, // so dynamic bindings propagate into the selected variant's call. Invoke delegates // to invokeIn against the root context. func (l *MultiArityFn) invokeIn(ec *ExecContext, pargs []Value) (Value, error) { - variant, err := l.variantFor(l, len(pargs)) - if err != nil { - return NIL, err + le := len(pargs) + if f, ok := l.fns[le]; ok { + return ec.Invoke(f, pargs) + } + if l.rest != nil && le >= l.rest.Arity() { + return ec.Invoke(l.rest, pargs) } - return ec.Invoke(variant, pargs) + return NIL, NewExecutionError(fmt.Sprintf("function %s doesn't have a %d-arity variant", l, le)) } func (l *MultiArityFn) String() string { diff --git a/pkg/vm/vm.go b/pkg/vm/vm.go index 88eeed4e7..38c3edeee 100644 --- a/pkg/vm/vm.go +++ b/pkg/vm/vm.go @@ -375,7 +375,6 @@ type exHandler struct { type Frame struct { stack []Value args []Value - argbuf []Value // frame-owned arguments used by same-frame tail replacement closedOvers []Value argc int consts *Consts @@ -386,9 +385,6 @@ type Frame struct { debug bool handlers []exHandler // exception handler stack (nil when unused) ec *ExecContext // per-execution context (dynamic bindings); nil = none installed - parent *Frame // suspended caller while the dispatch loop runs this frame - prevOp uint8 // opcode profiler state, preserved while this frame is suspended - profileOn bool // profiler gate sampled at frame entry } // Frame reuse via a mutex-guarded LIFO. @@ -443,7 +439,6 @@ func NewFrame(code *CodeChunk, args []Value) *Frame { f.stack = make([]Value, needed) } f.args = args - f.argbuf = f.argbuf[:0] f.argc = len(args) f.closedOvers = nil f.consts = code.consts @@ -453,9 +448,6 @@ func NewFrame(code *CodeChunk, args []Value) *Frame { f.sp = 0 f.debug = false f.ec = nil - f.parent = nil - f.prevOp = 0 - f.profileOn = false if f.handlers != nil { f.handlers = f.handlers[:0] } @@ -467,57 +459,13 @@ func NewFrame(code *CodeChunk, args []Value) *Frame { // We only nil out the large reference fields to avoid pinning code/const objects. func ReleaseFrame(f *Frame) { f.args = nil - clear(f.argbuf) - f.argbuf = nil f.closedOvers = nil f.consts = nil f.code = nil f.handlers = nil - f.ec = nil - f.parent = nil releaseFrame(f) } -// newFrameForBytecodeCall is the non-tail transition for a prepared bytecode -// target. Fixed-arity target.args may borrow the suspended parent's operand -// stack; that is safe for a child frame because the parent does not resume -// until the child has completed. -func newFrameForBytecodeCall(target bytecodeCallTarget, ec *ExecContext) *Frame { - child := NewFrame(target.fn.chunk, target.args) - child.closedOvers = target.closedOvers - child.ec = ec - return child -} - -// installBytecodeCall is the same-frame transition used by the existing -// direct-*Func tail-call path. target.args may be a window into f.stack, so -// copy it into the frame-owned buffer before resetting or reusing that stack. -func installBytecodeCall(f *Frame, target bytecodeCallTarget) { - argc := len(target.args) - if cap(f.argbuf) < argc { - f.argbuf = make([]Value, argc) - } else { - clear(f.argbuf) - f.argbuf = f.argbuf[:argc] - } - copy(f.argbuf, target.args) - - f.args = f.argbuf - f.argc = argc - f.closedOvers = target.closedOvers - f.code = target.fn.chunk - f.consts = f.code.consts - f.constsc = f.code.consts.count() - f.ip = 0 - f.sp = 0 - needed := max(f.code.maxStack, 4) - if cap(f.stack) < needed { - f.stack = make([]Value, needed) - } else { - f.stack = f.stack[:needed] - } -} - func NewDebugFrame(code *CodeChunk, args []Value) *Frame { f := NewFrame(code, args) f.debug = true @@ -670,58 +618,10 @@ func (f *Frame) RunProtected() (result Value, err error) { return f.Run() } -// suspendedCallArity validates the continuation stored in a suspended parent. -// Returning an error here is preferable to either panicking or trusting corrupt -// bytecode and silently dropping an arbitrary number of operand-stack values. -func suspendedCallArity(f *Frame) (int, *ExecutionError) { - if f == nil || f.code == nil { - return 0, NewExecutionError("invalid suspended call: missing frame code") - } - if f.ip < 0 || f.ip >= len(f.code.code) { - return 0, NewExecutionError(fmt.Sprintf("invalid suspended call: instruction pointer %d is out of bounds", f.ip)) - } - if f.ip+1 >= len(f.code.code) { - return 0, NewExecutionError(fmt.Sprintf("invalid suspended call at %d: missing arity operand", f.ip)) - } - if f.sp < 0 || f.sp > len(f.stack) { - return 0, NewExecutionError(fmt.Sprintf("invalid suspended call at %d: stack depth %d is out of bounds", f.ip, f.sp)) - } - op := f.code.code[f.ip] & 0xff - if op != OP_INVOKE && op != OP_TAIL_CALL { - return 0, NewExecutionError(fmt.Sprintf("invalid suspended call at %d: found %s", f.ip, OpcodeToString(f.code.code[f.ip]))) - } - arity := int(f.code.code[f.ip+1]) - if arity < 0 || arity+1 > f.sp { - return 0, NewExecutionError(fmt.Sprintf("invalid suspended call at %d: arity %d exceeds stack depth %d", f.ip, arity, f.sp)) - } - return arity, nil -} - -// wrapCallSite attributes err to the call currently under f.ip. The callee is -// still on f's operand stack (the call opcodes peek rather than pop), so the -// name is recoverable without extra bookkeeping. -func wrapCallSite(f *Frame, err error) error { - arity, siteErr := suspendedCallArity(f) - if siteErr != nil { - return NewExecutionError(siteErr.message).Wrap(err) - } - srcInfo := f.code.LookupSource(f.ip) - name := "fn" - calleeIndex := f.sp - 1 - arity - if fn, ok := AsFn(f.stack[calleeIndex]); ok { - name = fnName(fn) - } - return NewExecutionError(fmt.Sprintf("calling %s", name)).WithSource(srcInfo).Wrap(err) -} - -type frameRunState struct { - root *Frame - current *Frame -} - -func enterFrame(f *Frame) { +func (f *Frame) Run() (Value, error) { if allocAttrEnabled { attrPushFrame(f) + defer attrPopFrame() } // Dynamically-scoped tracing (*lg-trace*). Coarse gate first: TraceArmed is // false until *lg-trace* is first set truthy, so the precise per-frame Deref @@ -737,110 +637,21 @@ func enterFrame(f *Frame) { fmt.Print("run", f.args, "\n") f.code.Debug() } - // The newly entered frame starts a fresh opcode-pair sequence at zero. - // Suspended parents retain prevOp on their own Frame, so resuming records - // the next parent-local pair; cross-frame pairs are intentionally omitted. - f.prevOp = 0 - f.profileOn = ProfilingEnabled.Load() -} - -func leaveFrame(_ *Frame) { - if allocAttrEnabled { - attrPopFrame() - } -} - -// releaseFailedFrames drops the current failed frame and each unhandled -// suspended parent. The root is owned by Run's caller and is never pooled here. -// It returns the first parent whose handler accepts err, or nil if none does. -func releaseFailedFrames(state *frameRunState, err error) (*Frame, error) { - failed := state.current - parent := failed.parent - failed.parent = nil - if failed != state.root { - ReleaseFrame(failed) - } - for parent != nil { - next := parent.parent - err = wrapCallSite(parent, err) - if parent.handleError(err) { - state.current = parent - return parent, err - } - leaveFrame(parent) - parent.parent = nil - if parent != state.root { - ReleaseFrame(parent) - } - parent = next - } - return nil, err -} - -func releasePanickedFrames(state *frameRunState) { - failed := state.current - parent := failed.parent - failed.parent = nil - if failed != state.root { - ReleaseFrame(failed) - } - for parent != nil { - next := parent.parent - leaveFrame(parent) - parent.parent = nil - if parent != state.root { - ReleaseFrame(parent) - } - parent = next - } -} - -// Run executes this frame to completion. runLoop descends into bytecode -// callees within one dispatch loop, keeping the Go stack flat. A parent link -// on each live frame carries the suspended call chain without a retained slice. -func (f *Frame) Run() (result Value, resultErr error) { - f.parent = nil - state := frameRunState{root: f, current: f} - entering := true - defer func() { - if r := recover(); r != nil { - // runLoop's defer already left the current frame. Balance and release - // every suspended child-owned frame before preserving the panic. - releasePanickedFrames(&state) - panic(r) - } - }() - for { - result, resultErr = state.current.runLoop(&state, entering) - entering = false - if resultErr == nil { - return result, nil - } - var resumed *Frame - resumed, resultErr = releaseFailedFrames(&state, resultErr) - if resumed == nil { - return NIL, resultErr - } - } -} - -func (f *Frame) runLoop(state *frameRunState, entering bool) (Value, error) { - if entering { - enterFrame(f) - } - // f changes as the loop descends and returns. On a final return or error, - // leave whichever logical frame is current; suspended parents remain active. - defer func() { leaveFrame(f) }() + // Profiler state: previous opcode byte (0 at frame entry). Only + // touched when ProfilingEnabled — the load is a single atomic + // branch per opcode, well-predicted in the disabled case. + var prevOp uint8 + profileOn := ProfilingEnabled.Load() for { inst := f.code.code[f.ip] if f.debug { f.stackDbg() fmt.Println("#", f.ip, OpcodeToString(inst)) } - if f.profileOn { + if profileOn { currOp := uint8(inst & 0xff) - RecordOpcode(f.prevOp, currOp) - f.prevOp = currOp + RecordOpcode(prevOp, currOp) + prevOp = currOp } switch inst & 0xff { case OP_NOOP: @@ -873,30 +684,7 @@ func (f *Frame) runLoop(state *frameRunState, entering bool) (Value, error) { if err != nil { return NIL, NewExecutionError("return failed").Wrap(err) } - if f.parent == nil { - return v, nil - } - child := f - parent := child.parent - child.parent = nil - leaveFrame(child) - ReleaseFrame(child) - f = parent - state.current = f - callArity, callErr := suspendedCallArity(f) - if callErr != nil { - if f.handleError(callErr) { - continue - } - return NIL, callErr - } - if e := f.drop(callArity + 1); e != nil { - return NIL, NewExecutionError("cleaning stack after call").Wrap(e) - } - if e := f.push(v); e != nil { - return NIL, NewExecutionError("pushing return value failed").Wrap(e) - } - f.ip += 2 + return v, nil case OP_INVOKE: arity := f.code.code[f.ip+1] @@ -914,23 +702,6 @@ func (f *Frame) runLoop(state *frameRunState, entering bool) (Value, error) { if err != nil { return NIL, NewExecutionError("popping arguments failed").Wrap(err) } - target, direct, cerr := resolveBytecodeCall(fn, a) - if cerr != nil { - srcInfo := f.code.LookupSource(f.ip) - wrapped := NewExecutionError(fmt.Sprintf("calling %s", fnName(fn))).WithSource(srcInfo).Wrap(cerr) - if f.handleError(wrapped) { - continue - } - return NIL, wrapped - } - if direct { - child := newFrameForBytecodeCall(target, f.ec) - child.parent = f - f = child - state.current = f - enterFrame(f) - continue - } out, err = f.ec.Invoke(fn, a) if err != nil { srcInfo := f.code.LookupSource(f.ip) @@ -945,9 +716,7 @@ func (f *Frame) runLoop(state *frameRunState, entering bool) (Value, error) { return NIL, NewExecutionError("cleaning stack after call").Wrap(err) } } else { - // Peek rather than pop, so the callee slot remains for the - // uniform drop(arity+1) when a descended child returns. - fraw, err := f.nth(0) + fraw, err := f.pop() if err != nil { return NIL, NewExecutionError("invoke instruction failed").Wrap(err) } @@ -955,26 +724,6 @@ func (f *Frame) runLoop(state *frameRunState, entering bool) (Value, error) { if !ok { return NIL, NewTypeError(fraw, "is not a function", nil) } - target, direct, cerr := resolveBytecodeCall(fn, nil) - if cerr != nil { - srcInfo := f.code.LookupSource(f.ip) - wrapped := NewExecutionError(fmt.Sprintf("calling %s", fnName(fn))).WithSource(srcInfo).Wrap(cerr) - if f.handleError(wrapped) { - continue - } - return NIL, wrapped - } - if direct { - child := newFrameForBytecodeCall(target, f.ec) - child.parent = f - f = child - state.current = f - enterFrame(f) - continue - } - if _, perr := f.pop(); perr != nil { - return NIL, NewExecutionError("invoke instruction failed").Wrap(perr) - } out, err = f.ec.Invoke(fn, nil) if err != nil { srcInfo := f.code.LookupSource(f.ip) @@ -993,69 +742,118 @@ func (f *Frame) runLoop(state *frameRunState, entering bool) (Value, error) { case OP_TAIL_CALL: arity := f.code.code[f.ip+1] - // Keep the callee on the operand stack when descending so the - // return path can use one drop(arity+1) protocol for every arity. - fraw, err := f.nth(int(arity)) - if err != nil { - return NIL, NewExecutionError("invoke instruction failed").Wrap(err) - } - fn, ok := AsFn(fraw) - if !ok { - return NIL, NewTypeError(fraw, "is not a function", nil) - } - var a []Value + var out Value if arity > 0 { - a, err = f.mult(0, int(arity)) + fraw, err := f.nth(int(arity)) + if err != nil { + return NIL, NewExecutionError("invoke instruction failed").Wrap(err) + } + fn, ok := AsFn(fraw) + if !ok { + return NIL, NewTypeError(fraw, "is not a function", nil) + } + a, err := f.mult(0, int(arity)) if err != nil { return NIL, NewExecutionError("popping arguments failed").Wrap(err) } - } - - target, direct, resolveErr := resolveBytecodeCall(fn, a) - if resolveErr != nil { - srcInfo := f.code.LookupSource(f.ip) - wrapped := NewExecutionError(fmt.Sprintf("calling %s", fnName(fn))).WithSource(srcInfo).Wrap(resolveErr) - if f.handleError(wrapped) { + if _, ok := fn.(*Func); !ok { + out, err = f.ec.Invoke(fn, a) + if err != nil { + srcInfo := f.code.LookupSource(f.ip) + wrapped := NewExecutionError(fmt.Sprintf("calling %s", fnName(fn))).WithSource(srcInfo).Wrap(err) + if f.handleError(wrapped) { + continue + } + return NIL, wrapped + } + // TAIL_CALL is terminal: builtin's result is this + // frame's result. (The compiler still emits RETURN + // after TAIL_CALL but it's now dead code.) + return out, nil + } else { + ff := fn.(*Func) + // Package variadic args for direct frame reuse + if ff.isVariadric { + if len(a) < ff.arity-1 { + return NIL, NewExecutionError(fmt.Sprintf("function %s expected at least %d args, got %d", ff, ff.arity-1, len(a))) + } + sargs := a[0 : ff.arity-1] + rest := a[ff.arity-1:] + restlist, boxErr := boxRest(rest) + if boxErr != nil { + return NIL, boxErr + } + a = append(sargs, restlist) + } + f.code = ff.chunk + f.consts = f.code.consts + f.constsc = f.code.consts.count() + f.ip = 0 + f.sp = 0 + if len(f.stack) < f.code.maxStack { + f.stack = make([]Value, f.code.maxStack) + f.args = a + f.argc = len(a) + } else { + la := len(a) + if la <= f.argc { + copy(f.args, a) + } else { + f.args = make([]Value, la) + copy(f.args, a) + } + f.argc = len(a) + } continue } - return NIL, wrapped - } - if direct { - if _, reuse := fn.(*Func); reuse { - installBytecodeCall(f, target) - continue + } else { + fraw, err := f.pop() + if err != nil { + return NIL, NewExecutionError("invoke instruction failed").Wrap(err) } - // Closure, multi-arity, and metadata-wrapped bytecode - // callees descend without re-entering Frame.Run. #620 will - // move these targets onto the same-frame transition. - child := newFrameForBytecodeCall(target, f.ec) - child.parent = f - f = child - state.current = f - enterFrame(f) - continue - } - - out, err := f.ec.Invoke(fn, a) - if err != nil { - srcInfo := f.code.LookupSource(f.ip) - wrapped := NewExecutionError(fmt.Sprintf("calling %s", fnName(fn))).WithSource(srcInfo).Wrap(err) - if f.handleError(wrapped) { + fn, ok := AsFn(fraw) + if !ok { + return NIL, NewTypeError(fraw, "is not a function", nil) + } + if _, ok := fn.(*Func); !ok { + out, err = f.ec.Invoke(fn, nil) + if err != nil { + srcInfo := f.code.LookupSource(f.ip) + wrapped := NewExecutionError(fmt.Sprintf("calling %s", fnName(fn))).WithSource(srcInfo).Wrap(err) + if f.handleError(wrapped) { + continue + } + return NIL, wrapped + } + // TAIL_CALL is terminal: builtin's result is this + // frame's result. + return out, nil + } else { + ff := fn.(*Func) + // Package the (nil) rest binding for variadic + // functions, exactly like the arity > 0 path above; + // without it the reused frame runs LOAD_ARG against + // zero args. + var a []Value + if ff.isVariadric { + if ff.arity > 1 { + return NIL, NewExecutionError(fmt.Sprintf("function %s expected at least %d args, got 0", ff, ff.arity-1)) + } + a = []Value{NIL} + } + f.code = ff.chunk + f.consts = f.code.consts + f.constsc = f.code.consts.count() + f.ip = 0 + f.sp = 0 + if len(f.stack) < f.code.maxStack { + f.stack = make([]Value, f.code.maxStack) + } + f.args = a + f.argc = len(a) continue } - return NIL, wrapped - } - // A native/non-bytecode tail call is terminal. Land its result - // on the compiler-emitted RETURN so frame-chain unwind remains - // centralized in OP_RETURN. - if e := f.drop(int(arity) + 1); e != nil { - return NIL, NewExecutionError("cleaning stack after call").Wrap(e) } - if e := f.push(out); e != nil { - return NIL, NewExecutionError("pushing return value failed").Wrap(e) - } - f.ip += 2 - continue case OP_BRANCH_TRUE: offset := f.code.code[f.ip+1]