feat(lgbgen): compress the embedded core bundle (default off)

cmd/lgbgen/atomic_write_test.go

@@ -0,0 +1,83 @@
+//go:build bootstrap
+
+package main
+
+import (
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestWriteFileAtomicallyPreservesDestinationOnFailure(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "core_compiled.lgb")
+	if err := os.WriteFile(path, []byte("original"), 0o640); err != nil {
+		t.Fatal(err)
+	}
+
+	wantErr := errors.New("encode failed")
+	_, err := writeFileAtomically(path, func(w io.Writer) error {
+		if _, err := io.WriteString(w, "partial"); err != nil {
+			return err
+		}
+		return wantErr
+	})
+	if !errors.Is(err, wantErr) {
+		t.Fatalf("writeFileAtomically() error = %v, want %v", err, wantErr)
+	}
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(got) != "original" {
+		t.Fatalf("destination = %q after failed write, want original", got)
+	}
+	assertNoAtomicTemps(t, dir)
+}
+
+func TestWriteFileAtomicallyReplacesDestination(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "core_compiled.lgb")
+	if err := os.WriteFile(path, []byte("old"), 0o640); err != nil {
+		t.Fatal(err)
+	}
+
+	size, err := writeFileAtomically(path, func(w io.Writer) error {
+		_, err := io.WriteString(w, "replacement")
+		return err
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != int64(len("replacement")) {
+		t.Fatalf("size = %d, want %d", size, len("replacement"))
+	}
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(got) != "replacement" {
+		t.Fatalf("destination = %q, want replacement", got)
+	}
+	info, err := os.Stat(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if gotMode := info.Mode().Perm(); gotMode != 0o640 {
+		t.Fatalf("destination mode = %o, want 640", gotMode)
+	}
+	assertNoAtomicTemps(t, dir)
+}
+
+func assertNoAtomicTemps(t *testing.T, dir string) {
+	t.Helper()
+	matches, err := filepath.Glob(filepath.Join(dir, ".core_compiled.lgb.tmp-*"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(matches) != 0 {
+		t.Fatalf("temporary files left behind: %v", matches)
+	}
+}

cmd/lgbgen/main.go

@@ -15,6 +15,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
 	"io/fs"
 	"os"
 	"os/signal"
@@ -388,6 +389,13 @@ func main() {
 	fs.StringVar(&cpuProfilePath, "cpuprofile", "", "write Go CPU profile for the lgbgen process")
 	fs.StringVar(&memProfilePath, "memprofile", "", "write Go allocation profile (allocs) for the lgbgen process")
 	fs.StringVar(&codeDir, "code-dir", "", "base dir for generated gogen_ir wireup files (default: repo root)")
+	// --compress DEFLATE-compresses the bundle body. Default OFF for the
+	// committed core: it trims the binary but adds work and allocations to every
+	// process start (the core is decoded on every boot), and that tradeoff is a
+	// policy call for the maintainer, not a silent default. Decode inflates it
+	// transparently either way (see the FlagCompressed path in pkg/bytecode).
+	compress := false
+	fs.BoolVar(&compress, "compress", false, "DEFLATE-compress the .lgb bundle body (smaller binary, slower boot)")
 	fs.Parse(os.Args[1:])
 
 	switch target {
@@ -568,44 +576,81 @@ func main() {
 		return
 	}
 	if targetBoth {
-		writeBundle(outPath, consts, nsChunks, bundleOrder)
+		writeBundle(outPath, consts, nsChunks, bundleOrder, compress)
 		compileIRForLowering()
 		runGoTarget(goOutDir, codeDir)
 		return
 	}
 
 	// Bytecode mode: write .lgb bundle (ir.* excluded).
-	writeBundle(outPath, consts, nsChunks, bundleOrder)
+	writeBundle(outPath, consts, nsChunks, bundleOrder, compress)
 }
 
 // writeBundle encodes the compiled namespace chunks into the .lgb bundle at
-// outPath and closes the file before returning (so callers may proceed to the
-// Go-lowering target against the same in-memory state).
-func writeBundle(outPath string, consts *vm.Consts, nsChunks map[string]*vm.CodeChunk, nsOrder []string) {
-	f, err := os.Create(outPath)
+// outPath atomically before returning (so callers may proceed to the Go-lowering
+// target against the same in-memory state). Encoding happens in a temporary
+// file in the destination directory; the previous bundle remains intact unless
+// encoding, flushing, and closing all succeed.
+func writeBundle(outPath string, consts *vm.Consts, nsChunks map[string]*vm.CodeChunk, nsOrder []string, compress bool) {
+	// The core bundle is decoded on every process start, so the decode path
+	// inflates it transparently; compressing here trims the embedded bytes from
+	// every binary (see the FlagCompressed decode path in pkg/bytecode).
+	size, err := writeFileAtomically(outPath, func(w io.Writer) error {
+		return bytecode.EncodeBundleOrderedCompressed(w, consts, nsChunks, nsOrder, compress)
+	})
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "create %s: %v\n", outPath, err)
+		fmt.Fprintf(os.Stderr, "write %s: %v\n", outPath, err)
 		os.Exit(1)
 	}
 
-	if err := bytecode.EncodeBundleOrdered(f, consts, nsChunks, nsOrder); err != nil {
-		f.Close()
-		fmt.Fprintf(os.Stderr, "encode failed: %v\n", err)
-		os.Exit(1)
+	fmt.Printf("wrote %s (%d bytes, %d consts, %d namespaces)\n",
+		outPath, size, len(consts.Values()), len(nsChunks))
+	refreshManifest()
+}
+
+// writeFileAtomically writes path through a same-directory temporary file and
+// replaces path only after the contents are synced and closed successfully.
+// Any failure leaves an existing destination untouched.
+func writeFileAtomically(path string, write func(io.Writer) error) (int64, error) {
+	mode := fs.FileMode(0o644)
+	if info, err := os.Stat(path); err == nil {
+		mode = info.Mode().Perm()
+	} else if !os.IsNotExist(err) {
+		return 0, fmt.Errorf("stat destination: %w", err)
 	}
 
-	// Stat is best-effort: success here is just for the byte-count in the
-	// success line. If it fails, we still wrote the bundle, so report what we
-	// know without dereferencing a nil FileInfo.
-	if fi, err := f.Stat(); err == nil {
-		fmt.Printf("wrote %s (%d bytes, %d consts, %d namespaces)\n",
-			outPath, fi.Size(), len(consts.Values()), len(nsChunks))
-	} else {
-		fmt.Printf("wrote %s (%d consts, %d namespaces; stat failed: %v)\n",
-			outPath, len(consts.Values()), len(nsChunks), err)
+	dir := filepath.Dir(path)
+	tmp, err := os.CreateTemp(dir, "."+filepath.Base(path)+".tmp-*")
+	if err != nil {
+		return 0, fmt.Errorf("create temporary file: %w", err)
 	}
-	f.Close()
-	refreshManifest()
+	tmpName := tmp.Name()
+	defer os.Remove(tmpName)
+
+	closeOnError := func(err error) (int64, error) {
+		_ = tmp.Close()
+		return 0, err
+	}
+	if err := write(tmp); err != nil {
+		return closeOnError(fmt.Errorf("encode temporary file: %w", err))
+	}
+	if err := tmp.Sync(); err != nil {
+		return closeOnError(fmt.Errorf("sync temporary file: %w", err))
+	}
+	info, err := tmp.Stat()
+	if err != nil {
+		return closeOnError(fmt.Errorf("stat temporary file: %w", err))
+	}
+	if err := tmp.Chmod(mode); err != nil {
+		return closeOnError(fmt.Errorf("set temporary file mode: %w", err))
+	}
+	if err := tmp.Close(); err != nil {
+		return 0, fmt.Errorf("close temporary file: %w", err)
+	}
+	if err := os.Rename(tmpName, path); err != nil {
+		return 0, fmt.Errorf("replace destination: %w", err)
+	}
+	return info.Size(), nil
 }
 
 // refreshManifest records the content digest of all .lg + lgbgen sources

docs/guide/usage.md

@@ -24,11 +24,17 @@ lg -r myfile.lg                   # run file, then REPL
 ```bash
 lg -c app.lgb app.lg              # compile to bytecode
 lg app.lgb                        # run bytecode
+lg -c app.lgb -z app.lg           # compile with a compressed bytecode body
 
 lg -b myapp app.lg                # bundle into a self-contained binary
+lg -b myapp -z app.lg             # bundle with compressed bytecode
 ./myapp                           # runs anywhere, no lg needed
 ```
 
+`-z` is opt-in and applies to `-c` and `-b`. The header remains plaintext for
+early compatibility checks; the bytecode body is compressed with DEFLATE and
+inflated transparently by `lg` and `lg-runtime`.
+
 The standalone binary is a copy of `lg` with your bytecode appended — copy it to
 another machine and it runs.
 

lg.go

@@ -153,11 +153,11 @@ func bundleBinary(ctx *compiler.Context, nsRes *resolver.NSResolver, src string,
 		maps.Copy(nsChunks, nsRes.LoadedChunks)
 		nsChunks[mainNS] = chunk
 		nsOrder := append(nsRes.LoadOrder, mainNS)
-		if err := bytecode.EncodeBundleOrdered(&lgbBuf, ctx.Consts(), nsChunks, nsOrder); err != nil {
+		if err := bytecode.EncodeBundleOrderedCompressed(&lgbBuf, ctx.Consts(), nsChunks, nsOrder, compressBundle); err != nil {
 			return err
 		}
 	} else {
-		if err := bytecode.EncodeCompilation(&lgbBuf, ctx.Consts(), chunk); err != nil {
+		if err := bytecode.EncodeCompilationCompressed(&lgbBuf, ctx.Consts(), chunk, compressBundle); err != nil {
 			return err
 		}
 	}
@@ -242,9 +242,9 @@ func compileLG(ctx *compiler.Context, nsRes *resolver.NSResolver, src string, ds
 		maps.Copy(nsChunks, nsRes.LoadedChunks)
 		nsChunks[mainNS] = chunk
 		nsOrder := append(nsRes.LoadOrder, mainNS)
-		return bytecode.EncodeBundleOrdered(out, ctx.Consts(), nsChunks, nsOrder)
+		return bytecode.EncodeBundleOrderedCompressed(out, ctx.Consts(), nsChunks, nsOrder, compressBundle)
 	}
-	return bytecode.EncodeCompilation(out, ctx.Consts(), chunk)
+	return bytecode.EncodeCompilationCompressed(out, ctx.Consts(), chunk, compressBundle)
 }
 
 var nreplServer *nrepl.NreplServer
@@ -272,6 +272,7 @@ var debug bool
 var showVersion bool
 var compileOutput string
 var bundleOutput string
+var compressBundle bool
 var bundleBase string
 var wasmOutput string
 var wasmShell string
@@ -291,6 +292,7 @@ func init() {
 	flag.BoolVar(&showVersion, "version", false, "print version and exit")
 	flag.StringVar(&compileOutput, "c", "", "compile .lg file to .lgb bytecode (specify output path)")
 	flag.StringVar(&bundleOutput, "b", "", "bundle .lg file into a standalone executable (specify output path)")
+	flag.BoolVar(&compressBundle, "z", false, "with -c/-b: DEFLATE-compress the bundle body (smaller .lgb / standalone binary; transparently inflated at load)")
 	flag.StringVar(&bundleBase, "bundle-base", "", "path to target-platform lg binary for cross-OS bundling (defaults to current executable)")
 	flag.StringVar(&wasmOutput, "w", "", "build .lg file into a WASM web app (specify output directory)")
 	flag.StringVar(&wasmShell, "w-shell", "xterm", "shell for -w: 'xterm' (default), 'none' (emit core only; client supplies its own shell via window.LetGoHost), or a path to a custom HTML template containing __LG_HOST_JS_BODY_PLACEHOLDER__")
@@ -435,6 +437,10 @@ func runMain() int {
 	}
 
 	flag.Parse()
+	if compressBundle && compileOutput == "" && bundleOutput == "" {
+		fmt.Fprintln(os.Stderr, "error: -z requires -c or -b")
+		return 2
+	}
 
 	if showVersion {
 		fmt.Print(bytecode.FormatVersionReport("lg", versionString()))

pkg/bytecode/bench_test.go

@@ -28,6 +28,78 @@ func BenchmarkDecodeCore(b *testing.B) {
 	}
 }
 
+// BenchmarkDecodeCoreCompressed decodes a DEFLATE-compressed copy of the core
+// bundle, so the delta against BenchmarkDecodeCore is the boot-time cost of
+// shipping the embedded core compressed (it is decoded on every process start).
+func BenchmarkDecodeCoreCompressed(b *testing.B) {
+	b.ReportAllocs()
+	corePath := filepath.Join("..", "rt", "core_compiled.lgb")
+	if _, err := os.Stat(corePath); os.IsNotExist(err) {
+		b.Skip("core_compiled.lgb not found at expected path")
+	}
+	data, err := os.ReadFile(corePath)
+	if err != nil {
+		b.Fatal(err)
+	}
+	// Re-encode the committed (uncompressed) core with FlagCompressed so the
+	// comparison is the same content through the compressed path.
+	m, err := Decode(bytes.NewReader(data))
+	if err != nil {
+		b.Fatal(err)
+	}
+	m.Flags |= FlagCompressed
+	var comp bytes.Buffer
+	if err := Encode(&comp, m); err != nil {
+		b.Fatal(err)
+	}
+	compressed := comp.Bytes()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if _, err := Decode(bytes.NewReader(compressed)); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkEncodeCore(b *testing.B) {
+	benchmarkEncodeCore(b, false)
+}
+
+func BenchmarkEncodeCoreCompressed(b *testing.B) {
+	benchmarkEncodeCore(b, true)
+}
+
+func benchmarkEncodeCore(b *testing.B, compressed bool) {
+	b.Helper()
+	b.ReportAllocs()
+	corePath := filepath.Join("..", "rt", "core_compiled.lgb")
+	data, err := os.ReadFile(corePath)
+	if os.IsNotExist(err) {
+		b.Skip("core_compiled.lgb not found at expected path")
+	}
+	if err != nil {
+		b.Fatal(err)
+	}
+	m, err := Decode(bytes.NewReader(data))
+	if err != nil {
+		b.Fatal(err)
+	}
+	if compressed {
+		m.Flags |= FlagCompressed
+	}
+
+	b.SetBytes(int64(len(data)))
+	var out bytes.Buffer
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		out.Reset()
+		if err := Encode(&out, m); err != nil {
+			b.Fatal(err)
+		}
+	}
+	b.ReportMetric(float64(out.Len()), "bytes/output")
+}
+
 func BenchmarkDecodeModuleSmall(b *testing.B) {
 	b.ReportAllocs()
 	mb := NewModuleBuilder()

pkg/bytecode/capabilities.go

@@ -110,11 +110,15 @@ func UnsupportedCapabilityError(caps uint32) error {
 
 // FormatVersionReport returns the multi-line -v/--version body for tool
 // (e.g. "lg" or "lg-runtime") at the given version string.
+//
+// The lgb line reports both the default write version (plain bundles stay on
+// uncompressedFormatVersion) and FormatVersion (the newest this tree reads and
+// writes when a newer-only flag such as FlagCompressed is set).
 func FormatVersionReport(tool, versionStr string) string {
 	count, hash := vm.OpcodeSetSignature()
 	var b strings.Builder
 	fmt.Fprintf(&b, "%s %s\n", tool, versionStr)
-	fmt.Fprintf(&b, "lgb: format %d\n", FormatVersion)
+	fmt.Fprintf(&b, "lgb: format %d (default write), %d (max)\n", uncompressedFormatVersion, FormatVersion)
 	fmt.Fprintf(&b, "capabilities: %s\n", DescribeCapabilities(SupportedCapabilities))
 	fmt.Fprintf(&b, "opcodes: %d (signature %016x)\n", count, hash)
 	return b.String()

pkg/bytecode/capabilities_test.go

@@ -1,6 +1,7 @@
 package bytecode
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 )
@@ -50,7 +51,7 @@ func TestFormatVersionReport(t *testing.T) {
 	got := FormatVersionReport("lg", "1.99.0 (deadbee)")
 	for _, want := range []string{
 		"lg 1.99.0 (deadbee)\n",
-		"lgb: format 2\n",
+		fmt.Sprintf("lgb: format %d (default write), %d (max)\n", uncompressedFormatVersion, FormatVersion),
 		"capabilities: CapOpcodeSet",
 		"opcodes:",
 		"signature",