Include shell source and build source directories in proc.Spawn (#102)

* dev: refactor initdata

I needed to make some changes to initdata for #89, and in the process
made it simpler. Now the list of files is in one place, and we always
embed them into the bootloader. However, for debug builds we default
to embedding just the filepaths instead of their contents,
and fetching them at runtime. You still have the option of embedding
the file contents by setting the `PROD` environment variable.
This supports previous behavior while simplifying the codepaths.

* Embed shell source instead of binary and make spawn build source directories

Closes #88
Closes #89

I though it was a bit silly casting `s.fsys` everywhere,
so I modifed `fs.go` to store the `watchfs` seperately.

There was also an issue in `wasm.js:writeSync()` where it would be passed
a null value for `buf` after the build tool finished in `proc.go:Spawn()`.
I added a catch-all fix, but perhaps its a bug that the null is passed in the
first place.

There's still some kinks to work out before you can edit the shell seamlessly,
namely using better heuristics for when files should be updated/rebuilt. We can use
mtime/ctime for those.

* kernel: compare `mtime`'s when building source or copying files from `initfs`

Closes #66

Enables proper live editing within Wanix! `proc.Spawn` will rebuild commands
from source if it's been modified, and `kernel.fs` will copy in `initfs` files
if they've been modified externally *(note this will overwrite local files
inside Wanix)*.

Author: Parzival-3141

dev/bootloader.js

@@ -81,31 +81,24 @@ if (!globalThis["ServiceWorkerGlobalScope"]) {
     await setupServiceWorker();
 
     globalThis.initfs = {};
-    const load = async (path) => {
-      const basename = (path) => path.replace(/\\/g,'/').split('/').pop();
-      if (globalThis.initdata) {
-        // use embedded data if present
-        path = `./~init/${basename(path)}`;
+    const load = async (name, file) => {
+      // Determine if file contains a path to fetch or embedded file contents to load.
+      if(file.type === "text/plain") {
+        globalThis.initfs[name] = { mtimeMs: file.mtimeMs, blob: await (await fetch(`./sys/dev/${file.data}`)).blob() };
+      } else {
+        globalThis.initfs[name] = { mtimeMs: file.mtimeMs, blob: await (await fetch(`./~init/${name}`)).blob() };
       }
-      globalThis.initfs[basename(path)] = await (await fetch(path)).blob();
+    };
+
+    // allow loading concurrently
+    let loads = [];
+    for(const property in globalThis.initdata) {
+      loads.push(load(property, globalThis.initdata[property]));
     }
-    // TODO: define these in one place. duplicated in initdata.go
-    await Promise.all([
-      load("./sys/dev/kernel/web/lib/duplex.js"),
-      load("./sys/dev/kernel/web/lib/worker.js"),
-      load("./sys/dev/kernel/web/lib/syscall.js"),
-      load("./sys/dev/kernel/web/lib/task.js"),
-      load("./sys/dev/kernel/web/lib/wasm.js"),
-      load("./sys/dev/kernel/web/lib/host.js"),
-      load("./sys/dev/internal/indexedfs/indexedfs.js"), // maybe load from kernel?
-      load("./sys/dev/local/bin/kernel"),
-      load("./sys/dev/local/bin/shell"),
-      load("./sys/dev/local/bin/build"),
-      load("./sys/dev/local/bin/micro"),
-    ]);
-    
-    globalThis.duplex = await import(URL.createObjectURL(initfs["duplex.js"]));
-    globalThis.task = await import(URL.createObjectURL(initfs["task.js"]));
+    await Promise.all(loads);
+
+    globalThis.duplex = await import(URL.createObjectURL(initfs["duplex.js"].blob));
+    globalThis.task = await import(URL.createObjectURL(initfs["task.js"].blob));
 
     globalThis.sys = new task.Task(initfs);
 
@@ -114,7 +107,7 @@ if (!globalThis["ServiceWorkerGlobalScope"]) {
     await sys.exec("kernel");
 
     // load host API
-    await import(URL.createObjectURL(initfs["host.js"]));
+    await import(URL.createObjectURL(initfs["host.js"].blob));
 
   })();
 }

dev/bundle.go

@@ -14,7 +14,7 @@ func main() {
 	fatal(err)
 	defer f.Close()
 
-	PackFilesTo(f)
+	PackFilesTo(f, PackFileData)
 
 	src, err := os.ReadFile("./dev/bootloader.js")
 	fatal(err)

dev/initdata.go

@@ -7,42 +7,75 @@ import (
 	"io"
 	"log"
 	"os"
-	"path/filepath"
 	"strings"
 	"text/template"
 )
 
-func PackFilesTo(w io.Writer) {
-	var files []File
-	for _, path := range []string{
-		"./kernel/web/lib/duplex.js",
-		"./kernel/web/lib/worker.js",
-		"./kernel/web/lib/syscall.js",
-		"./kernel/web/lib/task.js",
-		"./kernel/web/lib/wasm.js",
-		"./kernel/web/lib/host.js",
-		"./internal/indexedfs/indexedfs.js",
-		"./local/bin/kernel",
-		"./local/bin/shell",
-		"./local/bin/build",
-		"./local/bin/micro",
-	} {
-		typ := "application/octet-stream"
-		if strings.HasSuffix(path, ".js") {
-			typ = "application/javascript"
+// Files must be local to the Wanix project root.
+var files = []File{
+	{Name: "duplex.js", Path: "./kernel/web/lib/duplex.js"},
+	{Name: "worker.js", Path: "./kernel/web/lib/worker.js"},
+	{Name: "syscall.js", Path: "./kernel/web/lib/syscall.js"},
+	{Name: "task.js", Path: "./kernel/web/lib/task.js"},
+	{Name: "wasm.js", Path: "./kernel/web/lib/wasm.js"},
+	{Name: "host.js", Path: "./kernel/web/lib/host.js"},
+	{Name: "indexedfs.js", Path: "./internal/indexedfs/indexedfs.js"},
+	{Name: "kernel", Path: "./local/bin/kernel"},
+	{Name: "build", Path: "./local/bin/build"},
+	{Name: "macro", Path: "./local/bin/micro"},
+
+	// Shell source files
+	{Name: "shell/main.go", Path: "shell/main.go"},
+	{Name: "shell/copy.go", Path: "shell/copy.go"},
+	{Name: "shell/download.go", Path: "shell/download.go"},
+	{Name: "shell/main.go", Path: "shell/main.go"},
+	{Name: "shell/open.go", Path: "shell/open.go"},
+	{Name: "shell/preprocessor.go", Path: "shell/preprocessor.go"},
+	{Name: "shell/smallcmds.go", Path: "shell/smallcmds.go"},
+	{Name: "shell/tree.go", Path: "shell/tree.go"},
+	{Name: "shell/util.go", Path: "shell/util.go"},
+	{Name: "shell/watch.go", Path: "shell/watch.go"},
+}
+
+type PackMode int
+
+const (
+	PackFileData PackMode = iota
+	PackFilePaths
+)
+
+func PackFilesTo(w io.Writer, mode PackMode) {
+	switch mode {
+	case PackFileData:
+		for i := range files {
+			if strings.HasSuffix(files[i].Path, ".js") {
+				files[i].Type = "application/javascript"
+			} else {
+				files[i].Type = "application/octet-stream"
+			}
+
+			fi, err := os.Stat(files[i].Path)
+			fatal(err)
+			files[i].Mtime = fi.ModTime().UnixMilli()
+
+			data, err := os.ReadFile(files[i].Path)
+			fatal(err)
+			var gzipBuffer bytes.Buffer
+			gzipWriter := gzip.NewWriter(&gzipBuffer)
+			_, err = gzipWriter.Write(data)
+			fatal(err)
+			fatal(gzipWriter.Close())
+			files[i].Data = base64.StdEncoding.EncodeToString(gzipBuffer.Bytes())
+		}
+
+	case PackFilePaths:
+		for i := range files {
+			files[i].Type = "text/plain"
+			files[i].Data = files[i].Path
+			fi, err := os.Stat(files[i].Path)
+			fatal(err)
+			files[i].Mtime = fi.ModTime().UnixMilli()
 		}
-		data, err := os.ReadFile(path)
-		fatal(err)
-		var gzipBuffer bytes.Buffer
-		gzipWriter := gzip.NewWriter(&gzipBuffer)
-		_, err = gzipWriter.Write(data)
-		fatal(err)
-		fatal(gzipWriter.Close())
-		files = append(files, File{
-			Name: filepath.Base(path),
-			Type: typ,
-			Data: base64.StdEncoding.EncodeToString(gzipBuffer.Bytes()),
-		})
 	}
 
 	t := template.Must(template.New("initdata.tmpl").ParseFiles("./dev/initdata.tmpl"))
@@ -52,9 +85,11 @@ func PackFilesTo(w io.Writer) {
 }
 
 type File struct {
-	Name string
-	Type string
-	Data string
+	Name  string
+	Path  string
+	Type  string
+	Data  string
+	Mtime int64
 }
 
 func fatal(err error) {

dev/initdata.tmpl

@@ -1,5 +1,5 @@
 globalThis.initdata = {
 {{range .}}
-  "{{.Name}}": {type: "{{.Type}}", data: "{{.Data}}"},
+  "{{.Name}}": {type: "{{.Type}}", mtimeMs: {{.Mtime}}, data: "{{.Data}}"},
 {{end}}
 }

dev/server.go

@@ -38,17 +38,21 @@ func main() {
 	mux.Handle(fmt.Sprintf("%s/wanix-bootloader.js", basePath), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("content-type", "application/javascript")
 
-		if os.Getenv("PROD") != "1" {
-			http.ServeFile(w, r, "./dev/bootloader.js")
-			return
+		var packMode PackMode
+		if os.Getenv("PROD") == "1" {
+			log.Printf("Packing self-contained bootloader...\n")
+			packMode = PackFileData
+		} else {
+			packMode = PackFilePaths
 		}
 
-		// emulate a build
-		PackFilesTo(w)
+		// TODO: Does this need to pack on every request for the bootloader?
+		// I don't think you want to be changing PROD at runtime, so we can
+		// probably cache this.
+		PackFilesTo(w, packMode)
 		f, err := os.ReadFile("./dev/bootloader.js")
 		fatal(err)
 		w.Write(f)
-
 	}))
 	mux.Handle(fmt.Sprintf("%s/~init/", basePath), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "Not found", http.StatusNotFound)

internal/app/terminal/index.html

@@ -61,7 +61,7 @@
     const enc = new TextEncoder();
     const dec = new TextDecoder();
 
-    const resp = await sys.call("tty.open", ["shell", [], { TERM: "xterm-256color" }]);
+    const resp = await sys.call("tty.open", ["sys/cmd/shell", [], { TERM: "xterm-256color" }]);
     const pid = resp.value;
     const ch = resp.channel;
     //parent.wanix.termCh = ch;

internal/indexedfs/indexedfs.go

@@ -54,7 +54,7 @@ type FS struct {
 
 func New() (*FS, error) {
 	if helper.IsUndefined() {
-		blob := js.Global().Get("initfs").Get("indexedfs.js")
+		blob := js.Global().Get("initfs").Get("indexedfs.js").Get("blob")
 		url := js.Global().Get("URL").Call("createObjectURL", blob)
 		helper = jsutil.Await(js.Global().Call("import", url))
 	}