Merge pull request #164 from gadget-inc/optional-esm

optional esm

Author: airhorns

.github/workflows/test.yml

@@ -13,8 +13,8 @@ jobs:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/setup-test-env
       - run: pnpm build
-      - run: pnpm zx integration-test/test.js
       - run: pnpm test
+      - run: pnpm integration-test
 
   lint:
     runs-on: ubuntu-latest

Readme.md

@@ -24,6 +24,7 @@ wds --inspect some-test.test.ts
 - Builds and runs TypeScript really fast using [`swc`](https://github.com/swc-project/swc)
 - Incrementally rebuilds only what has changed in `--watch` mode, restarting the process on file changes
 - Full support for CommonJS and ESM packages (subject to node's own interoperability rules)
+- Caches transformed files on disk for warm startups on process reload (with expiry when config or source changes)
 - Execute commands on demand with the `--commands` mode
 - Plays nice with node.js command line flags like `--inspect` or `--prof`
 - Supports node.js `ipc` channels between the process starting `wds` and the node.js process started by `wds`.

integration-test/cjs-with-esm-disabled/.swcrc

@@ -0,0 +1,15 @@
+{
+  "jsc": {
+    "parser": {
+      "syntax": "typescript",
+      "tsx": true,
+      "decorators": true,
+      "dynamicImport": true
+    },
+    "target": "esnext"
+  },
+  "module": {
+    "type": "commonjs",
+    "lazy": true
+  }
+}

integration-test/cjs-with-esm-disabled/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "simple",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "type": "commonjs",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "license": "ISC"
+}

integration-test/cjs-with-esm-disabled/run.ts

@@ -0,0 +1,3 @@
+import { utility } from "./utils";
+
+console.log(utility("It worked!"));

integration-test/cjs-with-esm-disabled/test.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+set -ex
+
+$DIR/../../pkg/wds.bin.js $@ $DIR/run.ts | grep "IT WORKED"

integration-test/cjs-with-esm-disabled/utils.ts

@@ -0,0 +1 @@
+export const utility = (str: string) => str.toUpperCase();

integration-test/cjs-with-esm-disabled/wds.js

@@ -0,0 +1,3 @@
+module.exports = {
+    esm: false,
+};

integration-test/oom/test.sh

@@ -33,6 +33,7 @@
     "lint:fix": "NODE_OPTIONS=\"--max-old-space-size=4096\" prettier --write --check \"src/**/*.{js,ts,tsx}\" && eslint --ext ts,tsx --fix src",
     "prerelease": "gitpkg publish",
     "test": "vitest run",
+    "integration-test": "pnpm run build && zx integration-test/test.js",
     "test:watch": "vitest"
   },
   "engines": {
@@ -47,9 +48,11 @@
     "globby": "^11.1.0",
     "lodash": "^4.17.20",
     "oxc-resolver": "^1.12.0",
+    "node-object-hash": "^3.0.0",
     "pkg-dir": "^5.0.0",
     "watcher": "^2.3.1",
     "write-file-atomic": "^6.0.0",
+    "xxhash-wasm": "^1.0.2",
     "yargs": "^16.2.0"
   },
   "devDependencies": {

integration-test/parent-crash/test.js

@@ -13,7 +13,7 @@ const compile = async (filename: string, root = "fixtures/src") => {
   const rootDir = path.join(dirname, root);
   const fullPath = path.join(rootDir, filename);
 
-  const compiler = new SwcCompiler(rootDir, workDir);
+  const compiler = await SwcCompiler.create(rootDir, workDir);
   await compiler.compile(fullPath);
   const compiledFilePath = (await compiler.fileGroup(fullPath))[fullPath]!;
 
@@ -58,7 +58,7 @@ test("logs error when a file in group fails compilation but continues", async ()
   const workDir = await fs.mkdtemp(path.join(os.tmpdir(), "wds-test"));
   const rootDir = path.join(dirname, "fixtures/failing");
   const fullPath = path.join(rootDir, "successful.ts");
-  const compiler = new SwcCompiler(rootDir, workDir);
+  const compiler = await SwcCompiler.create(rootDir, workDir);
   await compiler.compile(fullPath);
   const group = await compiler.fileGroup(fullPath);
 

integration-test/reload/test.sh

@@ -11,5 +11,7 @@ export interface RunOptions {
 export interface ProjectConfig {
   ignore: string[];
   swc?: SwcConfig;
+  esm?: boolean;
   extensions: string[];
+  cacheDir: string;
 }

integration-test/server/test.sh

@@ -66,6 +66,7 @@ export class Supervisor extends EventEmitter {
         ...process.env,
         WDS_SOCKET_PATH: this.socketPath,
         WDS_EXTENSIONS: this.project.config.extensions.join(","),
+        WDS_ESM_ENABLED: this.project.config.esm ? "true" : "false",
       },
       stdio: stdio,
       detached: true,

package.json

@@ -1,14 +1,29 @@
 import type { Config, Options } from "@swc/core";
-import { transformFile } from "@swc/core";
+import { transform } from "@swc/core";
+import { createRequire } from "node:module";
+import type { XXHashAPI } from "xxhash-wasm";
+import xxhash from "xxhash-wasm";
+
 import findRoot from "find-root";
 import * as fs from "fs/promises";
 import globby from "globby";
+import _ from "lodash";
+import { hasher } from "node-object-hash";
 import path from "path";
+import { fileURLToPath } from "url";
 import writeFileAtomic from "write-file-atomic";
 import type { Compiler } from "./Compiler.js";
 import type { ProjectConfig } from "./Options.js";
 import { log, projectConfig } from "./utils.js";
 
+const __filename = fileURLToPath(import.meta.url);
+const require = createRequire(import.meta.url);
+
+const getPackageVersion = async (packageDir: string) => {
+  const packageJson = JSON.parse(await fs.readFile(path.join(packageDir, "package.json"), "utf-8"));
+  return packageJson.version;
+};
+
 export class MissingDestinationError extends Error {
   ignoredFile: boolean;
 
@@ -83,11 +98,43 @@ class CompiledFiles {
 export class SwcCompiler implements Compiler {
   private compiledFiles: CompiledFiles;
   private invalidatedFiles: Set<string>;
+  private knownCacheEntries = new Set<string>();
+
+  static async create(workspaceRoot: string, outDir: string) {
+    const compiler = new SwcCompiler(workspaceRoot, outDir);
+    await compiler.initialize();
+    return compiler;
+  }
+
+  /** @private */
   constructor(readonly workspaceRoot: string, readonly outDir: string) {
     this.compiledFiles = new CompiledFiles();
     this.invalidatedFiles = new Set();
   }
 
+  private xxhash!: XXHashAPI;
+  private cacheEpoch!: string;
+
+  async initialize() {
+    this.xxhash = await xxhash();
+    try {
+      const files = await globby(path.join(this.outDir, "*", "*"), { onlyFiles: true });
+      for (const file of files) {
+        this.knownCacheEntries.add(path.basename(file));
+      }
+    } catch (error) {
+      // no complaints if the cache dir doesn't exist yet
+    }
+
+    // Get package versions for cache keys
+    const [thisPackageVersion, swcCoreVersion] = await Promise.all([
+      getPackageVersion(findRoot(__filename)),
+      getPackageVersion(findRoot(require.resolve("@swc/core"))),
+    ]);
+
+    this.cacheEpoch = `${thisPackageVersion}-${swcCoreVersion}`;
+  }
+
   async invalidateBuildSet() {
     this.invalidatedFiles = new Set();
     this.compiledFiles = new CompiledFiles();
@@ -151,20 +198,33 @@ export class SwcCompiler implements Compiler {
   }
 
   private async buildFile(filename: string, root: string, config: Config): Promise<CompiledFile> {
-    const output = await transformFile(filename, {
-      cwd: root,
-      filename: filename,
-      root: this.workspaceRoot,
-      rootMode: "root",
-      sourceMaps: "inline",
-      swcrc: false,
-      inlineSourcesContent: true,
-      ...config,
-    });
+    const content = await fs.readFile(filename, "utf8");
+
+    const contentHash = this.xxhash.h32ToString(this.cacheEpoch + "///" + filename + "///" + content);
+    const cacheKey = `${path.basename(filename).replace(/[^a-zA-Z0-9]/g, "")}-${contentHash.slice(2)}-${hashConfig(config)}`;
+    const destination = path.join(this.outDir, contentHash.slice(0, 2), cacheKey);
+
+    if (!this.knownCacheEntries.has(cacheKey)) {
+      const options: Options = {
+        cwd: root,
+        filename: filename,
+        root: this.workspaceRoot,
+        rootMode: "root",
+        sourceMaps: "inline",
+        swcrc: false,
+        inlineSourcesContent: true,
+        ...config,
+      };
+
+      const [transformResult, _] = await Promise.all([
+        transform(content, options),
+        fs.mkdir(path.dirname(destination), { recursive: true }),
+      ]);
+
+      await writeFileAtomic(destination, transformResult.code);
+      this.knownCacheEntries.add(cacheKey);
+    }
 
-    const destination = path.join(this.outDir, filename).replace(this.workspaceRoot, "");
-    await fs.mkdir(path.dirname(destination), { recursive: true });
-    await writeFileAtomic(destination, output.code);
     const file = { filename, root, destination, config };
 
     this.compiledFiles.addFile(file);
@@ -267,3 +327,6 @@ export class SwcCompiler implements Compiler {
     return;
   }
 }
+
+const hashObject = hasher({ sort: true });
+const hashConfig = _.memoize((config: Config) => hashObject.hash(config));

pnpm-lock.yaml

@@ -15,6 +15,9 @@ if (!workerData || !(workerData as SyncWorkerData).isWDSSyncWorker) {
       }
   > = {};
 
+  // enable source maps
+  process.setSourceMapsEnabled(true);
+
   // Compile a given file by sending it into our async-to-sync wrapper worker js file
   // The leader process returns us a list of all the files it just compiled, so that we don't have to pay the IPC boundary cost for each file after this one
   // So, we keep a map of all the files it's compiled so far, and check it first.
@@ -37,7 +40,9 @@ if (!workerData || !(workerData as SyncWorkerData).isWDSSyncWorker) {
 
   // Register our compiler for typescript files.
   // We don't do the best practice of chaining module._compile calls because esbuild won't know about any of the stuff any of the other extensions might do, so running them wouldn't do anything. wds must then be the first registered extension.
-  for (const extension of process.env["WDS_EXTENSIONS"]!.split(",")) {
+  const extensions = process.env["WDS_EXTENSIONS"]!.split(",");
+  log.debug("registering cjs hook for extensions", extensions);
+  for (const extension of extensions) {
     require.extensions[extension] = (module: any, filename: string) => {
       const compiledFilename = compileOffThread(filename);
       if (typeof compiledFilename === "string") {

spec/SwcCompiler.test.ts

@@ -2,18 +2,16 @@
  * Entrypoint file passed as --import to all child processes started by wds
  */
 import { register } from "node:module";
+import { log } from "./utils.cjs";
 
 if (!register) {
   throw new Error(
     `This version of Node.js (${process.version}) does not support module.register(). Please upgrade to Node v18.19 or v20.6 and above.`
   );
 }
 
-// enable source maps
-process.setSourceMapsEnabled(true);
-
 // register the CJS hook to intercept require calls the old way
-import "./child-process-cjs-hook.cjs";
 
+log.debug("registering wds ESM loader");
 // register the ESM loader the new way
 register("./child-process-esm-loader.js", import.meta.url);

src/Options.ts

@@ -8,7 +8,7 @@ import { fileURLToPath } from "url";
 import Watcher from "watcher";
 import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
-import type { RunOptions } from "./Options.js";
+import type { ProjectConfig, RunOptions } from "./Options.js";
 import { Project } from "./Project.js";
 import { Supervisor } from "./Supervisor.js";
 import { MissingDestinationError, SwcCompiler } from "./SwcCompiler.js";
@@ -142,14 +142,28 @@ const startIPCServer = async (socketPath: string, project: Project) => {
   return server;
 };
 
-const childProcessArgs = () => {
-  return ["--import", path.join(dirname, "hooks", "child-process-register.js")];
+const childProcessArgs = (config: ProjectConfig) => {
+  const args = ["--require", path.join(dirname, "hooks", "child-process-cjs-hook.cjs")];
+  if (config.esm) {
+    args.push("--import", path.join(dirname, "hooks", "child-process-esm-hook.js"));
+  }
+  return args;
 };
 
 export const wds = async (options: RunOptions) => {
-  const workspaceRoot = findWorkspaceRoot(process.cwd()) || process.cwd();
+  let workspaceRoot: string;
+  let projectRoot: string;
   const workDir = await fs.mkdtemp(path.join(os.tmpdir(), "wds"));
-  log.debug(`starting wds for workspace root ${workspaceRoot} and workdir ${workDir}`);
+
+  const firstNonOptionArg = options.argv.find((arg) => !arg.startsWith("-"));
+  if (firstNonOptionArg && fs.existsSync(firstNonOptionArg)) {
+    const absolutePath = path.resolve(firstNonOptionArg);
+    projectRoot = findRoot(path.dirname(absolutePath));
+    workspaceRoot = findWorkspaceRoot(projectRoot) || projectRoot;
+  } else {
+    projectRoot = findRoot(process.cwd());
+    workspaceRoot = findWorkspaceRoot(process.cwd()) || process.cwd();
+  }
 
   let serverSocketPath: string;
   if (os.platform() === "win32") {
@@ -158,10 +172,13 @@ export const wds = async (options: RunOptions) => {
     serverSocketPath = path.join(workDir, "ipc.sock");
   }
 
-  const compiler = new SwcCompiler(workspaceRoot, workDir);
+  const config = await projectConfig(projectRoot);
+  log.debug(`starting wds for workspace root ${workspaceRoot} and workdir ${workDir}`, config);
+
+  const compiler = await SwcCompiler.create(workspaceRoot, config.cacheDir);
+  const project = new Project(workspaceRoot, config, compiler);
 
-  const project = new Project(workspaceRoot, await projectConfig(findRoot(process.cwd())), compiler);
-  project.supervisor = new Supervisor([...childProcessArgs(), ...options.argv], serverSocketPath, options, project);
+  project.supervisor = new Supervisor([...childProcessArgs(config), ...options.argv], serverSocketPath, options, project);
 
   if (options.reloadOnChanges) startFilesystemWatcher(project);
   if (options.terminalCommands) startTerminalCommandListener(project);