Cleanup glob handling and do ignore checks in memory using micromatch

Author: airhorns

integration-test/reload-cross-workspace-lazy/test.sh

@@ -34,6 +34,8 @@ echo "Made initial request to server"
 # modify the file in the side package and expect the main script to reload
 sed -i 's/Hello, World/Hey, Pluto/g' $DIR/side/run-scratch.ts
 
+echo "Made change to side package"
+
 counter=0
 until curl -s localhost:8080 | grep "Pluto"
 do

integration-test/reload-cross-workspace/test.sh

@@ -34,6 +34,8 @@ echo "Made initial request to server"
 # modify the file in the side package and expect the main script to reload
 sed -i 's/Hello, World/Hey, Pluto/g' $DIR/side/run-scratch.ts
 
+echo "Made change to side package"
+
 counter=0
 until curl -s localhost:8080 | grep "Pluto"
 do

package.json

@@ -48,6 +48,7 @@
     "fs-extra": "^11.2.0",
     "globby": "^11.1.0",
     "lodash": "^4.17.20",
+    "micromatch": "^4.0.8",
     "node-object-hash": "^3.0.0",
     "oxc-resolver": "^1.12.0",
     "pkg-dir": "^5.0.0",
@@ -62,6 +63,7 @@
     "@types/find-root": "^1.1.4",
     "@types/fs-extra": "^11.0.4",
     "@types/lodash": "^4.17.13",
+    "@types/micromatch": "^4.0.9",
     "@types/node": "^22.9.3",
     "@types/write-file-atomic": "^4.0.3",
     "@types/yargs": "^15.0.19",

pnpm-lock.yaml

@@ -41,10 +41,10 @@ test("throws if the file is ignored", async () => {
     }
   }
 
-  expect(error).toBeDefined();
+  expect(error).toBeTruthy();
   expect(error?.ignoredFile).toBeTruthy();
   expect(error?.message).toMatch(
-    /File .+ignored\.ts is imported but not being built because it is explicitly ignored in the wds project config\. It is being ignored by the provided glob pattern '!ignored\.ts', remove this pattern from the project config or don't import this file to fix./
+    /File .+ignored\.ts is imported but not being built because it is explicitly ignored in the wds project config\. It is being ignored by the provided glob pattern 'ignored\.ts', remove this pattern from the project config or don't import this file to fix./
   );
 });
 

spec/SwcCompiler.test.ts

@@ -1,6 +1,7 @@
 import type { Options as SwcOptions } from "@swc/core";
 import fs from "fs-extra";
 import _ from "lodash";
+import micromatch from "micromatch";
 import path from "path";
 import { log } from "./utils.js";
 
@@ -13,44 +14,61 @@ export interface RunOptions {
 }
 
 export interface ProjectConfig {
+  root: string;
   ignore: string[];
+  includeGlob: string;
+  includedMatcher: (filePath: string) => boolean;
   swc?: SwcConfig;
   esm?: boolean;
   extensions: string[];
   cacheDir: string;
 }
 
-export const projectConfig = async (root: string): Promise<ProjectConfig> => {
+export const projectConfig = _.memoize(async (root: string): Promise<ProjectConfig> => {
   const location = path.join(root, "wds.js");
   const base: ProjectConfig = {
-    ignore: [],
+    root,
     extensions: [".ts", ".tsx", ".jsx"],
     cacheDir: path.join(root, "node_modules/.cache/wds"),
     esm: true,
+    /** The list of globby patterns to use when searching for files to build */
+    includeGlob: `**/*`,
+    /** The list of globby patterns to ignore use when searching for files to build */
+    ignore: [],
+    /** A micromatch matcher for userland checking if a file is included */
+    includedMatcher: () => true,
   };
 
+  let exists = false;
   try {
     await fs.access(location);
+    exists = true;
   } catch (error: any) {
     log.debug(`Not loading project config from ${location}`);
-    return base;
   }
 
-  let required = await import(location);
-  if (required.default) {
-    required = required.default;
+  let result: ProjectConfig;
+  if (exists) {
+    let required = await import(location);
+    if (required.default) {
+      required = required.default;
+    }
+    log.debug(`Loaded project config from ${location}`);
+    result = _.defaults(required, base);
+  } else {
+    result = base;
   }
-  log.debug(`Loaded project config from ${location}`);
-  const result = _.defaults(required, base);
 
   const projectRootDir = path.dirname(location);
   // absolutize the cacheDir if not already
   if (!result.cacheDir.startsWith("/")) {
     result.cacheDir = path.resolve(projectRootDir, result.cacheDir);
   }
 
-  // absolutize the ignore paths if not already
-  result.ignore = result.ignore.map((p: string) => (p.startsWith("/") ? p : path.resolve(projectRootDir, p)));
+  // build inclusion glob and matcher
+  result.ignore = _.uniq([`node_modules`, `**/*.d.ts`, `.git/**`, ...result.ignore]);
+  result.includeGlob = `**/*{${result.extensions.join(",")}}`;
+  result.includedMatcher = micromatch.matcher(result.includeGlob, { cwd: result.root, ignore: result.ignore });
 
   return result;
-};
+});

src/ProjectConfig.ts

@@ -13,7 +13,7 @@ import path from "path";
 import { fileURLToPath } from "url";
 import writeFileAtomic from "write-file-atomic";
 import type { Compiler } from "./Compiler.js";
-import { projectConfig, type ProjectConfig } from "./ProjectConfig.js";
+import { projectConfig } from "./ProjectConfig.js";
 import { log } from "./utils.js";
 
 const __filename = fileURLToPath(import.meta.url);
@@ -184,11 +184,14 @@ export class SwcCompiler implements Compiler {
       swcConfig = config.swc;
     }
 
-    const globs = [...this.fileGlobPatterns(config), ...this.ignoreFileGlobPatterns(config)];
+    log.debug("searching for filenames", { filename, config });
 
-    log.debug("searching for filenames", { filename, config, root, globs });
-
-    let fileNames = await globby(globs, { cwd: root, absolute: true });
+    let fileNames = await globby(config.includeGlob, {
+      onlyFiles: true,
+      cwd: root,
+      absolute: true,
+      ignore: config.ignore,
+    });
 
     if (process.platform === "win32") {
       fileNames = fileNames.map((fileName) => fileName.replace(/\//g, "\\"));
@@ -276,16 +279,6 @@ export class SwcCompiler implements Compiler {
     }
   }
 
-  /** The list of globby patterns to use when searching for files to build */
-  private fileGlobPatterns(config: ProjectConfig) {
-    return [`**/*{${config.extensions.join(",")}}`];
-  }
-
-  /** The list of globby patterns to ignore use when searching for files to build */
-  private ignoreFileGlobPatterns(config: ProjectConfig) {
-    return [`!node_modules`, `!**/*.d.ts`, ...(config.ignore || []).map((ignore) => `!${ignore}`)];
-  }
-
   /**
    * Detect if a file is being ignored by the ignore glob patterns for a given project
    *
@@ -294,16 +287,17 @@ export class SwcCompiler implements Compiler {
   private async isFilenameIgnored(filename: string): Promise<string | false> {
     const root = findRoot(filename);
     const config = await projectConfig(root);
-    const includeGlobs = this.fileGlobPatterns(config);
-    const ignoreGlobs = this.ignoreFileGlobPatterns(config);
-
-    const actual = await globby([...includeGlobs, ...ignoreGlobs], { cwd: root, absolute: true });
-    const all = await globby(includeGlobs, { cwd: root, absolute: true });
 
-    // if the file isn't returned when we use the ignores, but is when we don't use the ignores, it means were ignoring it. Figure out which ignore is causing this
-    if (!actual.includes(filename) && all.includes(filename)) {
-      for (const ignoreGlob of ignoreGlobs) {
-        const withThisIgnore = await globby([...includeGlobs, ignoreGlob], { cwd: root, absolute: true });
+    // check if the file is ignored by any of the ignore patterns using micromatch
+    const included = config.includedMatcher(filename.replace(root + path.sep, ""));
+    if (!included) {
+      // figure out which ignore pattern is causing the file to be ignored for a better error message
+      for (const ignoreGlob of config.ignore) {
+        const withThisIgnore = await globby(config.includeGlob, {
+          cwd: root,
+          absolute: true,
+          ignore: [ignoreGlob],
+        });
         if (!withThisIgnore.includes(filename)) {
           return ignoreGlob;
         }

src/SwcCompiler.ts

@@ -70,16 +70,19 @@ const startFilesystemWatcher = (project: Project) => {
   const watcher = new Watcher([project.workspaceRoot], {
     ignoreInitial: true,
     recursive: true,
-    ignore: ((filePath: string) => {
+    ignore: (filePath: string) => {
       if (filePath.includes(nodeModulesDir)) return true;
+      if (filePath == project.workspaceRoot) return false;
+      if (filePath == project.config.root) return false;
       if (filePath.endsWith(".d.ts")) return true;
       if (filePath.endsWith(".map")) return true;
       if (filePath.includes(gitDir)) return true;
       if (filePath.endsWith(".DS_Store")) return true;
       if (filePath.endsWith(".tsbuildinfo")) return true;
 
-      return project.config.ignore?.some((ignore) => filePath.startsWith(ignore)) ?? false;
-    }) as any,
+      // allow files that match the include glob to be watched, or directories (since they might contain files)
+      return !project.config.includedMatcher(filePath) && path.extname(filePath) != "";
+    },
   });
 
   log.debug("started watcher", { root: project.workspaceRoot });