[pull] main from millionco:main

.changeset/persistent-oxlint-worker-pool.md

@@ -0,0 +1,6 @@
+---
+"oxlint-plugin-react-doctor": patch
+"react-doctor": patch
+---
+
+Speed up large-repository scans: reuse warm oxlint worker processes across projects, overlap project discovery with linting, and trim rule hot paths (2.4–5.5x faster wall-clock on the large-repo corpus).

.gitignore

@@ -7,6 +7,7 @@ dist
 .cursor
 .claude
 tmp/
+.performance/
 review-report.md
 review-*.md
 *.review.md

package.json

@@ -15,6 +15,7 @@
     "dev": "turbo run dev --filter=react-doctor",
     "build": "turbo run build",
     "performance": "tsx scripts/performance/run-performance.ts",
+    "performance:corpus": "tsx scripts/performance/fetch-corpus.ts",
     "performance:stress": "tsx scripts/performance/run-stress-performance.ts",
     "performance:profile": "tsx scripts/performance/analyze-cpu-profile.ts",
     "performance:memory": "tsx scripts/performance/analyze-heap-profile.ts",

packages/api/src/diagnose.ts

@@ -31,7 +31,7 @@ import {
   type InspectOutput,
   type ResolvedScanTarget,
   type SourceFileEntry,
-  type WorkerSlots,
+  type OxlintSpawnSlotsHandle,
 } from "@react-doctor/core";
 import type {
   DiagnoseOptions,
@@ -62,7 +62,7 @@ interface DiagnoseLayerInput {
   readonly shouldRunLint: boolean;
   readonly shouldRunDeadCode: boolean;
   readonly oxlintConcurrency: number;
-  readonly oxlintSpawnSlots: WorkerSlots;
+  readonly oxlintSpawnSlots: OxlintSpawnSlotsHandle;
   readonly configOverrideTarget?: Pick<
     ResolvedScanTarget,
     "resolvedDirectory" | "configSourceDirectory"
@@ -214,7 +214,7 @@ const diagnoseProject = async (
   baseOptions: DiagnoseOptions,
   batchConfig: ReactDoctorConfig | undefined,
   oxlintConcurrency: number,
-  oxlintSpawnSlots: WorkerSlots,
+  oxlintSpawnSlots: OxlintSpawnSlotsHandle,
   precomputedSourceFiles: ReadonlyArray<SourceFileEntry> | undefined,
 ): Promise<ProjectResult> => {
   const startTime = globalThis.performance.now();

packages/api/src/duplicate-jsx-worker.ts

@@ -0,0 +1,3 @@
+import { startDuplicateJsxWorker } from "../../core/src/start-duplicate-jsx-worker.js";
+
+startDuplicateJsxWorker();

packages/api/src/oxlint-worker.ts

@@ -0,0 +1,3 @@
+import { startOxlintWorker } from "../../core/src/start-oxlint-worker.js";
+
+startOxlintWorker();

packages/api/vite.config.ts

@@ -1,22 +1,26 @@
 import { defineConfig } from "vite-plus";
+import { requireTypescriptPlugin } from "../../scripts/require-typescript-plugin.js";
 
 export default defineConfig({
   pack: [
     {
       entry: {
         index: "./src/index.ts",
         "project-analysis-worker": "./src/project-analysis-worker.ts",
+        "oxlint-worker": "./src/oxlint-worker.ts",
+        "duplicate-jsx-worker": "./src/duplicate-jsx-worker.ts",
       },
       deps: {
+        alwaysBundle: ["typescript"],
         neverBundle: [
           "effect",
           "oxc-parser",
           "oxc-resolver",
           "oxlint",
           "oxlint-plugin-react-doctor",
-          "typescript",
         ],
       },
+      plugins: [requireTypescriptPlugin()],
       dts: true,
       target: "node20",
       platform: "node",

packages/core/src/check-react-server-components-advisory.ts

@@ -9,6 +9,8 @@ import { getDependencySpec } from "./project-info/dependencies.js";
 import { findMonorepoRoot, isFile, readPackageJson } from "./project-info/index.js";
 import { getWorkspacePatterns, resolveWorkspaceDirectories } from "./project-info/workspaces.js";
 import type { Diagnostic, ProjectInfo } from "./types/index.js";
+import type { WorkspaceProbeCache } from "./utils/create-invocation-caches.js";
+import { getOrCompute } from "./utils/get-or-compute.js";
 
 const RULE_KEY = "no-vulnerable-react-server-components";
 
@@ -70,19 +72,23 @@ const buildAdvisoryDiagnostic = (input: BuildAdvisoryDiagnosticInput): Diagnosti
 // `listWorkspacePackages`, which keeps only React- or Three-bearing packages. A
 // workspace that declares only a `react-server-dom-*` package (or `next` solely
 // under `optionalDependencies`) must still have its `node_modules` probed.
-const enumerateWorkspaceDirectories = (workspaceRoot: string): string[] => {
-  const patterns = getWorkspacePatterns(
-    workspaceRoot,
-    readPackageJson(path.join(workspaceRoot, "package.json")),
-  );
-  const directories = new Set<string>();
-  for (const pattern of patterns) {
-    for (const directory of resolveWorkspaceDirectories(workspaceRoot, pattern)) {
-      directories.add(directory);
+const enumerateWorkspaceDirectories = (
+  workspaceRoot: string,
+  probeCache: WorkspaceProbeCache | null,
+): ReadonlyArray<string> =>
+  getOrCompute(probeCache?.workspaceDirectoriesByRoot ?? null, workspaceRoot, () => {
+    const patterns = getWorkspacePatterns(
+      workspaceRoot,
+      readPackageJson(path.join(workspaceRoot, "package.json")),
+    );
+    const directories = new Set<string>();
+    for (const pattern of patterns) {
+      for (const directory of resolveWorkspaceDirectories(workspaceRoot, pattern)) {
+        directories.add(directory);
+      }
     }
-  }
-  return [...directories];
-};
+    return [...directories];
+  });
 
 // Resolves the concrete version a package runs *in a single directory*,
 // preferring the installed manifest under that directory's `node_modules`
@@ -91,30 +97,38 @@ const enumerateWorkspaceDirectories = (workspaceRoot: string): string[] => {
 // (`^19.2.0`), so the check never guesses off an ambiguous range whose lockfile
 // may resolve higher. The caller probes every candidate directory (scan root +
 // each workspace package) so heterogeneous monorepo installs are all seen.
-const resolveVersionInDirectory = (
+const resolveConcreteVersionInDirectory = (
   directory: string,
   packageName: string,
-  declaredSpecOverride: string | null,
 ): string | null => {
   const manifestPath = path.join(directory, "node_modules", packageName, "package.json");
   if (isFile(manifestPath)) {
     const installedVersion = semver.valid(readPackageJson(manifestPath).version ?? null);
     if (installedVersion !== null) return installedVersion;
   }
+  const declaredSpec = getDependencySpec(
+    readPackageJson(path.join(directory, "package.json")),
+    packageName,
+  );
+  return declaredSpec === null ? null : semver.valid(declaredSpec);
+};
 
-  // Fall through to the first spec that is actually a concrete version. The
-  // directory's own declaration is tried first, then the seed (discovery's
-  // catalog-resolved `project.nextjsVersion`) — so an unparseable manifest spec
-  // like `catalog:` doesn't shadow an already-resolved concrete pin.
-  const candidateSpecs = [
-    getDependencySpec(readPackageJson(path.join(directory, "package.json")), packageName),
-    declaredSpecOverride,
-  ];
-  for (const spec of candidateSpecs) {
-    const pinnedVersion = spec === null ? null : semver.valid(spec);
-    if (pinnedVersion !== null) return pinnedVersion;
-  }
-  return null;
+// The directory's own install/declaration is tried first, then the seed
+// (discovery's catalog-resolved `project.nextjsVersion`) — so an unparseable
+// manifest spec like `catalog:` doesn't shadow an already-resolved concrete pin.
+const resolveVersionInDirectory = (
+  directory: string,
+  packageName: string,
+  declaredSpecOverride: string | null,
+  probeCache: WorkspaceProbeCache | null,
+): string | null => {
+  const concreteVersion = getOrCompute(
+    probeCache?.concreteVersionsByProbe ?? null,
+    `${directory}\0${packageName}`,
+    () => resolveConcreteVersionInDirectory(directory, packageName),
+  );
+  if (concreteVersion !== null) return concreteVersion;
+  return declaredSpecOverride === null ? null : semver.valid(declaredSpecOverride);
 };
 
 const checkReactServerDomAdvisory = (packageName: string, version: string): Diagnostic[] => {
@@ -214,6 +228,7 @@ const checkNextjsAdvisory = (version: string): Diagnostic[] => {
 export const checkReactServerComponentsAdvisory = (
   scanDirectory: string,
   project: ProjectInfo,
+  probeCache: WorkspaceProbeCache | null = null,
 ): Diagnostic[] => {
   // `project.rootDirectory` is the scanned directory, not necessarily the
   // monorepo root, so walk up to the real root: it enumerates every sibling
@@ -225,7 +240,7 @@ export const checkReactServerComponentsAdvisory = (
       scanDirectory,
       project.rootDirectory,
       workspaceRoot,
-      ...enumerateWorkspaceDirectories(workspaceRoot),
+      ...enumerateWorkspaceDirectories(workspaceRoot, probeCache),
     ]),
   ];
 
@@ -246,6 +261,7 @@ export const checkReactServerComponentsAdvisory = (
       directory,
       "next",
       directory === scanDirectory ? project.nextjsVersion : null,
+      probeCache,
     );
     if (nextVersion !== null) pushUnique(checkNextjsAdvisory(nextVersion));
 
@@ -258,7 +274,7 @@ export const checkReactServerComponentsAdvisory = (
     if (nextGovernsRsc) continue;
 
     for (const packageName of REACT_SERVER_DOM_PACKAGES) {
-      const version = resolveVersionInDirectory(directory, packageName, null);
+      const version = resolveVersionInDirectory(directory, packageName, null, probeCache);
       if (version !== null) pushUnique(checkReactServerDomAdvisory(packageName, version));
     }
   }

packages/core/src/constants.ts

@@ -286,6 +286,13 @@ export const AUTO_MAX_SCAN_CONCURRENCY = 10;
 // proportionally more speed.
 export const HARD_MAX_SCAN_CONCURRENCY = 32;
 
+// Rust parse threads per oxlint subprocess (`--threads`). oxlint defaults to one
+// rayon thread per core, so N concurrent workers would otherwise run N × cores
+// parse threads against N JS-plugin main threads; sharing the cores instead
+// (`floor(cores / workers)`, never below this floor) measured 7-8% less wall
+// and CPU on an 8-core box at 8 workers.
+export const MIN_OXLINT_THREADS_PER_WORKER = 1;
+
 // Memory one oxlint subprocess is budgeted at the OXLINT_MAX_FILES_PER_BATCH=200
 // batch size (the native binding's parser arena + the batch's ASTs + the
 // JS-plugin heap). The auto path takes `floor(availableMemory / this)` as a
@@ -306,6 +313,10 @@ export const PER_WORKER_MEM_BUDGET_BYTES = 1024 * 1024 * 1024;
 // default. Callers opt into more via `DiagnoseProjectsInput.concurrency`.
 export const DEFAULT_PROJECT_SCAN_CONCURRENCY = 4;
 
+// A project holds its batch slot through its trailing score round-trip, so one
+// project per worker would leave the shared oxlint pool idle on small packages.
+export const PROJECT_SCANS_IN_FLIGHT_PER_OXLINT_WORKER = 4;
+
 export const DEFAULT_BRANCH_CANDIDATES = ["main", "master"];
 
 // JSON-format oxlint / eslint configs react-doctor can fold into the
@@ -612,6 +623,56 @@ export const OXLINT_SPLIT_MAX_DEPTH = 9;
 // rescue pass work on Windows too.
 export const ABORT_EXIT_CODES: ReadonlySet<number> = new Set([134, 0xc0000409]);
 
+// Line the oxlint worker writes to fd 1 / fd 2 after each `lint()` job so the
+// parent can split one worker's stream into per-job outputs. oxlint's JSON
+// formatter escapes newlines inside strings, so a whole line equal to this
+// marker cannot occur inside the payload.
+export const OXLINT_WORKER_JOB_END_MARKER = "__REACT_DOCTOR_OXLINT_JOB_END__";
+
+// Boot budget for an oxlint worker (Node start + importing oxlint's native
+// binding and the plugin runtime). A worker that has not reported ready by
+// then is treated as unavailable and the batch falls back to the per-batch
+// spawn path.
+export const OXLINT_WORKER_READY_TIMEOUT_MS = 30_000;
+
+// An idle worker keeps ~150 MB of warmed plugin heap alive; long-lived hosts
+// (`@react-doctor/api`) reclaim it after this quiet period. The CLI is not
+// held open by idle workers — their handles are unref'd — so this only
+// matters between scans in one process.
+export const OXLINT_WORKER_IDLE_TIMEOUT_MS = 30_000;
+export const DUPLICATE_JSX_WORKER_IDLE_TIMEOUT_MS = 30_000;
+
+// Every pooled job hands the plugin a fresh oxlint transfer buffer and fresh
+// rule closures, and the full GC that follows drops every TurboFan code object
+// specialized on the previous job's objects ("weak objects" deopt). With V8's
+// default tier-up threshold (3 000 invocations) the ~300 functions that just
+// got invalidated re-optimize on every job, and the background compiler
+// threads end up burning more CPU than the lint itself. Raising the threshold
+// keeps TurboFan for the genuinely hot walkers while the rest stay on Maglev;
+// measured on refine/grafana/tldraw it cut worker CPU ~30% and wall ~15-25%.
+// The flag exists from V8 12 (Node 22); Node 20 keeps the default tiering.
+export const OXLINT_WORKER_TURBOFAN_INVOCATION_COUNT = 30_000;
+export const OXLINT_WORKER_TIERING_FLAGS_MIN_NODE_MAJOR = 22;
+
+// HACK: oxlint registers one 2 GiB fixed-size AST transfer buffer per native
+// thread as V8 external memory, and a pooled job allocates a new set before the
+// previous set is collected. External growth past half the old-space limit
+// triggers a synchronous full GC per job; a ceiling of two buffers per thread
+// (plus headroom) keeps V8 on incremental marking. Nothing is committed up front.
+export const OXLINT_WORKER_OLD_SPACE_MB_PER_NATIVE_THREAD = 4352;
+
+// Global registry key under which the react-doctor oxlint plugin publishes its
+// filesystem-cache reset. A warm worker calls it when a job's scan epoch differs
+// from the previous job's, so caches never outlive one invocation. Mirrored in
+// `oxlint-plugin-react-doctor/src/plugin/constants/host.ts`.
+export const REACT_DOCTOR_PLUGIN_RESET_HOOK_KEY = Symbol.for(
+  "react-doctor.reset-filesystem-caches",
+);
+
+// Bytes of each oxlint job's stdout kept in the performance-harness timeline
+// (`REACT_DOCTOR_OXLINT_SPAWN_LOG`), enough to tell JSON output from a crash.
+export const OXLINT_JOB_TIMELINE_STDOUT_PREVIEW_BYTES = 160;
+
 // Wall-clock cap on the serial OOM rescue pass (replaying OOM-dropped
 // files one at a time after the parallel pass). The rescue is unbounded
 // by batch count — each file that STILL fails re-waits a spawn timeout —

packages/core/src/duplicate-jsx-worker.ts

@@ -0,0 +1,3 @@
+import { startDuplicateJsxWorker } from "./start-duplicate-jsx-worker.js";
+
+startDuplicateJsxWorker();

packages/core/src/index.ts

@@ -96,6 +96,8 @@ export * from "./utils/collect-source-file-counts-by-directory.js";
 export * from "./utils/collect-source-files-by-directory.js";
 export * from "./utils/compute-config-fingerprint.js";
 export * from "./utils/create-oxlint-spawn-slots.js";
+export * from "./utils/create-git-repository-metadata-cache.js";
+export * from "./utils/create-invocation-caches.js";
 export * from "./utils/create-worker-slots.js";
 export * from "./utils/dedupe-diagnostics.js";
 export * from "./utils/define-config.js";
@@ -131,3 +133,4 @@ export * from "./utils/yield-to-event-loop.js";
 export * from "./project-info/capabilities.js";
 export * from "./runners/oxlint/config.js";
 export * from "./runners/oxlint/plugin-resolution.js";
+export { warmOxlintWorkerPool } from "./runners/oxlint/run-oxlint-job.js";

packages/core/src/oxlint-worker.ts

@@ -0,0 +1,3 @@
+import { startOxlintWorker } from "./start-oxlint-worker.js";
+
+startOxlintWorker();

packages/core/src/project-info/fs-utils.ts

@@ -34,15 +34,15 @@ export const readDirectoryEntries = (directoryPath: string): fs.Dirent[] => {
 
 export const isFile = (filePath: string): boolean => {
   try {
-    return fs.statSync(filePath).isFile();
+    return fs.statSync(filePath, { throwIfNoEntry: false })?.isFile() ?? false;
   } catch {
     return false;
   }
 };
 
 export const isDirectory = (directoryPath: string): boolean => {
   try {
-    return fs.statSync(directoryPath).isDirectory();
+    return fs.statSync(directoryPath, { throwIfNoEntry: false })?.isDirectory() ?? false;
   } catch {
     return false;
   }

packages/core/src/project-info/react-compiler-config-evaluator.ts

@@ -7,7 +7,7 @@ import { REACT_COMPILER_CONFIG_IMPORT_MAX_DEPTH } from "../constants.js";
 import type { PackageJson } from "../types/index.js";
 import { isProjectBoundary } from "../utils/is-project-boundary.js";
 import { unwrapTypescriptExpression } from "../utils/unwrap-typescript-expression.js";
-import { isFile, isPlainObject } from "./fs-utils.js";
+import { isFile, isPlainObject, readDirectoryEntries } from "./fs-utils.js";
 import { isLocalModuleSpecifier } from "./is-local-module-specifier.js";
 import { NEXT_CONFIG_FILENAMES } from "./detect-nextjs-static-export.js";
 import { readPackageJson } from "./package-json.js";
@@ -2171,8 +2171,16 @@ const analyzeConfigNode = (
 const hasCompilerInConfigFile = (filePath: string): boolean =>
   analyzeConfigModuleExport(filePath, "default", false, 0, new Set<string>());
 
-const hasCompilerInConfigFiles = (directory: string, filenames: string[]): boolean =>
-  filenames.some((filename) => hasCompilerInConfigFile(path.join(directory, filename)));
+const hasCompilerInConfigFiles = (directory: string, filenames: string[]): boolean => {
+  const presentEntryNames = new Set(
+    readDirectoryEntries(directory).map((entry) => entry.name.toLowerCase()),
+  );
+  return filenames.some(
+    (filename) =>
+      presentEntryNames.has(filename.toLowerCase()) &&
+      hasCompilerInConfigFile(path.join(directory, filename)),
+  );
+};
 
 const hasCompilerInPackageJsonConfig = (directory: string, packageJson: PackageJson): boolean => {
   if (!isPlainObject(packageJson.babel)) return false;