From 87c7302b294aebd5fbbba38f2d32c5b06b1241c2 Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Tue, 1 Jul 2025 13:15:18 +0100 Subject: [PATCH 01/19] wip: add `withIterator` Doesn't work yet, but future James will sort that out. --- __tests__/fdir.test.ts | 126 ++++++++++++++++------------ __tests__/utils.ts | 2 +- src/api/functions/group-files.ts | 8 +- src/api/functions/push-directory.ts | 20 +++-- src/api/functions/push-file.ts | 11 ++- src/api/iterator.ts | 70 ++++++++++++++++ src/api/walker.ts | 45 ++++++++-- src/builder/api-builder.ts | 10 ++- src/types.ts | 1 + 9 files changed, 219 insertions(+), 74 deletions(-) create mode 100644 src/api/iterator.ts diff --git a/__tests__/fdir.test.ts b/__tests__/fdir.test.ts index 5915ad9f..d1a50583 100644 --- a/__tests__/fdir.test.ts +++ b/__tests__/fdir.test.ts @@ -1,4 +1,4 @@ -import { fdir } from "../src/index"; +import { fdir, IterableOutput } from "../src/index"; import fs from "fs"; import mock from "mock-fs"; import { test, beforeEach, TestContext, vi } from "vitest"; @@ -6,6 +6,7 @@ import path, { sep } from "path"; import { convertSlashes } from "../src/utils"; import picomatch from "picomatch"; import { apiTypes, APITypes, cwd, restricted, root } from "./utils"; +import { APIBuilder } from "../src/builder/api-builder"; beforeEach(() => { mock.restore(); @@ -25,24 +26,38 @@ test(`crawl single depth directory with callback`, (t) => { }); }); -async function crawl(type: APITypes, path: string, t: TestContext) { +async function execute( + api: APIBuilder, + type: APITypes +): Promise { + let files: T[number][] = []; + + if (type === "withIterator") { + for await (const file of api[type]()) { + files.push(file); + } + } else { + files = await api[type](); + } + return files as T; +} + +async function crawl(type: APITypes, path: string) { const api = new fdir().crawl(path); - const files = await api[type](); - if (!files) throw new Error("files cannot be null."); - t.expect(files[0]).toBeDefined(); - t.expect(files.every((t) => t)).toBeTruthy(); - t.expect(files[0].length).toBeGreaterThan(0); - return files; + return execute(api, type); } for (const type of apiTypes) { test(`[${type}] crawl directory`, async (t) => { - await crawl(type, "__tests__", t); + const files = await crawl(type, "__tests__"); + t.expect(files[0]).toBeDefined(); + t.expect(files.every((t) => t)).toBeTruthy(); + t.expect(files[0].length).toBeGreaterThan(0); }); test(`[${type}] crawl directory with options`, async (t) => { const api = new fdir({ includeBasePath: true }).crawl("__tests__"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.every((file) => file.startsWith("__tests__"))).toBeTruthy(); }); @@ -51,7 +66,7 @@ for (const type of apiTypes) { maxDepth: 0, includeBasePath: true, }).crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files).not.toHaveLength(0); t.expect(files.every((file) => file.split(path.sep).length === 2)).toBe( true @@ -63,7 +78,7 @@ for (const type of apiTypes) { maxDepth: 1, includeBasePath: true, }).crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.some((file) => file.split(path.sep).length === 3)).toBe( true ); @@ -73,18 +88,21 @@ for (const type of apiTypes) { }); test(`[${type}] crawl multi depth directory`, async (t) => { - await crawl(type, "node_modules", t); + const files = await crawl(type, "node_modules"); + t.expect(files[0]).toBeDefined(); + t.expect(files.every((t) => t)).toBeTruthy(); + t.expect(files[0].length).toBeGreaterThan(0); }); test(`[${type}] crawl directory & limit files to 10`, async (t) => { const api = new fdir().withMaxFiles(10).crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files).toHaveLength(10); }); test(`[${type}] crawl and get both files and directories (withDirs)`, async (t) => { const api = new fdir().withDirs().crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files[0]).toBeDefined(); t.expect(files.every((t) => t)).toBeTruthy(); t.expect(files[0].length).toBeGreaterThan(0); @@ -93,7 +111,7 @@ for (const type of apiTypes) { test(`[${type}] crawl and get all files (withMaxDepth = 1)`, async (t) => { const api = new fdir().withMaxDepth(1).withBasePath().crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect( files.every((file) => file.split(path.sep).length <= 3) ).toBeTruthy(); @@ -104,7 +122,7 @@ for (const type of apiTypes) { .withMaxDepth(-1) .withBasePath() .crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.length).toBe(0); }); @@ -114,7 +132,7 @@ for (const type of apiTypes) { .glob("**/*.js") .glob("**/*.js") .crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.every((file) => file.endsWith(".js"))).toBeTruthy(); }); @@ -123,7 +141,7 @@ for (const type of apiTypes) { .withBasePath() .exclude((dir) => dir.includes("node_modules")) .crawl(cwd()); - const files = await api[type](); + const files = await execute(api, type); t.expect( files.every((file) => !file.includes("node_modules")) ).toBeTruthy(); @@ -134,7 +152,7 @@ for (const type of apiTypes) { .withBasePath() .filter((file) => file.includes(".git")) .crawl(cwd()); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.every((file) => file.includes(".git"))).toBeTruthy(); }); @@ -144,7 +162,7 @@ for (const type of apiTypes) { .filter((file) => file.includes(".git")) .filter((file) => file.includes(".js")) .crawl(cwd()); - const files = await api[type](); + const files = await execute(api, type); t.expect( files.every((file) => file.includes(".git") || file.includes(".js")) ).toBeTruthy(); @@ -154,7 +172,7 @@ for (const type of apiTypes) { const api = new fdir() .withBasePath() .crawl(path.join(cwd(), "node_modules")); - const files = await api[type](); + const files = await execute(api, type); t.expect( files.every((file) => file.startsWith("node_modules")) ).toBeTruthy(); @@ -162,18 +180,18 @@ for (const type of apiTypes) { test(`[${type}] get all files in a directory and output full paths (withFullPaths)`, async (t) => { const api = new fdir().withFullPaths().crawl(cwd()); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.every((file) => file.startsWith(root()))).toBeTruthy(); }); test(`[${type}] getting files from restricted directory should throw`, async (t) => { const api = new fdir().withErrors().crawl(restricted()); - t.expect(async () => await api[type]()).rejects.toThrowError(); + t.expect(async () => await execute(api, type)).rejects.toThrowError(); }); test(`[${type}] getting files from restricted directory shouldn't throw (suppressErrors)`, async (t) => { const api = new fdir().crawl(restricted()); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.length).toBeGreaterThanOrEqual(0); }); @@ -184,20 +202,22 @@ for (const type of apiTypes) { }, }); const api = new fdir().withBasePath().normalize().crawl("/"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.every((file) => !file.includes("//"))).toBeTruthy(); mock.restore(); }); - test(`[${type}] crawl all files with only counts`, async (t) => { - const api = new fdir().onlyCounts().crawl("node_modules"); - const result = await api[type](); - t.expect(result.files).toBeGreaterThan(0); - }); + if (type !== "withIterator") { + test(`[${type}] crawl all files with only counts`, async (t) => { + const api = new fdir().onlyCounts().crawl("node_modules"); + const result = await api[type](); + t.expect(result.files).toBeGreaterThan(0); + }); + } test(`[${type}] crawl and return only directories`, async (t) => { const api = new fdir().onlyDirs().crawl("node_modules"); - const result = await api[type](); + const result = await execute(api, type); t.expect(result.length).toBeGreaterThan(0); t.expect( result.every((dir) => { @@ -211,7 +231,7 @@ for (const type of apiTypes) { excludeFiles: true, includeDirs: true, }).crawl("node_modules"); - const result = await api[type](); + const result = await execute(api, type); t.expect(result.length).toBeGreaterThan(0); t.expect( result.every((dir) => { @@ -220,26 +240,28 @@ for (const type of apiTypes) { ).toBeTruthy(); }); - test(`[${type}] crawl and filter all files and get only counts`, async (t) => { - const api = new fdir() - .withBasePath() - .filter((file) => file.includes("node_modules")) - .onlyCounts() - .crawl(cwd()); - const result = await api[type](); - t.expect(result.files).toBeGreaterThan(0); - }); + if (type !== "withIterator") { + test(`[${type}] crawl and filter all files and get only counts`, async (t) => { + const api = new fdir() + .withBasePath() + .filter((file) => file.includes("node_modules")) + .onlyCounts() + .crawl(cwd()); + const result = await api[type](); + t.expect(result.files).toBeGreaterThan(0); + }); + } test("crawl all files in a directory (path with trailing slash)", async (t) => { const api = new fdir().normalize().crawl("node_modules/"); - const files = await api[type](); + const files = await execute(api, type); const res = files.every((file) => !file.includes("/")); t.expect(res).toBeDefined(); }); test(`[${type}] crawl all files and group them by directory`, async (t) => { const api = new fdir().withBasePath().group().crawl("node_modules"); - const result = await api[type](); + const result = await execute(api, type); t.expect(result.length).toBeGreaterThan(0); }); @@ -256,7 +278,7 @@ for (const type of apiTypes) { const api = new fdir() .withRelativePaths() .crawl(path.normalize(`node_modules/`)); - const paths = await api[type](); + const paths = await execute(api, type); t.expect(paths.every((p) => !p.startsWith("node_modules"))).toBeTruthy(); }); @@ -277,7 +299,7 @@ for (const type of apiTypes) { .withDirs() .withRelativePaths() .crawl("/some"); - const paths = await api[type](); + const paths = await execute(api, type); t.expect(paths.length).toBe(5); t.expect(paths.filter((p) => p === ".").length).toBe(1); @@ -303,7 +325,7 @@ for (const type of apiTypes) { .withRelativePaths() .filter((p) => p !== path.join("dir", "dir1/")) .crawl("/some"); - const paths = await api[type](); + const paths = await execute(api, type); t.expect(paths.length).toBe(4); t.expect(paths.includes(path.join("dir", "dir1/"))).toBe(false); @@ -314,7 +336,7 @@ for (const type of apiTypes) { test(`[${type}] crawl and return relative paths that end with /`, async (t) => { const api = new fdir().withRelativePaths().crawl("./node_modules/"); - const paths = await api[type](); + const paths = await execute(api, type); t.expect( paths.every((p) => !p.startsWith("node_modules") && !p.includes("//")) ).toBeTruthy(); @@ -324,7 +346,7 @@ for (const type of apiTypes) { const api = new fdir() .withPathSeparator(sep === "/" ? "\\" : "/") .crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.every((f) => !f.includes(sep))).toBeTruthy(); }); @@ -337,7 +359,7 @@ for (const type of apiTypes) { .withBasePath() .glob("**/*.js") .crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(globFunction).toHaveBeenCalled(); t.expect(files.every((file) => file.endsWith(".js"))).toBeTruthy(); }); @@ -352,7 +374,7 @@ for (const type of apiTypes) { .withBasePath() .globWithOptions(["**/*.js"], { foo: 5 }) .crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(globFunction).toHaveBeenCalled(); t.expect(files.every((file) => file.endsWith(".js"))).toBeTruthy(); }); @@ -363,7 +385,7 @@ for (const type of apiTypes) { .withBasePath() .glob("**/*.js") .crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.every((file) => file.endsWith(".js"))).toBeTruthy(); }); diff --git a/__tests__/utils.ts b/__tests__/utils.ts index f3c7cb02..1123ba11 100644 --- a/__tests__/utils.ts +++ b/__tests__/utils.ts @@ -1,7 +1,7 @@ import path from "path"; export type APITypes = (typeof apiTypes)[number]; -export const apiTypes = ["withPromise", "sync"] as const; +export const apiTypes = ["withPromise", "sync", "withIterator"] as const; export function root() { return process.platform === "win32" ? process.cwd().split(path.sep)[0] : "/"; diff --git a/src/api/functions/group-files.ts b/src/api/functions/group-files.ts index f53a04b3..7a89e2ac 100644 --- a/src/api/functions/group-files.ts +++ b/src/api/functions/group-files.ts @@ -3,15 +3,17 @@ import { Group, Options } from "../../types"; export type GroupFilesFunction = ( groups: Group[], directory: string, - files: string[] + files: string[], + pushGroup: (group: Group, arr: Group[]) => void ) => void; const groupFiles: GroupFilesFunction = ( groups: Group[], directory: string, - files: string[] + files: string[], + pushGroup ) => { - groups.push({ directory, files, dir: directory }); + pushGroup({ directory, files, dir: directory }, groups); }; const empty: GroupFilesFunction = () => {}; diff --git a/src/api/functions/push-directory.ts b/src/api/functions/push-directory.ts index 2de954b8..17c836a4 100644 --- a/src/api/functions/push-directory.ts +++ b/src/api/functions/push-directory.ts @@ -3,38 +3,44 @@ import { FilterPredicate, Options } from "../../types"; export type PushDirectoryFunction = ( directoryPath: string, paths: string[], + pushPath: (path: string, arr: string[]) => void, filters?: FilterPredicate[] ) => void; function pushDirectoryWithRelativePath(root: string): PushDirectoryFunction { - return function (directoryPath, paths) { - paths.push(directoryPath.substring(root.length) || "."); + return function (directoryPath, paths, pushPath) { + pushPath(directoryPath.substring(root.length) || ".", paths); }; } function pushDirectoryFilterWithRelativePath( root: string ): PushDirectoryFunction { - return function (directoryPath, paths, filters) { + return function (directoryPath, paths, pushPath, filters) { const relativePath = directoryPath.substring(root.length) || "."; if (filters!.every((filter) => filter(relativePath, true))) { - paths.push(relativePath); + pushPath(relativePath, paths); } }; } -const pushDirectory: PushDirectoryFunction = (directoryPath, paths) => { - paths.push(directoryPath || "."); +const pushDirectory: PushDirectoryFunction = ( + directoryPath, + paths, + pushPath +) => { + pushPath(directoryPath || ".", paths); }; const pushDirectoryFilter: PushDirectoryFunction = ( directoryPath, paths, + pushPath, filters ) => { const path = directoryPath || "."; if (filters!.every((filter) => filter(path, true))) { - paths.push(path); + pushPath(path, paths); } }; diff --git a/src/api/functions/push-file.ts b/src/api/functions/push-file.ts index 18ef77dc..2e850f54 100644 --- a/src/api/functions/push-file.ts +++ b/src/api/functions/push-file.ts @@ -3,6 +3,7 @@ import { FilterPredicate, Options, Counts } from "../../types"; export type PushFileFunction = ( directoryPath: string, paths: string[], + pushPath: (path: string, arr: string[]) => void, counts: Counts, filters?: FilterPredicate[] ) => void; @@ -10,6 +11,7 @@ export type PushFileFunction = ( const pushFileFilterAndCount: PushFileFunction = ( filename, _paths, + _pushPath, counts, filters ) => { @@ -19,23 +21,26 @@ const pushFileFilterAndCount: PushFileFunction = ( const pushFileFilter: PushFileFunction = ( filename, paths, + pushPath, _counts, filters ) => { - if (filters!.every((filter) => filter(filename, false))) paths.push(filename); + if (filters!.every((filter) => filter(filename, false))) + pushPath(filename, paths); }; const pushFileCount: PushFileFunction = ( _filename, _paths, + _pushPath, counts, _filters ) => { counts.files++; }; -const pushFile: PushFileFunction = (filename, paths) => { - paths.push(filename); +const pushFile: PushFileFunction = (filename, paths, pushPath) => { + pushPath(filename, paths); }; const empty: PushFileFunction = () => {}; diff --git a/src/api/iterator.ts b/src/api/iterator.ts new file mode 100644 index 00000000..7641b3dd --- /dev/null +++ b/src/api/iterator.ts @@ -0,0 +1,70 @@ +import { Options, IterableOutput } from "../types"; +import { Walker } from "./walker"; + +class WalkerIterator { + #next: Promise; + #resolver: (value: TOutput[number] | null) => void; + #walker: Walker; + #currentGroup?: string[]; + + public constructor(root: string, options: Options) { + this.#resolver = () => {}; + this.#next = this.#createNext(); + const pushPath = options.group ? this.#pushPath : this.#pushResult; + this.#walker = new Walker( + root, + options, + this.#onComplete, + pushPath, + this.#pushResult + ); + } + + #pushPath = (path: string, arr: string[]) => { + if (arr !== this.#currentGroup) { + this.#currentGroup = arr; + } + arr.push(path); + }; + + #pushResult = async (result: TOutput[number]) => { + this.#resolver(result); + this.#next = this.#createNext(); + }; + + #onComplete = () => { + this.#currentGroup = undefined; + this.#resolver(null); + }; + + #createNext(): Promise { + return new Promise((resolve) => { + this.#resolver = resolve; + }); + } + + async *[Symbol.asyncIterator]() { + let promise = this.#next; + this.#walker.start(); + + let finished = false; + + while (!finished) { + const result = await promise; + promise = this.#next; + if (result === null) { + finished = true; + } else { + yield result; + } + } + } +} + +export function iterator( + root: string, + options: Options +): AsyncIterable { + const iterator = new WalkerIterator(root, options); + return iterator; +} diff --git a/src/api/walker.ts b/src/api/walker.ts index e6edc0dc..9d9d3546 100644 --- a/src/api/walker.ts +++ b/src/api/walker.ts @@ -1,6 +1,6 @@ import { basename, dirname } from "path"; import { isRootDirectory, normalizePath } from "../utils"; -import { ResultCallback, WalkerState, Options } from "../types"; +import { ResultCallback, WalkerState, Options, Group } from "../types"; import * as joinPath from "./functions/join-path"; import * as pushDirectory from "./functions/push-directory"; import * as pushFile from "./functions/push-file"; @@ -28,11 +28,15 @@ export class Walker { private readonly resolveSymlink: resolveSymlink.ResolveSymlinkFunction | null; private readonly walkDirectory: walkDirectory.WalkDirectoryFunction; private readonly callbackInvoker: invokeCallback.InvokeCallbackFunction; + private readonly pushPath: (path: string, arr: string[]) => void; + private readonly pushGroup: (group: Group, arr: Group[]) => void; constructor( root: string, options: Options, - callback?: ResultCallback + callback?: ResultCallback, + pushPath?: (path: string, arr: string[]) => void, + pushGroup?: (group: Group, arr: Group[]) => void ) { this.isSynchronous = !callback; this.callbackInvoker = invokeCallback.build(options, this.isSynchronous); @@ -66,10 +70,25 @@ export class Walker { this.groupFiles = groupFiles.build(options); this.resolveSymlink = resolveSymlink.build(options, this.isSynchronous); this.walkDirectory = walkDirectory.build(this.isSynchronous); + this.pushPath = + pushPath ?? + ((p, arr) => { + arr.push(p); + }); + this.pushGroup = + pushGroup ?? + ((group, arr) => { + arr.push(group); + }); } start(): TOutput | null { - this.pushDirectory(this.root, this.state.paths, this.state.options.filters); + this.pushDirectory( + this.root, + this.state.paths, + this.pushPath, + this.state.options.filters + ); this.walkDirectory( this.state, this.root, @@ -112,7 +131,13 @@ export class Walker { (entry.isSymbolicLink() && !resolveSymlinks && !excludeSymlinks) ) { const filename = this.joinPath(entry.name, directoryPath); - this.pushFile(filename, files, this.state.counts, filters); + this.pushFile( + filename, + files, + this.pushPath, + this.state.counts, + filters + ); } else if (entry.isDirectory()) { let path = joinPath.joinDirectoryPath( entry.name, @@ -120,7 +145,7 @@ export class Walker { this.state.options.pathSeparator ); if (exclude && exclude(entry.name, path)) continue; - this.pushDirectory(path, paths, filters); + this.pushDirectory(path, files, this.pushPath, filters); this.walkDirectory(this.state, path, path, depth - 1, this.walk); } else if (this.resolveSymlink && entry.isSymbolicLink()) { let path = joinPath.joinPathWithBasePath(entry.name, directoryPath); @@ -151,12 +176,18 @@ export class Walker { this.state.options ); resolvedPath = this.joinPath(filename, directoryPath); - this.pushFile(resolvedPath, files, this.state.counts, filters); + this.pushFile( + resolvedPath, + files, + this.pushPath, + this.state.counts, + filters + ); } }); } } - this.groupFiles(this.state.groups, directoryPath, files); + this.groupFiles(this.state.groups, directoryPath, files, this.pushGroup); }; } diff --git a/src/builder/api-builder.ts b/src/builder/api-builder.ts index 4c1af175..743aa0e0 100644 --- a/src/builder/api-builder.ts +++ b/src/builder/api-builder.ts @@ -1,6 +1,7 @@ import { callback, promise } from "../api/async"; import { sync } from "../api/sync"; -import { Options, Output, ResultCallback } from "../types"; +import { iterator } from "../api/iterator"; +import { Options, Output, ResultCallback, IterableOutput } from "../types"; export class APIBuilder { constructor( @@ -19,4 +20,11 @@ export class APIBuilder { sync(): TReturnType { return sync(this.root, this.options); } + + withIterator(): TReturnType extends IterableOutput + ? AsyncIterable + : never { + // TODO (43081j): get rid of this awful `never` + return iterator(this.root, this.options) as never; + } } diff --git a/src/types.ts b/src/types.ts index da900e01..32f52834 100644 --- a/src/types.ts +++ b/src/types.ts @@ -26,6 +26,7 @@ export type OnlyCountsOutput = Counts; export type PathsOutput = string[]; export type Output = OnlyCountsOutput | PathsOutput | GroupOutput; +export type IterableOutput = PathsOutput | GroupOutput; export type FSLike = { readdir: typeof nativeFs.readdir; From 7c3bd05cd886c5533891cfda8531209088ae1c17 Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Tue, 1 Jul 2025 15:05:28 +0100 Subject: [PATCH 02/19] wip: crazy next function Works almost but is an absolute trainwreck. Will write it better now that i know the problem. --- src/api/iterator.ts | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/api/iterator.ts b/src/api/iterator.ts index 7641b3dd..0e034d94 100644 --- a/src/api/iterator.ts +++ b/src/api/iterator.ts @@ -2,7 +2,7 @@ import { Options, IterableOutput } from "../types"; import { Walker } from "./walker"; class WalkerIterator { - #next: Promise; + #next: Promise; #resolver: (value: TOutput[number] | null) => void; #walker: Walker; #currentGroup?: string[]; @@ -29,7 +29,7 @@ class WalkerIterator { #pushResult = async (result: TOutput[number]) => { this.#resolver(result); - this.#next = this.#createNext(); + this.#next = this.#createNext(this.#next); }; #onComplete = () => { @@ -37,10 +37,21 @@ class WalkerIterator { this.#resolver(null); }; - #createNext(): Promise { - return new Promise((resolve) => { + async #createNext(prev?: Promise): Promise { + const next = new Promise((resolve) => { this.#resolver = resolve; }); + if (prev) { + const prevResult = await prev; + const nextResult = await next; + if (prevResult === null || nextResult === null) { + return null; + } + return [...prevResult, nextResult] as TOutput | null; + } else { + const nextResult = await next; + return nextResult === null ? nextResult : ([nextResult] as TOutput); + } } async *[Symbol.asyncIterator]() { @@ -55,7 +66,7 @@ class WalkerIterator { if (result === null) { finished = true; } else { - yield result; + yield* result; } } } From a8bf947eed2c87d75ccceca44a080ab87226c2c7 Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Wed, 2 Jul 2025 12:42:46 +0100 Subject: [PATCH 03/19] wip: use a queue Of course, we need a queue! Now things work because we're trying to async iterate a synchronous task within an asynchronous task. --- src/api/iterator.ts | 53 +++++++++++++++++---------------------------- 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/src/api/iterator.ts b/src/api/iterator.ts index 0e034d94..da79592f 100644 --- a/src/api/iterator.ts +++ b/src/api/iterator.ts @@ -2,14 +2,12 @@ import { Options, IterableOutput } from "../types"; import { Walker } from "./walker"; class WalkerIterator { - #next: Promise; - #resolver: (value: TOutput[number] | null) => void; + #resolver?: (result: TOutput[number]) => void; #walker: Walker; #currentGroup?: string[]; + #queue: TOutput[number][] = []; public constructor(root: string, options: Options) { - this.#resolver = () => {}; - this.#next = this.#createNext(); const pushPath = options.group ? this.#pushPath : this.#pushResult; this.#walker = new Walker( root, @@ -28,48 +26,37 @@ class WalkerIterator { }; #pushResult = async (result: TOutput[number]) => { - this.#resolver(result); - this.#next = this.#createNext(this.#next); + this.#queue.push(result); + if (this.#resolver) { + const resolver = this.#resolver; + this.#resolver = undefined; + resolver(result); + } }; #onComplete = () => { this.#currentGroup = undefined; - this.#resolver(null); + this.#complete = true; }; - async #createNext(prev?: Promise): Promise { - const next = new Promise((resolve) => { - this.#resolver = resolve; - }); - if (prev) { - const prevResult = await prev; - const nextResult = await next; - if (prevResult === null || nextResult === null) { - return null; - } - return [...prevResult, nextResult] as TOutput | null; - } else { - const nextResult = await next; - return nextResult === null ? nextResult : ([nextResult] as TOutput); - } - } - async *[Symbol.asyncIterator]() { - let promise = this.#next; this.#walker.start(); - let finished = false; + while (true) { + yield* this.#queue; + this.#queue = []; - while (!finished) { - const result = await promise; - promise = this.#next; - if (result === null) { - finished = true; - } else { - yield* result; + if (this.#complete) { + return; } + + await new Promise((resolve) => { + this.#resolver = resolve; + }); } } + + #complete: boolean = false; } export function iterator( From 5577c62e8512fe3e2776bd0d28f8bb6888fb3779 Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Wed, 2 Jul 2025 12:56:40 +0100 Subject: [PATCH 04/19] fix: resolve on complete Sometimes it is possible we complete before the next iteration has run, so we leave a promise dangling or some such thing. --- src/api/iterator.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/api/iterator.ts b/src/api/iterator.ts index da79592f..358ba0bd 100644 --- a/src/api/iterator.ts +++ b/src/api/iterator.ts @@ -2,7 +2,7 @@ import { Options, IterableOutput } from "../types"; import { Walker } from "./walker"; class WalkerIterator { - #resolver?: (result: TOutput[number]) => void; + #resolver?: () => void; #walker: Walker; #currentGroup?: string[]; #queue: TOutput[number][] = []; @@ -30,13 +30,18 @@ class WalkerIterator { if (this.#resolver) { const resolver = this.#resolver; this.#resolver = undefined; - resolver(result); + resolver(); } }; #onComplete = () => { this.#currentGroup = undefined; this.#complete = true; + if (this.#resolver) { + const resolver = this.#resolver; + this.#resolver = undefined; + resolver(); + } }; async *[Symbol.asyncIterator]() { @@ -50,7 +55,7 @@ class WalkerIterator { return; } - await new Promise((resolve) => { + await new Promise((resolve) => { this.#resolver = resolve; }); } From 13af68e1b657316cbbb485e09fbbed6002729bb2 Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Wed, 2 Jul 2025 13:58:57 +0100 Subject: [PATCH 05/19] fix: support maxFiles without slicing We currently exceed `maxFiles` and slice the result later. This doesn't work in an iterator since there is no end "result". This change instead makes the walker stop walking when `maxFiles` is met, so we already have the correctly sized result set. --- __tests__/fdir.test.ts | 25 ++++------------------ __tests__/symlinks.test.ts | 32 ++++++++++++++--------------- __tests__/utils.ts | 18 ++++++++++++++++ src/api/functions/push-directory.ts | 15 ++++++++++---- src/api/functions/push-file.ts | 9 +++++--- src/api/functions/walk-directory.ts | 2 -- src/api/iterator.ts | 10 ++++++++- src/api/walker.ts | 11 +++++++--- 8 files changed, 72 insertions(+), 50 deletions(-) diff --git a/__tests__/fdir.test.ts b/__tests__/fdir.test.ts index d1a50583..0c355227 100644 --- a/__tests__/fdir.test.ts +++ b/__tests__/fdir.test.ts @@ -1,12 +1,11 @@ -import { fdir, IterableOutput } from "../src/index"; +import { fdir } from "../src/index"; import fs from "fs"; import mock from "mock-fs"; -import { test, beforeEach, TestContext, vi } from "vitest"; +import { test, beforeEach, vi } from "vitest"; import path, { sep } from "path"; import { convertSlashes } from "../src/utils"; import picomatch from "picomatch"; -import { apiTypes, APITypes, cwd, restricted, root } from "./utils"; -import { APIBuilder } from "../src/builder/api-builder"; +import { apiTypes, APITypes, cwd, restricted, root, execute } from "./utils"; beforeEach(() => { mock.restore(); @@ -26,22 +25,6 @@ test(`crawl single depth directory with callback`, (t) => { }); }); -async function execute( - api: APIBuilder, - type: APITypes -): Promise { - let files: T[number][] = []; - - if (type === "withIterator") { - for await (const file of api[type]()) { - files.push(file); - } - } else { - files = await api[type](); - } - return files as T; -} - async function crawl(type: APITypes, path: string) { const api = new fdir().crawl(path); return execute(api, type); @@ -270,7 +253,7 @@ for (const type of apiTypes) { .onlyDirs() .filter((path) => path.includes("api")) .crawl("./src"); - const result = await api[type](); + const result = await execute(api, type); t.expect(result).toHaveLength(2); }); diff --git a/__tests__/symlinks.test.ts b/__tests__/symlinks.test.ts index 0a61ec2e..3a311616 100644 --- a/__tests__/symlinks.test.ts +++ b/__tests__/symlinks.test.ts @@ -1,7 +1,7 @@ -import { afterAll, beforeAll, beforeEach, describe, test } from "vitest"; -import { apiTypes, normalize, root } from "./utils"; +import { afterAll, beforeAll, describe, test } from "vitest"; +import { apiTypes, normalize, root, execute } from "./utils"; import mock from "mock-fs"; -import { fdir, Options } from "../src"; +import { fdir } from "../src"; import path from "path"; const fsWithRelativeSymlinks = { @@ -154,7 +154,7 @@ for (const type of apiTypes) { test(`resolve symlinks`, async (t) => { const api = new fdir().withSymlinks().crawl("/some/dir"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize([ "/other/dir/file-2", @@ -166,7 +166,7 @@ for (const type of apiTypes) { test(`resolve recursive symlinks`, async (t) => { const api = new fdir().withSymlinks().crawl("/recursive"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize([ "/double/recursive/another-file", @@ -184,7 +184,7 @@ for (const type of apiTypes) { const api = new fdir() .withSymlinks({ resolvePaths: false }) .crawl("/recursive"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize([ "/recursive/dir/not-recursive/another-file", @@ -234,7 +234,7 @@ for (const type of apiTypes) { .withRelativePaths() .withErrors() .crawl("./recursive"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize([ "dir/not-recursive/another-file", @@ -284,7 +284,7 @@ for (const type of apiTypes) { .withRelativePaths() .withErrors() .crawl("./recursive"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize([ "..//double/recursive/another-file", @@ -302,7 +302,7 @@ for (const type of apiTypes) { const api = new fdir() .withSymlinks({ resolvePaths: false }) .crawl("/some/dir"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize([ "/some/dir/dirSymlink/file-1", @@ -317,7 +317,7 @@ for (const type of apiTypes) { .withSymlinks({ resolvePaths: false }) .withRelativePaths() .crawl("/some/dir"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize([ "dirSymlink/file-1", @@ -332,7 +332,7 @@ for (const type of apiTypes) { .withSymlinks() .withRelativePaths() .crawl("./relative/dir"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize([ "../../../../other-relative/dir/file-2", @@ -347,7 +347,7 @@ for (const type of apiTypes) { .withSymlinks() .exclude((_name, path) => path === resolveSymlinkRoot("/sym/linked/")) .crawl("/some/dir"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual(normalize(["/other/dir/file-2"])); }); @@ -358,7 +358,7 @@ for (const type of apiTypes) { (_name, path) => path === resolveSymlinkRoot("/some/dir/dirSymlink/") ) .crawl("/some/dir"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize(["/some/dir/fileSymlink"]) ); @@ -366,7 +366,7 @@ for (const type of apiTypes) { test(`do not resolve symlinks`, async (t) => { const api = new fdir().crawl("/some/dir"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize(["dirSymlink", "fileSymlink", "fileSymlink2"]) ); @@ -374,7 +374,7 @@ for (const type of apiTypes) { test(`exclude symlinks`, async (t) => { const api = new fdir({ excludeSymlinks: true }).crawl("/some/dir"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files).toHaveLength(0); }); @@ -382,7 +382,7 @@ for (const type of apiTypes) { "doesn't hang when resolving symlinks in the root directory", async (t) => { const api = new fdir().withSymlinks({ resolvePaths: false }).crawl("/"); - const files = await api[type](); + const files = await execute(api, type); const expectedFiles = normalize(["/lib/file-1", "/usr/lib/file-1"]); for (const expectedFile of expectedFiles) { t.expect(files).toContain(expectedFile); diff --git a/__tests__/utils.ts b/__tests__/utils.ts index 1123ba11..b7db4888 100644 --- a/__tests__/utils.ts +++ b/__tests__/utils.ts @@ -1,4 +1,6 @@ import path from "path"; +import type { APIBuilder } from "../src/builder/api-builder"; +import type { IterableOutput } from "../src/types"; export type APITypes = (typeof apiTypes)[number]; export const apiTypes = ["withPromise", "sync", "withIterator"] as const; @@ -22,3 +24,19 @@ export function normalize(paths: string[]) { path.isAbsolute(p) ? path.resolve(p) : path.normalize(p) ); } + +export async function execute( + api: APIBuilder, + type: APITypes +): Promise { + let files: T[number][] = []; + + if (type === "withIterator") { + for await (const file of api[type]()) { + files.push(file); + } + } else { + files = await api[type](); + } + return files as T; +} diff --git a/src/api/functions/push-directory.ts b/src/api/functions/push-directory.ts index 17c836a4..e2720792 100644 --- a/src/api/functions/push-directory.ts +++ b/src/api/functions/push-directory.ts @@ -1,25 +1,28 @@ -import { FilterPredicate, Options } from "../../types"; +import { FilterPredicate, Options, Counts } from "../../types"; export type PushDirectoryFunction = ( directoryPath: string, paths: string[], pushPath: (path: string, arr: string[]) => void, + counts: Counts, filters?: FilterPredicate[] ) => void; function pushDirectoryWithRelativePath(root: string): PushDirectoryFunction { - return function (directoryPath, paths, pushPath) { + return function (directoryPath, paths, pushPath, counts) { pushPath(directoryPath.substring(root.length) || ".", paths); + counts.directories++; }; } function pushDirectoryFilterWithRelativePath( root: string ): PushDirectoryFunction { - return function (directoryPath, paths, pushPath, filters) { + return function (directoryPath, paths, pushPath, counts, filters) { const relativePath = directoryPath.substring(root.length) || "."; if (filters!.every((filter) => filter(relativePath, true))) { pushPath(relativePath, paths); + counts.directories++; } }; } @@ -27,20 +30,24 @@ function pushDirectoryFilterWithRelativePath( const pushDirectory: PushDirectoryFunction = ( directoryPath, paths, - pushPath + pushPath, + counts ) => { pushPath(directoryPath || ".", paths); + counts.directories++; }; const pushDirectoryFilter: PushDirectoryFunction = ( directoryPath, paths, pushPath, + counts, filters ) => { const path = directoryPath || "."; if (filters!.every((filter) => filter(path, true))) { pushPath(path, paths); + counts.directories++; } }; diff --git a/src/api/functions/push-file.ts b/src/api/functions/push-file.ts index 2e850f54..26a92050 100644 --- a/src/api/functions/push-file.ts +++ b/src/api/functions/push-file.ts @@ -22,11 +22,13 @@ const pushFileFilter: PushFileFunction = ( filename, paths, pushPath, - _counts, + counts, filters ) => { - if (filters!.every((filter) => filter(filename, false))) + if (filters!.every((filter) => filter(filename, false))) { pushPath(filename, paths); + counts.files++; + } }; const pushFileCount: PushFileFunction = ( @@ -39,8 +41,9 @@ const pushFileCount: PushFileFunction = ( counts.files++; }; -const pushFile: PushFileFunction = (filename, paths, pushPath) => { +const pushFile: PushFileFunction = (filename, paths, pushPath, counts) => { pushPath(filename, paths); + counts.files++; }; const empty: PushFileFunction = () => {}; diff --git a/src/api/functions/walk-directory.ts b/src/api/functions/walk-directory.ts index f2b9d368..a5c5f8fa 100644 --- a/src/api/functions/walk-directory.ts +++ b/src/api/functions/walk-directory.ts @@ -25,7 +25,6 @@ const walkAsync: WalkDirectoryFunction = ( const { fs } = state; state.visited.push(crawlPath); - state.counts.directories++; // Perf: Node >= 10 introduced withFileTypes that helps us // skip an extra fs.stat call. @@ -46,7 +45,6 @@ const walkSync: WalkDirectoryFunction = ( const { fs } = state; if (currentDepth < 0) return; state.visited.push(crawlPath); - state.counts.directories++; let entries: Dirent[] = []; try { diff --git a/src/api/iterator.ts b/src/api/iterator.ts index 358ba0bd..a31330d1 100644 --- a/src/api/iterator.ts +++ b/src/api/iterator.ts @@ -6,6 +6,7 @@ class WalkerIterator { #walker: Walker; #currentGroup?: string[]; #queue: TOutput[number][] = []; + #error?: unknown; public constructor(root: string, options: Options) { const pushPath = options.group ? this.#pushPath : this.#pushResult; @@ -34,9 +35,12 @@ class WalkerIterator { } }; - #onComplete = () => { + #onComplete = (err: unknown) => { this.#currentGroup = undefined; this.#complete = true; + if (err) { + this.#error = err; + } if (this.#resolver) { const resolver = this.#resolver; this.#resolver = undefined; @@ -51,6 +55,10 @@ class WalkerIterator { yield* this.#queue; this.#queue = []; + if (this.#error) { + throw this.#error; + } + if (this.#complete) { return; } diff --git a/src/api/walker.ts b/src/api/walker.ts index 9d9d3546..775a0f71 100644 --- a/src/api/walker.ts +++ b/src/api/walker.ts @@ -87,6 +87,7 @@ export class Walker { this.root, this.state.paths, this.pushPath, + this.state.counts, this.state.options.filters ); this.walkDirectory( @@ -101,7 +102,7 @@ export class Walker { private walk = (entries: Dirent[], directoryPath: string, depth: number) => { const { - paths, + counts, options: { filters, resolveSymlinks, @@ -118,12 +119,16 @@ export class Walker { if ( controller.aborted || (signal && signal.aborted) || - (maxFiles && paths.length > maxFiles) + (maxFiles && counts.directories + counts.files > maxFiles) ) return; const files = this.getArray(this.state.paths); for (let i = 0; i < entries.length; ++i) { + if (maxFiles && counts.directories + counts.files >= maxFiles) { + break; + } + const entry = entries[i]; if ( @@ -145,7 +150,7 @@ export class Walker { this.state.options.pathSeparator ); if (exclude && exclude(entry.name, path)) continue; - this.pushDirectory(path, files, this.pushPath, filters); + this.pushDirectory(path, files, this.pushPath, counts, filters); this.walkDirectory(this.state, path, path, depth - 1, this.walk); } else if (this.resolveSymlink && entry.isSymbolicLink()) { let path = joinPath.joinPathWithBasePath(entry.name, directoryPath); From d5dc023f3f7d5b16564d3a60983b2b7f3bdb07b8 Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Sun, 6 Jul 2025 18:58:25 +0100 Subject: [PATCH 06/19] fix: abort the walker when returned early --- src/api/iterator.ts | 28 ++++++++++++++++------------ src/api/walker.ts | 4 ++++ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/api/iterator.ts b/src/api/iterator.ts index a31330d1..c2ce7a6c 100644 --- a/src/api/iterator.ts +++ b/src/api/iterator.ts @@ -51,21 +51,25 @@ class WalkerIterator { async *[Symbol.asyncIterator]() { this.#walker.start(); - while (true) { - yield* this.#queue; - this.#queue = []; + try { + while (true) { + yield* this.#queue; + this.#queue = []; - if (this.#error) { - throw this.#error; - } + if (this.#error) { + throw this.#error; + } - if (this.#complete) { - return; - } + if (this.#complete) { + return; + } - await new Promise((resolve) => { - this.#resolver = resolve; - }); + await new Promise((resolve) => { + this.#resolver = resolve; + }); + } + } finally { + this.#walker.stop(); } } diff --git a/src/api/walker.ts b/src/api/walker.ts index 775a0f71..bb9fedf1 100644 --- a/src/api/walker.ts +++ b/src/api/walker.ts @@ -100,6 +100,10 @@ export class Walker { return this.isSynchronous ? this.callbackInvoker(this.state, null) : null; } + stop(): void { + this.state.controller.abort(); + } + private walk = (entries: Dirent[], directoryPath: string, depth: number) => { const { counts, From 966dac8f038f608b7b08339bcec4761db94f12ff Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Sun, 6 Jul 2025 19:36:12 +0100 Subject: [PATCH 07/19] fix: break walker if aborted --- src/api/walker.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/api/walker.ts b/src/api/walker.ts index bb9fedf1..39fdf998 100644 --- a/src/api/walker.ts +++ b/src/api/walker.ts @@ -133,6 +133,10 @@ export class Walker { break; } + if (this.state.controller.aborted || (signal && signal.aborted)) { + break; + } + const entry = entries[i]; if ( From a3801e126f2ee58f4bf83b9326755ce9abf6ca7e Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Sun, 6 Jul 2025 20:10:24 +0100 Subject: [PATCH 08/19] fix: expose aborted state of walker --- src/api/iterator.ts | 9 +++++++-- src/api/walker.ts | 15 ++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/api/iterator.ts b/src/api/iterator.ts index c2ce7a6c..5137f15b 100644 --- a/src/api/iterator.ts +++ b/src/api/iterator.ts @@ -53,14 +53,19 @@ class WalkerIterator { try { while (true) { - yield* this.#queue; + for (const item of this.#queue) { + if (this.#walker.aborted || this.#complete) { + break; + } + yield item; + } this.#queue = []; if (this.#error) { throw this.#error; } - if (this.#complete) { + if (this.#complete || this.#walker.aborted) { return; } diff --git a/src/api/walker.ts b/src/api/walker.ts index 39fdf998..2b85be7f 100644 --- a/src/api/walker.ts +++ b/src/api/walker.ts @@ -82,6 +82,14 @@ export class Walker { }); } + get aborted(): boolean { + const { + controller, + options: { signal }, + } = this.state; + return controller.aborted || (signal !== undefined && signal.aborted); + } + start(): TOutput | null { this.pushDirectory( this.root, @@ -113,16 +121,13 @@ export class Walker { excludeSymlinks, exclude, maxFiles, - signal, useRealPaths, pathSeparator, }, - controller, } = this.state; if ( - controller.aborted || - (signal && signal.aborted) || + this.aborted || (maxFiles && counts.directories + counts.files > maxFiles) ) return; @@ -133,7 +138,7 @@ export class Walker { break; } - if (this.state.controller.aborted || (signal && signal.aborted)) { + if (this.aborted) { break; } From f4ecba0fe9d57f80477889fdf750bc8913fb54b5 Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Sun, 6 Jul 2025 22:04:01 +0100 Subject: [PATCH 09/19] fix: only bail if aborted --- src/api/iterator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/iterator.ts b/src/api/iterator.ts index 5137f15b..571acd0c 100644 --- a/src/api/iterator.ts +++ b/src/api/iterator.ts @@ -54,7 +54,7 @@ class WalkerIterator { try { while (true) { for (const item of this.#queue) { - if (this.#walker.aborted || this.#complete) { + if (this.#walker.aborted) { break; } yield item; From 0013e5bd88389a103e701b5f70fcc19a9ea65f7f Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Sun, 6 Jul 2025 23:28:59 +0100 Subject: [PATCH 10/19] test: add interrupt tests --- __tests__/fdir.test.ts | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/__tests__/fdir.test.ts b/__tests__/fdir.test.ts index 0c355227..78902686 100644 --- a/__tests__/fdir.test.ts +++ b/__tests__/fdir.test.ts @@ -401,8 +401,8 @@ for (const type of apiTypes) { const api = new fdir({ fs: fakeFs, }).crawl("node_modules"); - await api[type](); - if (type === "withPromise") { + await execute(api, type); + if (type === "withPromise" || type === "withIterator") { t.expect(readdirStub).toHaveBeenCalled(); } else { t.expect(readdirSyncStub).toHaveBeenCalled(); @@ -491,3 +491,27 @@ test(`do not convert \\\\ to \\`, async (t) => { "\\\\wsl.localhost\\Ubuntu\\home\\" ); }); + +test("interrupted iterator should stop yielding results", async (t) => { + const api = new fdir().crawl("./src"); + const iterator = api.withIterator()[Symbol.asyncIterator](); + const results: string[] = []; + let next = await iterator.next(); + do { + results.push(next.value); + iterator.return?.(); + } while (next.done !== false); + t.expect(results.length).toBe(1); +}); + +test("aborted iterator should stop yielding results", async (t) => { + const aborter = new AbortController(); + const api = new fdir().withAbortSignal(aborter.signal).crawl("./src"); + const iterator = api.withIterator(); + const results: string[] = []; + for await (const value of iterator) { + results.push(value); + aborter.abort(); + } + t.expect(results.length).toBe(1); +}); From 4860e5f73a46f51944178a412ac5453baac5801d Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Mon, 7 Jul 2025 08:58:41 +0100 Subject: [PATCH 11/19] test: use conditional tests for aborts --- __tests__/fdir.test.ts | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/__tests__/fdir.test.ts b/__tests__/fdir.test.ts index 78902686..5564a4ea 100644 --- a/__tests__/fdir.test.ts +++ b/__tests__/fdir.test.ts @@ -7,6 +7,9 @@ import { convertSlashes } from "../src/utils"; import picomatch from "picomatch"; import { apiTypes, APITypes, cwd, restricted, root, execute } from "./utils"; +// AbortController is not present on Node v14 +const hasAbortController = "AbortController" in globalThis; + beforeEach(() => { mock.restore(); }); @@ -410,9 +413,9 @@ for (const type of apiTypes) { }); } -// AbortController is not present on Node v14 -if ("AbortController" in globalThis) { - test(`[async] crawl directory & use abort signal to abort`, async (t) => { +test.runIf(hasAbortController)( + `[async] crawl directory & use abort signal to abort`, + async (t) => { const totalFiles = new fdir().onlyCounts().crawl("node_modules").sync(); const abortController = new AbortController(); const api = new fdir() @@ -424,8 +427,8 @@ if ("AbortController" in globalThis) { .crawl("node_modules"); const files = await api.withPromise(); t.expect(files.length).toBeLessThan(totalFiles.files); - }); -} + } +); test(`paths should never start with ./`, async (t) => { const apis = [ @@ -504,14 +507,17 @@ test("interrupted iterator should stop yielding results", async (t) => { t.expect(results.length).toBe(1); }); -test("aborted iterator should stop yielding results", async (t) => { - const aborter = new AbortController(); - const api = new fdir().withAbortSignal(aborter.signal).crawl("./src"); - const iterator = api.withIterator(); - const results: string[] = []; - for await (const value of iterator) { - results.push(value); - aborter.abort(); +test.runIf(hasAbortController)( + "aborted iterator should stop yielding results", + async (t) => { + const aborter = new AbortController(); + const api = new fdir().withAbortSignal(aborter.signal).crawl("./src"); + const iterator = api.withIterator(); + const results: string[] = []; + for await (const value of iterator) { + results.push(value); + aborter.abort(); + } + t.expect(results.length).toBe(1); } - t.expect(results.length).toBe(1); -}); +); From 8cc1f858cf68eb12950e45df5d07d100892687ff Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Mon, 7 Jul 2025 08:59:25 +0100 Subject: [PATCH 12/19] fix: avoid nullish coalesce --- src/api/walker.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/walker.ts b/src/api/walker.ts index 2b85be7f..6f68c21d 100644 --- a/src/api/walker.ts +++ b/src/api/walker.ts @@ -71,12 +71,12 @@ export class Walker { this.resolveSymlink = resolveSymlink.build(options, this.isSynchronous); this.walkDirectory = walkDirectory.build(this.isSynchronous); this.pushPath = - pushPath ?? + pushPath || ((p, arr) => { arr.push(p); }); this.pushGroup = - pushGroup ?? + pushGroup || ((group, arr) => { arr.push(group); }); From d04ed18235414947ba98038825b9d49f6499d1ca Mon Sep 17 00:00:00 2001 From: Superchupu <53496941+SuperchupuDev@users.noreply.github.com> Date: Fri, 11 Jul 2025 12:55:02 +0200 Subject: [PATCH 13/19] refactor: return an `AsyncGenerator` instead (#1) * refactor: return an `AsyncGenerator` instead * chore: update test --- __tests__/fdir.test.ts | 6 +++--- src/api/iterator.ts | 8 ++++---- src/builder/api-builder.ts | 10 ++++++++-- src/types.ts | 5 +++++ 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/__tests__/fdir.test.ts b/__tests__/fdir.test.ts index 5564a4ea..4557d14f 100644 --- a/__tests__/fdir.test.ts +++ b/__tests__/fdir.test.ts @@ -497,12 +497,12 @@ test(`do not convert \\\\ to \\`, async (t) => { test("interrupted iterator should stop yielding results", async (t) => { const api = new fdir().crawl("./src"); - const iterator = api.withIterator()[Symbol.asyncIterator](); - const results: string[] = []; + const iterator = api.withIterator(); + const results: (string | void)[] = []; let next = await iterator.next(); do { results.push(next.value); - iterator.return?.(); + iterator.return(); } while (next.done !== false); t.expect(results.length).toBe(1); }); diff --git a/src/api/iterator.ts b/src/api/iterator.ts index 571acd0c..8c5a42cd 100644 --- a/src/api/iterator.ts +++ b/src/api/iterator.ts @@ -1,4 +1,4 @@ -import { Options, IterableOutput } from "../types"; +import { Options, IterableOutput, OutputIterator } from "../types"; import { Walker } from "./walker"; class WalkerIterator { @@ -48,7 +48,7 @@ class WalkerIterator { } }; - async *[Symbol.asyncIterator]() { + async *start(): OutputIterator { this.#walker.start(); try { @@ -85,6 +85,6 @@ export function iterator( root: string, options: Options ): AsyncIterable { - const iterator = new WalkerIterator(root, options); - return iterator; + const walker = new WalkerIterator(root, options); + return walker.start(); } diff --git a/src/builder/api-builder.ts b/src/builder/api-builder.ts index 743aa0e0..f3de1712 100644 --- a/src/builder/api-builder.ts +++ b/src/builder/api-builder.ts @@ -1,7 +1,13 @@ import { callback, promise } from "../api/async"; import { sync } from "../api/sync"; import { iterator } from "../api/iterator"; -import { Options, Output, ResultCallback, IterableOutput } from "../types"; +import { + Options, + Output, + ResultCallback, + IterableOutput, + OutputIterator, +} from "../types"; export class APIBuilder { constructor( @@ -22,7 +28,7 @@ export class APIBuilder { } withIterator(): TReturnType extends IterableOutput - ? AsyncIterable + ? OutputIterator : never { // TODO (43081j): get rid of this awful `never` return iterator(this.root, this.options) as never; diff --git a/src/types.ts b/src/types.ts index 32f52834..03eac737 100644 --- a/src/types.ts +++ b/src/types.ts @@ -27,6 +27,11 @@ export type PathsOutput = string[]; export type Output = OnlyCountsOutput | PathsOutput | GroupOutput; export type IterableOutput = PathsOutput | GroupOutput; +export type OutputIterator = AsyncGenerator< + T[number], + void, + void +>; export type FSLike = { readdir: typeof nativeFs.readdir; From 913225b61bd2399af0f944a8e5085c52b5f2e5b8 Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Fri, 11 Jul 2025 12:04:56 +0100 Subject: [PATCH 14/19] test: only push non-done values --- __tests__/fdir.test.ts | 6 ++++-- src/types.ts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/__tests__/fdir.test.ts b/__tests__/fdir.test.ts index 4557d14f..def6098a 100644 --- a/__tests__/fdir.test.ts +++ b/__tests__/fdir.test.ts @@ -498,10 +498,12 @@ test(`do not convert \\\\ to \\`, async (t) => { test("interrupted iterator should stop yielding results", async (t) => { const api = new fdir().crawl("./src"); const iterator = api.withIterator(); - const results: (string | void)[] = []; + const results: string[] = []; let next = await iterator.next(); do { - results.push(next.value); + if (!next.done) { + results.push(next.value); + } iterator.return(); } while (next.done !== false); t.expect(results.length).toBe(1); diff --git a/src/types.ts b/src/types.ts index 03eac737..3a74b25f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -30,7 +30,7 @@ export type IterableOutput = PathsOutput | GroupOutput; export type OutputIterator = AsyncGenerator< T[number], void, - void + undefined >; export type FSLike = { From 182b717b40d451eb67b7c675096e8cd5037cd450 Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Sat, 12 Jul 2025 16:33:18 +0100 Subject: [PATCH 15/19] wip: rip it up and do it again --- src/api/iterator-walker.ts | 280 +++++++++++++++++++++++++++++++++++++ src/api/iterator.ts | 3 +- src/types.ts | 2 +- 3 files changed, 283 insertions(+), 2 deletions(-) create mode 100644 src/api/iterator-walker.ts diff --git a/src/api/iterator-walker.ts b/src/api/iterator-walker.ts new file mode 100644 index 00000000..4441b2f0 --- /dev/null +++ b/src/api/iterator-walker.ts @@ -0,0 +1,280 @@ +import { basename, dirname } from "path"; +import { isRootDirectory, normalizePath } from "../utils"; +import { WalkerState, Options, OutputIterator, IterableOutput } from "../types"; +import * as joinPath from "./functions/join-path"; +import * as pushDirectory from "./functions/push-directory"; +import * as pushFile from "./functions/push-file"; +import * as resolveSymlink from "./functions/resolve-symlink"; +import * as walkDirectory from "./functions/walk-directory"; +import { Queue } from "./queue"; +import type { Dirent } from "fs"; +import * as nativeFs from "fs"; +import { Counter } from "./counter"; +import { Aborter } from "./aborter"; + +export class IteratorWalker { + private readonly root: string; + private readonly state: WalkerState; + private readonly joinPath: joinPath.JoinPathFunction; + private readonly pushDirectory: pushDirectory.PushDirectoryFunction; + private readonly pushFile: pushFile.PushFileFunction; + private readonly resolveSymlink: resolveSymlink.ResolveSymlinkFunction | null; + private readonly walkDirectory: walkDirectory.WalkDirectoryFunction; + #complete = false; + + constructor( + root: string, + options: Options, + ) { + this.root = normalizePath(root, options); + this.state = { + root: isRootDirectory(this.root) ? this.root : this.root.slice(0, -1), + // Perf: we explicitly tell the compiler to optimize for String arrays + paths: [""].slice(0, 0), + groups: [], + counts: new Counter(), + options, + queue: new Queue((error, state) => { + this.#complete = true; + }), + symlinks: new Map(), + visited: [""].slice(0, 0), + controller: new Aborter(), + fs: options.fs || nativeFs, + }; + + /* + * Perf: We conditionally change functions according to options. This gives a slight + * performance boost. Since these functions are so small, they are automatically inlined + * by the javascript engine so there's no function call overhead (in most cases). + */ + this.joinPath = joinPath.build(this.root, options); + this.pushDirectory = pushDirectory.build(this.root, options); + this.pushFile = pushFile.build(options); + this.resolveSymlink = resolveSymlink.build(options, false); + this.walkDirectory = walkDirectory.build(false); + } + + get aborted(): boolean { + const { + controller, + options: { signal }, + } = this.state; + return controller.aborted || (signal !== undefined && signal.aborted); + } + + #pushDirectory( + directoryPath: string, + ): Promise { + return new Promise((resolve) => { + let pushed: string | null = null; + // this is synchronous. if we ever make pushDirectory async, + // rework everything! + this.pushDirectory( + directoryPath, + this.state.paths, + (pushedPath) => { + pushed = pushedPath; + }, + this.state.counts, + this.state.options.filters + ); + resolve(pushed); + }); + } + + #pushFile( + filePath: string, + ): Promise { + return new Promise((resolve) => { + let pushed: string | null = null; + // this is synchronous. if we ever make pushFile async, + // rework everything! + this.pushFile( + filePath, + this.state.paths, + (pushedPath) => { + pushed = pushedPath; + }, + this.state.counts, + this.state.options.filters + ); + resolve(pushed); + }); + } + + #resolveSymlink( + symlinkPath: string + ): Promise<{stat: nativeFs.Stats; resolvedPath: string;} | null> { + return new Promise((resolve) => { + if (!this.resolveSymlink) { + resolve(null); + return; + } + + // WONT ACTUALLY RESOLVE! terrible promise + this.resolveSymlink(symlinkPath, this.state, (stat, resolvedPath) => { + resolve({stat, resolvedPath}); + }); + }); + } + + #walkDirectory( + crawlPath: string, + directoryPath: string, + depth: number, + ): Promise<{ + entries: Dirent[]; + directoryPath: string; + depth: number; + }> { + return new Promise<{ + entries: Dirent[]; + directoryPath: string; + depth: number; + }>((resolve) => { + this.walkDirectory( + this.state, + crawlPath, + directoryPath, + depth, + (entries, resultDirectoryPath, resultDepth) => + resolve({entries, directoryPath: resultDirectoryPath, depth: resultDepth}) + ); + }); + } + + async *start(): OutputIterator { + let pushedPath = await this.#pushDirectory(this.root); + + if (pushedPath !== null) { + yield pushedPath; + } + + const toWalk: Array<{ + crawlPath: string; + directoryPath: string; + depth: number; + }> = []; + let currentWalk: { + crawlPath: string; + directoryPath: string; + depth: number; + } | undefined = { + crawlPath: this.root, + directoryPath: this.root, + depth: this.state.options.maxDepth, + }; + const { + counts, + options: { + resolveSymlinks, + excludeSymlinks, + exclude, + maxFiles, + useRealPaths, + pathSeparator, + }, + } = this.state; + + while (currentWalk) { + if (this.aborted || this.#complete) { + break; + } + if (maxFiles && counts.directories + counts.files > maxFiles) { + break; + } + + const results = await this.#walkDirectory( + currentWalk.crawlPath, + currentWalk.directoryPath, + currentWalk.depth + ); + + for (const entry of results.entries) { + if (maxFiles && counts.directories + counts.files >= maxFiles) { + break; + } + + if (this.aborted) { + break; + } + + if ( + entry.isFile() || + (entry.isSymbolicLink() && !resolveSymlinks && !excludeSymlinks) + ) { + const filename = this.joinPath(entry.name, results.directoryPath); + pushedPath = await this.#pushFile(filename); + if (pushedPath !== null) { + yield pushedPath; + } + } else if (entry.isDirectory()) { + let path = joinPath.joinDirectoryPath( + entry.name, + results.directoryPath, + this.state.options.pathSeparator + ); + if (exclude && exclude(entry.name, path)) continue; + pushedPath = await this.#pushDirectory(path); + if (pushedPath !== null) { + yield pushedPath; + } + toWalk.push({ + directoryPath: path, + crawlPath: path, + depth: results.depth - 1 + }); + } else if (this.resolveSymlink && entry.isSymbolicLink()) { + let path = joinPath.joinPathWithBasePath(entry.name, results.directoryPath); + const resolvedSymlink = await this.#resolveSymlink( + path + ); + + if (resolvedSymlink === null) { + continue; + } + + if (resolvedSymlink.stat.isDirectory()) { + const normalized = normalizePath(resolvedSymlink.resolvedPath, this.state.options); + + if ( + exclude && + exclude( + entry.name, + useRealPaths ? normalized : path + pathSeparator + ) + ) { + continue; + } + + toWalk.push({ + crawlPath: normalized, + directoryPath: useRealPaths ? normalized : path + pathSeparator, + depth: results.depth - 1 + }); + } else { + const normalized = useRealPaths ? resolvedSymlink.resolvedPath : path; + const filename = basename(normalized); + const directoryPath = normalizePath( + dirname(normalized), + this.state.options + ); + pushedPath = await this.#pushFile( + this.joinPath(filename, directoryPath) + ); + if (pushedPath !== null) { + yield pushedPath; + } + } + } + } + + currentWalk = toWalk.pop(); + } + } + + stop(): void { + this.state.controller.abort(); + } +} diff --git a/src/api/iterator.ts b/src/api/iterator.ts index 8c5a42cd..b662d234 100644 --- a/src/api/iterator.ts +++ b/src/api/iterator.ts @@ -1,5 +1,6 @@ import { Options, IterableOutput, OutputIterator } from "../types"; import { Walker } from "./walker"; +import { IteratorWalker } from "./iterator-walker"; class WalkerIterator { #resolver?: () => void; @@ -85,6 +86,6 @@ export function iterator( root: string, options: Options ): AsyncIterable { - const walker = new WalkerIterator(root, options); + const walker = new IteratorWalker(root, options); return walker.start(); } diff --git a/src/types.ts b/src/types.ts index 3a74b25f..5f10c6fc 100644 --- a/src/types.ts +++ b/src/types.ts @@ -26,7 +26,7 @@ export type OnlyCountsOutput = Counts; export type PathsOutput = string[]; export type Output = OnlyCountsOutput | PathsOutput | GroupOutput; -export type IterableOutput = PathsOutput | GroupOutput; +export type IterableOutput = PathsOutput; export type OutputIterator = AsyncGenerator< T[number], void, From fb2bd70de5c092bd34f6be0a51bc8dbaf751c179 Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Sun, 13 Jul 2025 11:20:15 +0100 Subject: [PATCH 16/19] wip: churn more of it out --- src/api/functions/resolve-symlink.ts | 6 +- src/api/iterator-walker.ts | 171 ++++++++++++++------------- src/api/iterator.ts | 83 +------------ 3 files changed, 93 insertions(+), 167 deletions(-) diff --git a/src/api/functions/resolve-symlink.ts b/src/api/functions/resolve-symlink.ts index b23cdb2b..b572287b 100644 --- a/src/api/functions/resolve-symlink.ts +++ b/src/api/functions/resolve-symlink.ts @@ -68,7 +68,11 @@ export function build( return isSynchronous ? resolveSymlinks : resolveSymlinksAsync; } -function isRecursive(path: string, resolved: string, state: WalkerState) { +export function isRecursive( + path: string, + resolved: string, + state: WalkerState +) { if (state.options.useRealPaths) return isRecursiveUsingRealPaths(resolved, state); diff --git a/src/api/iterator-walker.ts b/src/api/iterator-walker.ts index 4441b2f0..c948bc90 100644 --- a/src/api/iterator-walker.ts +++ b/src/api/iterator-walker.ts @@ -5,12 +5,12 @@ import * as joinPath from "./functions/join-path"; import * as pushDirectory from "./functions/push-directory"; import * as pushFile from "./functions/push-file"; import * as resolveSymlink from "./functions/resolve-symlink"; -import * as walkDirectory from "./functions/walk-directory"; import { Queue } from "./queue"; import type { Dirent } from "fs"; import * as nativeFs from "fs"; import { Counter } from "./counter"; import { Aborter } from "./aborter"; +import { promisify } from "node:util"; export class IteratorWalker { private readonly root: string; @@ -19,40 +19,27 @@ export class IteratorWalker { private readonly pushDirectory: pushDirectory.PushDirectoryFunction; private readonly pushFile: pushFile.PushFileFunction; private readonly resolveSymlink: resolveSymlink.ResolveSymlinkFunction | null; - private readonly walkDirectory: walkDirectory.WalkDirectoryFunction; #complete = false; - constructor( - root: string, - options: Options, - ) { + constructor(root: string, options: Options) { this.root = normalizePath(root, options); this.state = { root: isRootDirectory(this.root) ? this.root : this.root.slice(0, -1), - // Perf: we explicitly tell the compiler to optimize for String arrays - paths: [""].slice(0, 0), + paths: [], groups: [], counts: new Counter(), options, - queue: new Queue((error, state) => { - this.#complete = true; - }), + queue: new Queue(() => {}), symlinks: new Map(), visited: [""].slice(0, 0), controller: new Aborter(), fs: options.fs || nativeFs, }; - /* - * Perf: We conditionally change functions according to options. This gives a slight - * performance boost. Since these functions are so small, they are automatically inlined - * by the javascript engine so there's no function call overhead (in most cases). - */ this.joinPath = joinPath.build(this.root, options); this.pushDirectory = pushDirectory.build(this.root, options); this.pushFile = pushFile.build(options); this.resolveSymlink = resolveSymlink.build(options, false); - this.walkDirectory = walkDirectory.build(false); } get aborted(): boolean { @@ -63,9 +50,7 @@ export class IteratorWalker { return controller.aborted || (signal !== undefined && signal.aborted); } - #pushDirectory( - directoryPath: string, - ): Promise { + #pushDirectory(directoryPath: string): Promise { return new Promise((resolve) => { let pushed: string | null = null; // this is synchronous. if we ever make pushDirectory async, @@ -83,9 +68,7 @@ export class IteratorWalker { }); } - #pushFile( - filePath: string, - ): Promise { + #pushFile(filePath: string): Promise { return new Promise((resolve) => { let pushed: string | null = null; // this is synchronous. if we ever make pushFile async, @@ -103,48 +86,72 @@ export class IteratorWalker { }); } - #resolveSymlink( + async #resolveSymlink( symlinkPath: string - ): Promise<{stat: nativeFs.Stats; resolvedPath: string;} | null> { - return new Promise((resolve) => { - if (!this.resolveSymlink) { - resolve(null); - return; + ): Promise<{ stat: nativeFs.Stats; resolvedPath: string } | null> { + const { fs, options } = this.state; + + if (!options.resolveSymlinks || options.excludeSymlinks) { + return null; + } + + try { + const resolvedPath = await promisify(fs.realpath)(symlinkPath); + const stat = await promisify(fs.stat)(resolvedPath); + + if ( + !stat.isDirectory() || + !resolveSymlink.isRecursive(symlinkPath, resolvedPath, this.state) + ) { + return { stat, resolvedPath }; + } + } catch (err) { + if (!options.suppressErrors) { + throw err; } + } - // WONT ACTUALLY RESOLVE! terrible promise - this.resolveSymlink(symlinkPath, this.state, (stat, resolvedPath) => { - resolve({stat, resolvedPath}); - }); - }); + return null; } - #walkDirectory( - crawlPath: string, - directoryPath: string, - depth: number, - ): Promise<{ - entries: Dirent[]; - directoryPath: string; - depth: number; - }> { - return new Promise<{ - entries: Dirent[]; - directoryPath: string; - depth: number; - }>((resolve) => { - this.walkDirectory( - this.state, - crawlPath, - directoryPath, - depth, - (entries, resultDirectoryPath, resultDepth) => - resolve({entries, directoryPath: resultDirectoryPath, depth: resultDepth}) - ); + async #walkDirectory(crawlPath: string, depth: number): Promise { + const { state } = this; + const { + fs, + options: { suppressErrors }, + } = state; + + if (depth < 0) { + return []; + } + + state.visited.push(crawlPath); + + try { + const entries = await promisify(fs.readdir)(crawlPath || ".", { + withFileTypes: true, }); + return entries; + } catch (err) { + if (suppressErrors) { + return []; + } + throw err; + } } async *start(): OutputIterator { + const { + counts, + options: { + resolveSymlinks, + excludeSymlinks, + exclude, + maxFiles, + useRealPaths, + pathSeparator, + }, + } = this.state; let pushedPath = await this.#pushDirectory(this.root); if (pushedPath !== null) { @@ -152,30 +159,21 @@ export class IteratorWalker { } const toWalk: Array<{ - crawlPath: string; + crawlPath: string; directoryPath: string; depth: number; }> = []; - let currentWalk: { - crawlPath: string; - directoryPath: string; - depth: number; - } | undefined = { + let currentWalk: + | { + crawlPath: string; + directoryPath: string; + depth: number; + } + | undefined = { crawlPath: this.root, directoryPath: this.root, depth: this.state.options.maxDepth, }; - const { - counts, - options: { - resolveSymlinks, - excludeSymlinks, - exclude, - maxFiles, - useRealPaths, - pathSeparator, - }, - } = this.state; while (currentWalk) { if (this.aborted || this.#complete) { @@ -187,11 +185,10 @@ export class IteratorWalker { const results = await this.#walkDirectory( currentWalk.crawlPath, - currentWalk.directoryPath, currentWalk.depth ); - for (const entry of results.entries) { + for (const entry of results) { if (maxFiles && counts.directories + counts.files >= maxFiles) { break; } @@ -204,7 +201,7 @@ export class IteratorWalker { entry.isFile() || (entry.isSymbolicLink() && !resolveSymlinks && !excludeSymlinks) ) { - const filename = this.joinPath(entry.name, results.directoryPath); + const filename = this.joinPath(entry.name, currentWalk.directoryPath); pushedPath = await this.#pushFile(filename); if (pushedPath !== null) { yield pushedPath; @@ -212,7 +209,7 @@ export class IteratorWalker { } else if (entry.isDirectory()) { let path = joinPath.joinDirectoryPath( entry.name, - results.directoryPath, + currentWalk.directoryPath, this.state.options.pathSeparator ); if (exclude && exclude(entry.name, path)) continue; @@ -223,20 +220,24 @@ export class IteratorWalker { toWalk.push({ directoryPath: path, crawlPath: path, - depth: results.depth - 1 + depth: currentWalk.depth - 1, }); } else if (this.resolveSymlink && entry.isSymbolicLink()) { - let path = joinPath.joinPathWithBasePath(entry.name, results.directoryPath); - const resolvedSymlink = await this.#resolveSymlink( - path + let path = joinPath.joinPathWithBasePath( + entry.name, + currentWalk.directoryPath ); + const resolvedSymlink = await this.#resolveSymlink(path); if (resolvedSymlink === null) { continue; } if (resolvedSymlink.stat.isDirectory()) { - const normalized = normalizePath(resolvedSymlink.resolvedPath, this.state.options); + const normalized = normalizePath( + resolvedSymlink.resolvedPath, + this.state.options + ); if ( exclude && @@ -251,10 +252,12 @@ export class IteratorWalker { toWalk.push({ crawlPath: normalized, directoryPath: useRealPaths ? normalized : path + pathSeparator, - depth: results.depth - 1 + depth: currentWalk.depth - 1, }); } else { - const normalized = useRealPaths ? resolvedSymlink.resolvedPath : path; + const normalized = useRealPaths + ? resolvedSymlink.resolvedPath + : path; const filename = basename(normalized); const directoryPath = normalizePath( dirname(normalized), diff --git a/src/api/iterator.ts b/src/api/iterator.ts index b662d234..2514d838 100644 --- a/src/api/iterator.ts +++ b/src/api/iterator.ts @@ -1,87 +1,6 @@ -import { Options, IterableOutput, OutputIterator } from "../types"; -import { Walker } from "./walker"; +import { Options, IterableOutput } from "../types"; import { IteratorWalker } from "./iterator-walker"; -class WalkerIterator { - #resolver?: () => void; - #walker: Walker; - #currentGroup?: string[]; - #queue: TOutput[number][] = []; - #error?: unknown; - - public constructor(root: string, options: Options) { - const pushPath = options.group ? this.#pushPath : this.#pushResult; - this.#walker = new Walker( - root, - options, - this.#onComplete, - pushPath, - this.#pushResult - ); - } - - #pushPath = (path: string, arr: string[]) => { - if (arr !== this.#currentGroup) { - this.#currentGroup = arr; - } - arr.push(path); - }; - - #pushResult = async (result: TOutput[number]) => { - this.#queue.push(result); - if (this.#resolver) { - const resolver = this.#resolver; - this.#resolver = undefined; - resolver(); - } - }; - - #onComplete = (err: unknown) => { - this.#currentGroup = undefined; - this.#complete = true; - if (err) { - this.#error = err; - } - if (this.#resolver) { - const resolver = this.#resolver; - this.#resolver = undefined; - resolver(); - } - }; - - async *start(): OutputIterator { - this.#walker.start(); - - try { - while (true) { - for (const item of this.#queue) { - if (this.#walker.aborted) { - break; - } - yield item; - } - this.#queue = []; - - if (this.#error) { - throw this.#error; - } - - if (this.#complete || this.#walker.aborted) { - return; - } - - await new Promise((resolve) => { - this.#resolver = resolve; - }); - } - } finally { - this.#walker.stop(); - } - } - - #complete: boolean = false; -} - export function iterator( root: string, options: Options From 73abe34bf7dad9602001482658ecbc5d040743e8 Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Sun, 13 Jul 2025 11:55:40 +0100 Subject: [PATCH 17/19] wip: inline the various push functions Seems a waste jumping through hoops with sync-async when we can just inline cleaner logic. --- src/api/iterator-walker.ts | 110 +++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 54 deletions(-) diff --git a/src/api/iterator-walker.ts b/src/api/iterator-walker.ts index c948bc90..d6f02b09 100644 --- a/src/api/iterator-walker.ts +++ b/src/api/iterator-walker.ts @@ -2,8 +2,6 @@ import { basename, dirname } from "path"; import { isRootDirectory, normalizePath } from "../utils"; import { WalkerState, Options, OutputIterator, IterableOutput } from "../types"; import * as joinPath from "./functions/join-path"; -import * as pushDirectory from "./functions/push-directory"; -import * as pushFile from "./functions/push-file"; import * as resolveSymlink from "./functions/resolve-symlink"; import { Queue } from "./queue"; import type { Dirent } from "fs"; @@ -16,10 +14,7 @@ export class IteratorWalker { private readonly root: string; private readonly state: WalkerState; private readonly joinPath: joinPath.JoinPathFunction; - private readonly pushDirectory: pushDirectory.PushDirectoryFunction; - private readonly pushFile: pushFile.PushFileFunction; private readonly resolveSymlink: resolveSymlink.ResolveSymlinkFunction | null; - #complete = false; constructor(root: string, options: Options) { this.root = normalizePath(root, options); @@ -37,8 +32,6 @@ export class IteratorWalker { }; this.joinPath = joinPath.build(this.root, options); - this.pushDirectory = pushDirectory.build(this.root, options); - this.pushFile = pushFile.build(options); this.resolveSymlink = resolveSymlink.build(options, false); } @@ -50,40 +43,44 @@ export class IteratorWalker { return controller.aborted || (signal !== undefined && signal.aborted); } - #pushDirectory(directoryPath: string): Promise { - return new Promise((resolve) => { - let pushed: string | null = null; - // this is synchronous. if we ever make pushDirectory async, - // rework everything! - this.pushDirectory( - directoryPath, - this.state.paths, - (pushedPath) => { - pushed = pushedPath; - }, - this.state.counts, - this.state.options.filters - ); - resolve(pushed); - }); + #shouldPushDirectory(directoryPath: string): boolean { + const { options } = this.state; + const { includeDirs, filters } = options; + + if (!includeDirs) { + return false; + } + + if (filters && filters.length) { + return filters.every((filter) => filter(directoryPath, true)); + } + + return true; } - #pushFile(filePath: string): Promise { - return new Promise((resolve) => { - let pushed: string | null = null; - // this is synchronous. if we ever make pushFile async, - // rework everything! - this.pushFile( - filePath, - this.state.paths, - (pushedPath) => { - pushed = pushedPath; - }, - this.state.counts, - this.state.options.filters - ); - resolve(pushed); - }); + #normalizeDirectoryPath(path: string): string { + const { options } = this.state; + const { relativePaths } = options; + + if (relativePaths) { + return path.substring(this.root.length) || "."; + } + return path || "."; + } + + #shouldPushFile(filePath: string): boolean { + const { options } = this.state; + const { excludeFiles, filters } = options; + + if (excludeFiles) { + return false; + } + + if (filters && filters.length) { + return filters.every((filter) => filter(filePath, false)); + } + + return true; } async #resolveSymlink( @@ -96,6 +93,9 @@ export class IteratorWalker { } try { + // TODO (43081j): probably just enforce the FSLike interface has a + // `promises` property, and use the normal async methods instead of + // promisifying const resolvedPath = await promisify(fs.realpath)(symlinkPath); const stat = await promisify(fs.stat)(resolvedPath); @@ -152,10 +152,12 @@ export class IteratorWalker { pathSeparator, }, } = this.state; - let pushedPath = await this.#pushDirectory(this.root); - if (pushedPath !== null) { - yield pushedPath; + const normalizedRoot = this.#normalizeDirectoryPath(this.root); + + if (this.#shouldPushDirectory(normalizedRoot)) { + counts.directories++; + yield normalizedRoot; } const toWalk: Array<{ @@ -176,7 +178,7 @@ export class IteratorWalker { }; while (currentWalk) { - if (this.aborted || this.#complete) { + if (this.aborted) { break; } if (maxFiles && counts.directories + counts.files > maxFiles) { @@ -202,9 +204,9 @@ export class IteratorWalker { (entry.isSymbolicLink() && !resolveSymlinks && !excludeSymlinks) ) { const filename = this.joinPath(entry.name, currentWalk.directoryPath); - pushedPath = await this.#pushFile(filename); - if (pushedPath !== null) { - yield pushedPath; + if (this.#shouldPushFile(filename)) { + counts.files++; + yield filename; } } else if (entry.isDirectory()) { let path = joinPath.joinDirectoryPath( @@ -213,9 +215,10 @@ export class IteratorWalker { this.state.options.pathSeparator ); if (exclude && exclude(entry.name, path)) continue; - pushedPath = await this.#pushDirectory(path); - if (pushedPath !== null) { - yield pushedPath; + const normalizedPath = this.#normalizeDirectoryPath(path); + if (this.#shouldPushDirectory(normalizedPath)) { + counts.directories++; + yield normalizedPath; } toWalk.push({ directoryPath: path, @@ -263,11 +266,10 @@ export class IteratorWalker { dirname(normalized), this.state.options ); - pushedPath = await this.#pushFile( - this.joinPath(filename, directoryPath) - ); - if (pushedPath !== null) { - yield pushedPath; + const fullPath = this.joinPath(filename, directoryPath); + if (this.#shouldPushFile(fullPath)) { + counts.files++; + yield fullPath; } } } From 89d4b6ae67571e9d47a2a52650fea7c5892e597e Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Sun, 13 Jul 2025 14:06:52 +0100 Subject: [PATCH 18/19] chore: use typed output --- src/api/iterator.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/iterator.ts b/src/api/iterator.ts index 2514d838..fe908d68 100644 --- a/src/api/iterator.ts +++ b/src/api/iterator.ts @@ -1,10 +1,10 @@ -import { Options, IterableOutput } from "../types"; +import { Options, IterableOutput, OutputIterator } from "../types"; import { IteratorWalker } from "./iterator-walker"; export function iterator( root: string, options: Options -): AsyncIterable { +): OutputIterator { const walker = new IteratorWalker(root, options); return walker.start(); } From de9055868153119c3728ef01f4b129ea134d5a37 Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Sun, 13 Jul 2025 14:10:46 +0100 Subject: [PATCH 19/19] chore: node 12 compat members --- src/api/iterator-walker.ts | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/api/iterator-walker.ts b/src/api/iterator-walker.ts index d6f02b09..5989e279 100644 --- a/src/api/iterator-walker.ts +++ b/src/api/iterator-walker.ts @@ -14,7 +14,6 @@ export class IteratorWalker { private readonly root: string; private readonly state: WalkerState; private readonly joinPath: joinPath.JoinPathFunction; - private readonly resolveSymlink: resolveSymlink.ResolveSymlinkFunction | null; constructor(root: string, options: Options) { this.root = normalizePath(root, options); @@ -32,7 +31,6 @@ export class IteratorWalker { }; this.joinPath = joinPath.build(this.root, options); - this.resolveSymlink = resolveSymlink.build(options, false); } get aborted(): boolean { @@ -43,7 +41,7 @@ export class IteratorWalker { return controller.aborted || (signal !== undefined && signal.aborted); } - #shouldPushDirectory(directoryPath: string): boolean { + private shouldPushDirectory(directoryPath: string): boolean { const { options } = this.state; const { includeDirs, filters } = options; @@ -58,7 +56,7 @@ export class IteratorWalker { return true; } - #normalizeDirectoryPath(path: string): string { + private normalizeDirectoryPath(path: string): string { const { options } = this.state; const { relativePaths } = options; @@ -68,7 +66,7 @@ export class IteratorWalker { return path || "."; } - #shouldPushFile(filePath: string): boolean { + private shouldPushFile(filePath: string): boolean { const { options } = this.state; const { excludeFiles, filters } = options; @@ -83,7 +81,7 @@ export class IteratorWalker { return true; } - async #resolveSymlink( + private async resolveSymlink( symlinkPath: string ): Promise<{ stat: nativeFs.Stats; resolvedPath: string } | null> { const { fs, options } = this.state; @@ -114,7 +112,7 @@ export class IteratorWalker { return null; } - async #walkDirectory(crawlPath: string, depth: number): Promise { + async walkDirectory(crawlPath: string, depth: number): Promise { const { state } = this; const { fs, @@ -153,9 +151,9 @@ export class IteratorWalker { }, } = this.state; - const normalizedRoot = this.#normalizeDirectoryPath(this.root); + const normalizedRoot = this.normalizeDirectoryPath(this.root); - if (this.#shouldPushDirectory(normalizedRoot)) { + if (this.shouldPushDirectory(normalizedRoot)) { counts.directories++; yield normalizedRoot; } @@ -185,7 +183,7 @@ export class IteratorWalker { break; } - const results = await this.#walkDirectory( + const results = await this.walkDirectory( currentWalk.crawlPath, currentWalk.depth ); @@ -204,7 +202,7 @@ export class IteratorWalker { (entry.isSymbolicLink() && !resolveSymlinks && !excludeSymlinks) ) { const filename = this.joinPath(entry.name, currentWalk.directoryPath); - if (this.#shouldPushFile(filename)) { + if (this.shouldPushFile(filename)) { counts.files++; yield filename; } @@ -215,8 +213,8 @@ export class IteratorWalker { this.state.options.pathSeparator ); if (exclude && exclude(entry.name, path)) continue; - const normalizedPath = this.#normalizeDirectoryPath(path); - if (this.#shouldPushDirectory(normalizedPath)) { + const normalizedPath = this.normalizeDirectoryPath(path); + if (this.shouldPushDirectory(normalizedPath)) { counts.directories++; yield normalizedPath; } @@ -225,12 +223,12 @@ export class IteratorWalker { crawlPath: path, depth: currentWalk.depth - 1, }); - } else if (this.resolveSymlink && entry.isSymbolicLink()) { + } else if (entry.isSymbolicLink()) { let path = joinPath.joinPathWithBasePath( entry.name, currentWalk.directoryPath ); - const resolvedSymlink = await this.#resolveSymlink(path); + const resolvedSymlink = await this.resolveSymlink(path); if (resolvedSymlink === null) { continue; @@ -267,7 +265,7 @@ export class IteratorWalker { this.state.options ); const fullPath = this.joinPath(filename, directoryPath); - if (this.#shouldPushFile(fullPath)) { + if (this.shouldPushFile(fullPath)) { counts.files++; yield fullPath; }