diff --git a/__tests__/fdir.test.ts b/__tests__/fdir.test.ts index 5915ad9..def6098 100644 --- a/__tests__/fdir.test.ts +++ b/__tests__/fdir.test.ts @@ -1,11 +1,14 @@ import { fdir } from "../src/index"; import fs from "fs"; import mock from "mock-fs"; -import { test, beforeEach, TestContext, vi } from "vitest"; +import { test, beforeEach, vi } from "vitest"; import path, { sep } from "path"; import { convertSlashes } from "../src/utils"; import picomatch from "picomatch"; -import { apiTypes, APITypes, cwd, restricted, root } from "./utils"; +import { apiTypes, APITypes, cwd, restricted, root, execute } from "./utils"; + +// AbortController is not present on Node v14 +const hasAbortController = "AbortController" in globalThis; beforeEach(() => { mock.restore(); @@ -25,24 +28,22 @@ test(`crawl single depth directory with callback`, (t) => { }); }); -async function crawl(type: APITypes, path: string, t: TestContext) { +async function crawl(type: APITypes, path: string) { const api = new fdir().crawl(path); - const files = await api[type](); - if (!files) throw new Error("files cannot be null."); - t.expect(files[0]).toBeDefined(); - t.expect(files.every((t) => t)).toBeTruthy(); - t.expect(files[0].length).toBeGreaterThan(0); - return files; + return execute(api, type); } for (const type of apiTypes) { test(`[${type}] crawl directory`, async (t) => { - await crawl(type, "__tests__", t); + const files = await crawl(type, "__tests__"); + t.expect(files[0]).toBeDefined(); + t.expect(files.every((t) => t)).toBeTruthy(); + t.expect(files[0].length).toBeGreaterThan(0); }); test(`[${type}] crawl directory with options`, async (t) => { const api = new fdir({ includeBasePath: true }).crawl("__tests__"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.every((file) => file.startsWith("__tests__"))).toBeTruthy(); }); @@ -51,7 +52,7 @@ for (const type of apiTypes) { maxDepth: 0, includeBasePath: true, }).crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files).not.toHaveLength(0); t.expect(files.every((file) => file.split(path.sep).length === 2)).toBe( true @@ -63,7 +64,7 @@ for (const type of apiTypes) { maxDepth: 1, includeBasePath: true, }).crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.some((file) => file.split(path.sep).length === 3)).toBe( true ); @@ -73,18 +74,21 @@ for (const type of apiTypes) { }); test(`[${type}] crawl multi depth directory`, async (t) => { - await crawl(type, "node_modules", t); + const files = await crawl(type, "node_modules"); + t.expect(files[0]).toBeDefined(); + t.expect(files.every((t) => t)).toBeTruthy(); + t.expect(files[0].length).toBeGreaterThan(0); }); test(`[${type}] crawl directory & limit files to 10`, async (t) => { const api = new fdir().withMaxFiles(10).crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files).toHaveLength(10); }); test(`[${type}] crawl and get both files and directories (withDirs)`, async (t) => { const api = new fdir().withDirs().crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files[0]).toBeDefined(); t.expect(files.every((t) => t)).toBeTruthy(); t.expect(files[0].length).toBeGreaterThan(0); @@ -93,7 +97,7 @@ for (const type of apiTypes) { test(`[${type}] crawl and get all files (withMaxDepth = 1)`, async (t) => { const api = new fdir().withMaxDepth(1).withBasePath().crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect( files.every((file) => file.split(path.sep).length <= 3) ).toBeTruthy(); @@ -104,7 +108,7 @@ for (const type of apiTypes) { .withMaxDepth(-1) .withBasePath() .crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.length).toBe(0); }); @@ -114,7 +118,7 @@ for (const type of apiTypes) { .glob("**/*.js") .glob("**/*.js") .crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.every((file) => file.endsWith(".js"))).toBeTruthy(); }); @@ -123,7 +127,7 @@ for (const type of apiTypes) { .withBasePath() .exclude((dir) => dir.includes("node_modules")) .crawl(cwd()); - const files = await api[type](); + const files = await execute(api, type); t.expect( files.every((file) => !file.includes("node_modules")) ).toBeTruthy(); @@ -134,7 +138,7 @@ for (const type of apiTypes) { .withBasePath() .filter((file) => file.includes(".git")) .crawl(cwd()); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.every((file) => file.includes(".git"))).toBeTruthy(); }); @@ -144,7 +148,7 @@ for (const type of apiTypes) { .filter((file) => file.includes(".git")) .filter((file) => file.includes(".js")) .crawl(cwd()); - const files = await api[type](); + const files = await execute(api, type); t.expect( files.every((file) => file.includes(".git") || file.includes(".js")) ).toBeTruthy(); @@ -154,7 +158,7 @@ for (const type of apiTypes) { const api = new fdir() .withBasePath() .crawl(path.join(cwd(), "node_modules")); - const files = await api[type](); + const files = await execute(api, type); t.expect( files.every((file) => file.startsWith("node_modules")) ).toBeTruthy(); @@ -162,18 +166,18 @@ for (const type of apiTypes) { test(`[${type}] get all files in a directory and output full paths (withFullPaths)`, async (t) => { const api = new fdir().withFullPaths().crawl(cwd()); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.every((file) => file.startsWith(root()))).toBeTruthy(); }); test(`[${type}] getting files from restricted directory should throw`, async (t) => { const api = new fdir().withErrors().crawl(restricted()); - t.expect(async () => await api[type]()).rejects.toThrowError(); + t.expect(async () => await execute(api, type)).rejects.toThrowError(); }); test(`[${type}] getting files from restricted directory shouldn't throw (suppressErrors)`, async (t) => { const api = new fdir().crawl(restricted()); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.length).toBeGreaterThanOrEqual(0); }); @@ -184,20 +188,22 @@ for (const type of apiTypes) { }, }); const api = new fdir().withBasePath().normalize().crawl("/"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.every((file) => !file.includes("//"))).toBeTruthy(); mock.restore(); }); - test(`[${type}] crawl all files with only counts`, async (t) => { - const api = new fdir().onlyCounts().crawl("node_modules"); - const result = await api[type](); - t.expect(result.files).toBeGreaterThan(0); - }); + if (type !== "withIterator") { + test(`[${type}] crawl all files with only counts`, async (t) => { + const api = new fdir().onlyCounts().crawl("node_modules"); + const result = await api[type](); + t.expect(result.files).toBeGreaterThan(0); + }); + } test(`[${type}] crawl and return only directories`, async (t) => { const api = new fdir().onlyDirs().crawl("node_modules"); - const result = await api[type](); + const result = await execute(api, type); t.expect(result.length).toBeGreaterThan(0); t.expect( result.every((dir) => { @@ -211,7 +217,7 @@ for (const type of apiTypes) { excludeFiles: true, includeDirs: true, }).crawl("node_modules"); - const result = await api[type](); + const result = await execute(api, type); t.expect(result.length).toBeGreaterThan(0); t.expect( result.every((dir) => { @@ -220,26 +226,28 @@ for (const type of apiTypes) { ).toBeTruthy(); }); - test(`[${type}] crawl and filter all files and get only counts`, async (t) => { - const api = new fdir() - .withBasePath() - .filter((file) => file.includes("node_modules")) - .onlyCounts() - .crawl(cwd()); - const result = await api[type](); - t.expect(result.files).toBeGreaterThan(0); - }); + if (type !== "withIterator") { + test(`[${type}] crawl and filter all files and get only counts`, async (t) => { + const api = new fdir() + .withBasePath() + .filter((file) => file.includes("node_modules")) + .onlyCounts() + .crawl(cwd()); + const result = await api[type](); + t.expect(result.files).toBeGreaterThan(0); + }); + } test("crawl all files in a directory (path with trailing slash)", async (t) => { const api = new fdir().normalize().crawl("node_modules/"); - const files = await api[type](); + const files = await execute(api, type); const res = files.every((file) => !file.includes("/")); t.expect(res).toBeDefined(); }); test(`[${type}] crawl all files and group them by directory`, async (t) => { const api = new fdir().withBasePath().group().crawl("node_modules"); - const result = await api[type](); + const result = await execute(api, type); t.expect(result.length).toBeGreaterThan(0); }); @@ -248,7 +256,7 @@ for (const type of apiTypes) { .onlyDirs() .filter((path) => path.includes("api")) .crawl("./src"); - const result = await api[type](); + const result = await execute(api, type); t.expect(result).toHaveLength(2); }); @@ -256,7 +264,7 @@ for (const type of apiTypes) { const api = new fdir() .withRelativePaths() .crawl(path.normalize(`node_modules/`)); - const paths = await api[type](); + const paths = await execute(api, type); t.expect(paths.every((p) => !p.startsWith("node_modules"))).toBeTruthy(); }); @@ -277,7 +285,7 @@ for (const type of apiTypes) { .withDirs() .withRelativePaths() .crawl("/some"); - const paths = await api[type](); + const paths = await execute(api, type); t.expect(paths.length).toBe(5); t.expect(paths.filter((p) => p === ".").length).toBe(1); @@ -303,7 +311,7 @@ for (const type of apiTypes) { .withRelativePaths() .filter((p) => p !== path.join("dir", "dir1/")) .crawl("/some"); - const paths = await api[type](); + const paths = await execute(api, type); t.expect(paths.length).toBe(4); t.expect(paths.includes(path.join("dir", "dir1/"))).toBe(false); @@ -314,7 +322,7 @@ for (const type of apiTypes) { test(`[${type}] crawl and return relative paths that end with /`, async (t) => { const api = new fdir().withRelativePaths().crawl("./node_modules/"); - const paths = await api[type](); + const paths = await execute(api, type); t.expect( paths.every((p) => !p.startsWith("node_modules") && !p.includes("//")) ).toBeTruthy(); @@ -324,7 +332,7 @@ for (const type of apiTypes) { const api = new fdir() .withPathSeparator(sep === "/" ? "\\" : "/") .crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.every((f) => !f.includes(sep))).toBeTruthy(); }); @@ -337,7 +345,7 @@ for (const type of apiTypes) { .withBasePath() .glob("**/*.js") .crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(globFunction).toHaveBeenCalled(); t.expect(files.every((file) => file.endsWith(".js"))).toBeTruthy(); }); @@ -352,7 +360,7 @@ for (const type of apiTypes) { .withBasePath() .globWithOptions(["**/*.js"], { foo: 5 }) .crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(globFunction).toHaveBeenCalled(); t.expect(files.every((file) => file.endsWith(".js"))).toBeTruthy(); }); @@ -363,7 +371,7 @@ for (const type of apiTypes) { .withBasePath() .glob("**/*.js") .crawl("node_modules"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.every((file) => file.endsWith(".js"))).toBeTruthy(); }); @@ -396,8 +404,8 @@ for (const type of apiTypes) { const api = new fdir({ fs: fakeFs, }).crawl("node_modules"); - await api[type](); - if (type === "withPromise") { + await execute(api, type); + if (type === "withPromise" || type === "withIterator") { t.expect(readdirStub).toHaveBeenCalled(); } else { t.expect(readdirSyncStub).toHaveBeenCalled(); @@ -405,9 +413,9 @@ for (const type of apiTypes) { }); } -// AbortController is not present on Node v14 -if ("AbortController" in globalThis) { - test(`[async] crawl directory & use abort signal to abort`, async (t) => { +test.runIf(hasAbortController)( + `[async] crawl directory & use abort signal to abort`, + async (t) => { const totalFiles = new fdir().onlyCounts().crawl("node_modules").sync(); const abortController = new AbortController(); const api = new fdir() @@ -419,8 +427,8 @@ if ("AbortController" in globalThis) { .crawl("node_modules"); const files = await api.withPromise(); t.expect(files.length).toBeLessThan(totalFiles.files); - }); -} + } +); test(`paths should never start with ./`, async (t) => { const apis = [ @@ -486,3 +494,32 @@ test(`do not convert \\\\ to \\`, async (t) => { "\\\\wsl.localhost\\Ubuntu\\home\\" ); }); + +test("interrupted iterator should stop yielding results", async (t) => { + const api = new fdir().crawl("./src"); + const iterator = api.withIterator(); + const results: string[] = []; + let next = await iterator.next(); + do { + if (!next.done) { + results.push(next.value); + } + iterator.return(); + } while (next.done !== false); + t.expect(results.length).toBe(1); +}); + +test.runIf(hasAbortController)( + "aborted iterator should stop yielding results", + async (t) => { + const aborter = new AbortController(); + const api = new fdir().withAbortSignal(aborter.signal).crawl("./src"); + const iterator = api.withIterator(); + const results: string[] = []; + for await (const value of iterator) { + results.push(value); + aborter.abort(); + } + t.expect(results.length).toBe(1); + } +); diff --git a/__tests__/symlinks.test.ts b/__tests__/symlinks.test.ts index 0a61ec2..3a31161 100644 --- a/__tests__/symlinks.test.ts +++ b/__tests__/symlinks.test.ts @@ -1,7 +1,7 @@ -import { afterAll, beforeAll, beforeEach, describe, test } from "vitest"; -import { apiTypes, normalize, root } from "./utils"; +import { afterAll, beforeAll, describe, test } from "vitest"; +import { apiTypes, normalize, root, execute } from "./utils"; import mock from "mock-fs"; -import { fdir, Options } from "../src"; +import { fdir } from "../src"; import path from "path"; const fsWithRelativeSymlinks = { @@ -154,7 +154,7 @@ for (const type of apiTypes) { test(`resolve symlinks`, async (t) => { const api = new fdir().withSymlinks().crawl("/some/dir"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize([ "/other/dir/file-2", @@ -166,7 +166,7 @@ for (const type of apiTypes) { test(`resolve recursive symlinks`, async (t) => { const api = new fdir().withSymlinks().crawl("/recursive"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize([ "/double/recursive/another-file", @@ -184,7 +184,7 @@ for (const type of apiTypes) { const api = new fdir() .withSymlinks({ resolvePaths: false }) .crawl("/recursive"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize([ "/recursive/dir/not-recursive/another-file", @@ -234,7 +234,7 @@ for (const type of apiTypes) { .withRelativePaths() .withErrors() .crawl("./recursive"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize([ "dir/not-recursive/another-file", @@ -284,7 +284,7 @@ for (const type of apiTypes) { .withRelativePaths() .withErrors() .crawl("./recursive"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize([ "..//double/recursive/another-file", @@ -302,7 +302,7 @@ for (const type of apiTypes) { const api = new fdir() .withSymlinks({ resolvePaths: false }) .crawl("/some/dir"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize([ "/some/dir/dirSymlink/file-1", @@ -317,7 +317,7 @@ for (const type of apiTypes) { .withSymlinks({ resolvePaths: false }) .withRelativePaths() .crawl("/some/dir"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize([ "dirSymlink/file-1", @@ -332,7 +332,7 @@ for (const type of apiTypes) { .withSymlinks() .withRelativePaths() .crawl("./relative/dir"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize([ "../../../../other-relative/dir/file-2", @@ -347,7 +347,7 @@ for (const type of apiTypes) { .withSymlinks() .exclude((_name, path) => path === resolveSymlinkRoot("/sym/linked/")) .crawl("/some/dir"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual(normalize(["/other/dir/file-2"])); }); @@ -358,7 +358,7 @@ for (const type of apiTypes) { (_name, path) => path === resolveSymlinkRoot("/some/dir/dirSymlink/") ) .crawl("/some/dir"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize(["/some/dir/fileSymlink"]) ); @@ -366,7 +366,7 @@ for (const type of apiTypes) { test(`do not resolve symlinks`, async (t) => { const api = new fdir().crawl("/some/dir"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files.sort()).toStrictEqual( normalize(["dirSymlink", "fileSymlink", "fileSymlink2"]) ); @@ -374,7 +374,7 @@ for (const type of apiTypes) { test(`exclude symlinks`, async (t) => { const api = new fdir({ excludeSymlinks: true }).crawl("/some/dir"); - const files = await api[type](); + const files = await execute(api, type); t.expect(files).toHaveLength(0); }); @@ -382,7 +382,7 @@ for (const type of apiTypes) { "doesn't hang when resolving symlinks in the root directory", async (t) => { const api = new fdir().withSymlinks({ resolvePaths: false }).crawl("/"); - const files = await api[type](); + const files = await execute(api, type); const expectedFiles = normalize(["/lib/file-1", "/usr/lib/file-1"]); for (const expectedFile of expectedFiles) { t.expect(files).toContain(expectedFile); diff --git a/__tests__/utils.ts b/__tests__/utils.ts index f3c7cb0..b7db488 100644 --- a/__tests__/utils.ts +++ b/__tests__/utils.ts @@ -1,7 +1,9 @@ import path from "path"; +import type { APIBuilder } from "../src/builder/api-builder"; +import type { IterableOutput } from "../src/types"; export type APITypes = (typeof apiTypes)[number]; -export const apiTypes = ["withPromise", "sync"] as const; +export const apiTypes = ["withPromise", "sync", "withIterator"] as const; export function root() { return process.platform === "win32" ? process.cwd().split(path.sep)[0] : "/"; @@ -22,3 +24,19 @@ export function normalize(paths: string[]) { path.isAbsolute(p) ? path.resolve(p) : path.normalize(p) ); } + +export async function execute( + api: APIBuilder, + type: APITypes +): Promise { + let files: T[number][] = []; + + if (type === "withIterator") { + for await (const file of api[type]()) { + files.push(file); + } + } else { + files = await api[type](); + } + return files as T; +} diff --git a/src/api/functions/group-files.ts b/src/api/functions/group-files.ts index f53a04b..7a89e2a 100644 --- a/src/api/functions/group-files.ts +++ b/src/api/functions/group-files.ts @@ -3,15 +3,17 @@ import { Group, Options } from "../../types"; export type GroupFilesFunction = ( groups: Group[], directory: string, - files: string[] + files: string[], + pushGroup: (group: Group, arr: Group[]) => void ) => void; const groupFiles: GroupFilesFunction = ( groups: Group[], directory: string, - files: string[] + files: string[], + pushGroup ) => { - groups.push({ directory, files, dir: directory }); + pushGroup({ directory, files, dir: directory }, groups); }; const empty: GroupFilesFunction = () => {}; diff --git a/src/api/functions/push-directory.ts b/src/api/functions/push-directory.ts index 2de954b..e272079 100644 --- a/src/api/functions/push-directory.ts +++ b/src/api/functions/push-directory.ts @@ -1,40 +1,53 @@ -import { FilterPredicate, Options } from "../../types"; +import { FilterPredicate, Options, Counts } from "../../types"; export type PushDirectoryFunction = ( directoryPath: string, paths: string[], + pushPath: (path: string, arr: string[]) => void, + counts: Counts, filters?: FilterPredicate[] ) => void; function pushDirectoryWithRelativePath(root: string): PushDirectoryFunction { - return function (directoryPath, paths) { - paths.push(directoryPath.substring(root.length) || "."); + return function (directoryPath, paths, pushPath, counts) { + pushPath(directoryPath.substring(root.length) || ".", paths); + counts.directories++; }; } function pushDirectoryFilterWithRelativePath( root: string ): PushDirectoryFunction { - return function (directoryPath, paths, filters) { + return function (directoryPath, paths, pushPath, counts, filters) { const relativePath = directoryPath.substring(root.length) || "."; if (filters!.every((filter) => filter(relativePath, true))) { - paths.push(relativePath); + pushPath(relativePath, paths); + counts.directories++; } }; } -const pushDirectory: PushDirectoryFunction = (directoryPath, paths) => { - paths.push(directoryPath || "."); +const pushDirectory: PushDirectoryFunction = ( + directoryPath, + paths, + pushPath, + counts +) => { + pushPath(directoryPath || ".", paths); + counts.directories++; }; const pushDirectoryFilter: PushDirectoryFunction = ( directoryPath, paths, + pushPath, + counts, filters ) => { const path = directoryPath || "."; if (filters!.every((filter) => filter(path, true))) { - paths.push(path); + pushPath(path, paths); + counts.directories++; } }; diff --git a/src/api/functions/push-file.ts b/src/api/functions/push-file.ts index 18ef77d..26a9205 100644 --- a/src/api/functions/push-file.ts +++ b/src/api/functions/push-file.ts @@ -3,6 +3,7 @@ import { FilterPredicate, Options, Counts } from "../../types"; export type PushFileFunction = ( directoryPath: string, paths: string[], + pushPath: (path: string, arr: string[]) => void, counts: Counts, filters?: FilterPredicate[] ) => void; @@ -10,6 +11,7 @@ export type PushFileFunction = ( const pushFileFilterAndCount: PushFileFunction = ( filename, _paths, + _pushPath, counts, filters ) => { @@ -19,23 +21,29 @@ const pushFileFilterAndCount: PushFileFunction = ( const pushFileFilter: PushFileFunction = ( filename, paths, - _counts, + pushPath, + counts, filters ) => { - if (filters!.every((filter) => filter(filename, false))) paths.push(filename); + if (filters!.every((filter) => filter(filename, false))) { + pushPath(filename, paths); + counts.files++; + } }; const pushFileCount: PushFileFunction = ( _filename, _paths, + _pushPath, counts, _filters ) => { counts.files++; }; -const pushFile: PushFileFunction = (filename, paths) => { - paths.push(filename); +const pushFile: PushFileFunction = (filename, paths, pushPath, counts) => { + pushPath(filename, paths); + counts.files++; }; const empty: PushFileFunction = () => {}; diff --git a/src/api/functions/resolve-symlink.ts b/src/api/functions/resolve-symlink.ts index b23cdb2..b572287 100644 --- a/src/api/functions/resolve-symlink.ts +++ b/src/api/functions/resolve-symlink.ts @@ -68,7 +68,11 @@ export function build( return isSynchronous ? resolveSymlinks : resolveSymlinksAsync; } -function isRecursive(path: string, resolved: string, state: WalkerState) { +export function isRecursive( + path: string, + resolved: string, + state: WalkerState +) { if (state.options.useRealPaths) return isRecursiveUsingRealPaths(resolved, state); diff --git a/src/api/functions/walk-directory.ts b/src/api/functions/walk-directory.ts index f2b9d36..a5c5f8f 100644 --- a/src/api/functions/walk-directory.ts +++ b/src/api/functions/walk-directory.ts @@ -25,7 +25,6 @@ const walkAsync: WalkDirectoryFunction = ( const { fs } = state; state.visited.push(crawlPath); - state.counts.directories++; // Perf: Node >= 10 introduced withFileTypes that helps us // skip an extra fs.stat call. @@ -46,7 +45,6 @@ const walkSync: WalkDirectoryFunction = ( const { fs } = state; if (currentDepth < 0) return; state.visited.push(crawlPath); - state.counts.directories++; let entries: Dirent[] = []; try { diff --git a/src/api/iterator-walker.ts b/src/api/iterator-walker.ts new file mode 100644 index 0000000..5989e27 --- /dev/null +++ b/src/api/iterator-walker.ts @@ -0,0 +1,283 @@ +import { basename, dirname } from "path"; +import { isRootDirectory, normalizePath } from "../utils"; +import { WalkerState, Options, OutputIterator, IterableOutput } from "../types"; +import * as joinPath from "./functions/join-path"; +import * as resolveSymlink from "./functions/resolve-symlink"; +import { Queue } from "./queue"; +import type { Dirent } from "fs"; +import * as nativeFs from "fs"; +import { Counter } from "./counter"; +import { Aborter } from "./aborter"; +import { promisify } from "node:util"; + +export class IteratorWalker { + private readonly root: string; + private readonly state: WalkerState; + private readonly joinPath: joinPath.JoinPathFunction; + + constructor(root: string, options: Options) { + this.root = normalizePath(root, options); + this.state = { + root: isRootDirectory(this.root) ? this.root : this.root.slice(0, -1), + paths: [], + groups: [], + counts: new Counter(), + options, + queue: new Queue(() => {}), + symlinks: new Map(), + visited: [""].slice(0, 0), + controller: new Aborter(), + fs: options.fs || nativeFs, + }; + + this.joinPath = joinPath.build(this.root, options); + } + + get aborted(): boolean { + const { + controller, + options: { signal }, + } = this.state; + return controller.aborted || (signal !== undefined && signal.aborted); + } + + private shouldPushDirectory(directoryPath: string): boolean { + const { options } = this.state; + const { includeDirs, filters } = options; + + if (!includeDirs) { + return false; + } + + if (filters && filters.length) { + return filters.every((filter) => filter(directoryPath, true)); + } + + return true; + } + + private normalizeDirectoryPath(path: string): string { + const { options } = this.state; + const { relativePaths } = options; + + if (relativePaths) { + return path.substring(this.root.length) || "."; + } + return path || "."; + } + + private shouldPushFile(filePath: string): boolean { + const { options } = this.state; + const { excludeFiles, filters } = options; + + if (excludeFiles) { + return false; + } + + if (filters && filters.length) { + return filters.every((filter) => filter(filePath, false)); + } + + return true; + } + + private async resolveSymlink( + symlinkPath: string + ): Promise<{ stat: nativeFs.Stats; resolvedPath: string } | null> { + const { fs, options } = this.state; + + if (!options.resolveSymlinks || options.excludeSymlinks) { + return null; + } + + try { + // TODO (43081j): probably just enforce the FSLike interface has a + // `promises` property, and use the normal async methods instead of + // promisifying + const resolvedPath = await promisify(fs.realpath)(symlinkPath); + const stat = await promisify(fs.stat)(resolvedPath); + + if ( + !stat.isDirectory() || + !resolveSymlink.isRecursive(symlinkPath, resolvedPath, this.state) + ) { + return { stat, resolvedPath }; + } + } catch (err) { + if (!options.suppressErrors) { + throw err; + } + } + + return null; + } + + async walkDirectory(crawlPath: string, depth: number): Promise { + const { state } = this; + const { + fs, + options: { suppressErrors }, + } = state; + + if (depth < 0) { + return []; + } + + state.visited.push(crawlPath); + + try { + const entries = await promisify(fs.readdir)(crawlPath || ".", { + withFileTypes: true, + }); + return entries; + } catch (err) { + if (suppressErrors) { + return []; + } + throw err; + } + } + + async *start(): OutputIterator { + const { + counts, + options: { + resolveSymlinks, + excludeSymlinks, + exclude, + maxFiles, + useRealPaths, + pathSeparator, + }, + } = this.state; + + const normalizedRoot = this.normalizeDirectoryPath(this.root); + + if (this.shouldPushDirectory(normalizedRoot)) { + counts.directories++; + yield normalizedRoot; + } + + const toWalk: Array<{ + crawlPath: string; + directoryPath: string; + depth: number; + }> = []; + let currentWalk: + | { + crawlPath: string; + directoryPath: string; + depth: number; + } + | undefined = { + crawlPath: this.root, + directoryPath: this.root, + depth: this.state.options.maxDepth, + }; + + while (currentWalk) { + if (this.aborted) { + break; + } + if (maxFiles && counts.directories + counts.files > maxFiles) { + break; + } + + const results = await this.walkDirectory( + currentWalk.crawlPath, + currentWalk.depth + ); + + for (const entry of results) { + if (maxFiles && counts.directories + counts.files >= maxFiles) { + break; + } + + if (this.aborted) { + break; + } + + if ( + entry.isFile() || + (entry.isSymbolicLink() && !resolveSymlinks && !excludeSymlinks) + ) { + const filename = this.joinPath(entry.name, currentWalk.directoryPath); + if (this.shouldPushFile(filename)) { + counts.files++; + yield filename; + } + } else if (entry.isDirectory()) { + let path = joinPath.joinDirectoryPath( + entry.name, + currentWalk.directoryPath, + this.state.options.pathSeparator + ); + if (exclude && exclude(entry.name, path)) continue; + const normalizedPath = this.normalizeDirectoryPath(path); + if (this.shouldPushDirectory(normalizedPath)) { + counts.directories++; + yield normalizedPath; + } + toWalk.push({ + directoryPath: path, + crawlPath: path, + depth: currentWalk.depth - 1, + }); + } else if (entry.isSymbolicLink()) { + let path = joinPath.joinPathWithBasePath( + entry.name, + currentWalk.directoryPath + ); + const resolvedSymlink = await this.resolveSymlink(path); + + if (resolvedSymlink === null) { + continue; + } + + if (resolvedSymlink.stat.isDirectory()) { + const normalized = normalizePath( + resolvedSymlink.resolvedPath, + this.state.options + ); + + if ( + exclude && + exclude( + entry.name, + useRealPaths ? normalized : path + pathSeparator + ) + ) { + continue; + } + + toWalk.push({ + crawlPath: normalized, + directoryPath: useRealPaths ? normalized : path + pathSeparator, + depth: currentWalk.depth - 1, + }); + } else { + const normalized = useRealPaths + ? resolvedSymlink.resolvedPath + : path; + const filename = basename(normalized); + const directoryPath = normalizePath( + dirname(normalized), + this.state.options + ); + const fullPath = this.joinPath(filename, directoryPath); + if (this.shouldPushFile(fullPath)) { + counts.files++; + yield fullPath; + } + } + } + } + + currentWalk = toWalk.pop(); + } + } + + stop(): void { + this.state.controller.abort(); + } +} diff --git a/src/api/iterator.ts b/src/api/iterator.ts new file mode 100644 index 0000000..fe908d6 --- /dev/null +++ b/src/api/iterator.ts @@ -0,0 +1,10 @@ +import { Options, IterableOutput, OutputIterator } from "../types"; +import { IteratorWalker } from "./iterator-walker"; + +export function iterator( + root: string, + options: Options +): OutputIterator { + const walker = new IteratorWalker(root, options); + return walker.start(); +} diff --git a/src/api/walker.ts b/src/api/walker.ts index e6edc0d..6f68c21 100644 --- a/src/api/walker.ts +++ b/src/api/walker.ts @@ -1,6 +1,6 @@ import { basename, dirname } from "path"; import { isRootDirectory, normalizePath } from "../utils"; -import { ResultCallback, WalkerState, Options } from "../types"; +import { ResultCallback, WalkerState, Options, Group } from "../types"; import * as joinPath from "./functions/join-path"; import * as pushDirectory from "./functions/push-directory"; import * as pushFile from "./functions/push-file"; @@ -28,11 +28,15 @@ export class Walker { private readonly resolveSymlink: resolveSymlink.ResolveSymlinkFunction | null; private readonly walkDirectory: walkDirectory.WalkDirectoryFunction; private readonly callbackInvoker: invokeCallback.InvokeCallbackFunction; + private readonly pushPath: (path: string, arr: string[]) => void; + private readonly pushGroup: (group: Group, arr: Group[]) => void; constructor( root: string, options: Options, - callback?: ResultCallback + callback?: ResultCallback, + pushPath?: (path: string, arr: string[]) => void, + pushGroup?: (group: Group, arr: Group[]) => void ) { this.isSynchronous = !callback; this.callbackInvoker = invokeCallback.build(options, this.isSynchronous); @@ -66,10 +70,34 @@ export class Walker { this.groupFiles = groupFiles.build(options); this.resolveSymlink = resolveSymlink.build(options, this.isSynchronous); this.walkDirectory = walkDirectory.build(this.isSynchronous); + this.pushPath = + pushPath || + ((p, arr) => { + arr.push(p); + }); + this.pushGroup = + pushGroup || + ((group, arr) => { + arr.push(group); + }); + } + + get aborted(): boolean { + const { + controller, + options: { signal }, + } = this.state; + return controller.aborted || (signal !== undefined && signal.aborted); } start(): TOutput | null { - this.pushDirectory(this.root, this.state.paths, this.state.options.filters); + this.pushDirectory( + this.root, + this.state.paths, + this.pushPath, + this.state.counts, + this.state.options.filters + ); this.walkDirectory( this.state, this.root, @@ -80,31 +108,40 @@ export class Walker { return this.isSynchronous ? this.callbackInvoker(this.state, null) : null; } + stop(): void { + this.state.controller.abort(); + } + private walk = (entries: Dirent[], directoryPath: string, depth: number) => { const { - paths, + counts, options: { filters, resolveSymlinks, excludeSymlinks, exclude, maxFiles, - signal, useRealPaths, pathSeparator, }, - controller, } = this.state; if ( - controller.aborted || - (signal && signal.aborted) || - (maxFiles && paths.length > maxFiles) + this.aborted || + (maxFiles && counts.directories + counts.files > maxFiles) ) return; const files = this.getArray(this.state.paths); for (let i = 0; i < entries.length; ++i) { + if (maxFiles && counts.directories + counts.files >= maxFiles) { + break; + } + + if (this.aborted) { + break; + } + const entry = entries[i]; if ( @@ -112,7 +149,13 @@ export class Walker { (entry.isSymbolicLink() && !resolveSymlinks && !excludeSymlinks) ) { const filename = this.joinPath(entry.name, directoryPath); - this.pushFile(filename, files, this.state.counts, filters); + this.pushFile( + filename, + files, + this.pushPath, + this.state.counts, + filters + ); } else if (entry.isDirectory()) { let path = joinPath.joinDirectoryPath( entry.name, @@ -120,7 +163,7 @@ export class Walker { this.state.options.pathSeparator ); if (exclude && exclude(entry.name, path)) continue; - this.pushDirectory(path, paths, filters); + this.pushDirectory(path, files, this.pushPath, counts, filters); this.walkDirectory(this.state, path, path, depth - 1, this.walk); } else if (this.resolveSymlink && entry.isSymbolicLink()) { let path = joinPath.joinPathWithBasePath(entry.name, directoryPath); @@ -151,12 +194,18 @@ export class Walker { this.state.options ); resolvedPath = this.joinPath(filename, directoryPath); - this.pushFile(resolvedPath, files, this.state.counts, filters); + this.pushFile( + resolvedPath, + files, + this.pushPath, + this.state.counts, + filters + ); } }); } } - this.groupFiles(this.state.groups, directoryPath, files); + this.groupFiles(this.state.groups, directoryPath, files, this.pushGroup); }; } diff --git a/src/builder/api-builder.ts b/src/builder/api-builder.ts index 4c1af17..f3de171 100644 --- a/src/builder/api-builder.ts +++ b/src/builder/api-builder.ts @@ -1,6 +1,13 @@ import { callback, promise } from "../api/async"; import { sync } from "../api/sync"; -import { Options, Output, ResultCallback } from "../types"; +import { iterator } from "../api/iterator"; +import { + Options, + Output, + ResultCallback, + IterableOutput, + OutputIterator, +} from "../types"; export class APIBuilder { constructor( @@ -19,4 +26,11 @@ export class APIBuilder { sync(): TReturnType { return sync(this.root, this.options); } + + withIterator(): TReturnType extends IterableOutput + ? OutputIterator + : never { + // TODO (43081j): get rid of this awful `never` + return iterator(this.root, this.options) as never; + } } diff --git a/src/types.ts b/src/types.ts index da900e0..5f10c6f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -26,6 +26,12 @@ export type OnlyCountsOutput = Counts; export type PathsOutput = string[]; export type Output = OnlyCountsOutput | PathsOutput | GroupOutput; +export type IterableOutput = PathsOutput; +export type OutputIterator = AsyncGenerator< + T[number], + void, + undefined +>; export type FSLike = { readdir: typeof nativeFs.readdir;