Skip to content

Commit d35efc4

Browse files
authored
performance: Improve metro-file-map's Node crawler hot-path (#1677)
## Summary When a larger amount of paths are being crawled, the Node crawler becomes a hot-path and minor tweaks can have a larger impact on startup time, than previously anticipated (at least for Expo projects) - Remove repeated `absoluteToNormal` calls (medium impact) - Add a dictionary for extensions lookup (very low impact) - Strip `forEach` calls for for-of (formatting only; virtually no impact) - Refactor Promise code (no impact, just formatting) Changelog: ``` - **[Performance]**: 33% faster non-Watchman filesystem crawl on startup ``` ## Test plan - CI tests should pass unchanged - (Rob: See benchmarks and tests in PR comments)
1 parent a5d11f7 commit d35efc4

2 files changed

Lines changed: 201 additions & 105 deletions

File tree

‎packages/metro-file-map/src/crawlers/__tests__/node-test.js‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,49 @@ jest.mock('graceful-fs', () => {
8383
]),
8484
0,
8585
);
86+
} else if (slash(dir) === '/project/prototype') {
87+
setTimeout(
88+
() =>
89+
callback(
90+
null,
91+
[
92+
'a.js',
93+
'b.constructor',
94+
'c.toString',
95+
'd.__proto__',
96+
'e.hasOwnProperty',
97+
].map(name => ({
98+
isDirectory: () => false,
99+
isSymbolicLink: () => false,
100+
name,
101+
})),
102+
),
103+
0,
104+
);
105+
} else if (slash(dir) === '/project') {
106+
setTimeout(
107+
() =>
108+
callback(null, [
109+
{
110+
isDirectory: () => true,
111+
isSymbolicLink: () => false,
112+
name: 'fruits',
113+
},
114+
]),
115+
0,
116+
);
117+
} else if (slash(dir) === '/') {
118+
setTimeout(
119+
() =>
120+
callback(null, [
121+
{
122+
isDirectory: () => true,
123+
isSymbolicLink: () => false,
124+
name: 'project',
125+
},
126+
]),
127+
0,
128+
);
86129
} else if (slash(dir) == '/error') {
87130
setTimeout(() => callback({code: 'ENOTDIR'}, undefined), 0);
88131
}
@@ -274,4 +317,67 @@ describe('node crawler', () => {
274317
}),
275318
).rejects.toThrow(err);
276319
});
320+
321+
test('only crawls files with a listed extension', async () => {
322+
nodeCrawl = require('../node').default;
323+
324+
const {changedFiles} = await nodeCrawl({
325+
console: global.console,
326+
previousState: {fileSystem: emptyFS},
327+
extensions: ['js'],
328+
ignore: pearMatcher,
329+
rootDir,
330+
roots: ['/project/prototype'],
331+
});
332+
333+
// Extensions that name Object.prototype properties are not listed.
334+
expect([...changedFiles.keys()]).toEqual([normalize('prototype/a.js')]);
335+
});
336+
337+
test('crawls a root that is the parent of rootDir', async () => {
338+
nodeCrawl = require('../node').default;
339+
340+
const {changedFiles} = await nodeCrawl({
341+
console: global.console,
342+
previousState: {fileSystem: new TreeFS({rootDir: '/project/fruits'})},
343+
extensions: ['js'],
344+
ignore: pearMatcher,
345+
rootDir: '/project/fruits',
346+
roots: ['/project'],
347+
});
348+
349+
// Files under rootDir are normal paths relative to it, not '../fruits/…'.
350+
expect([...changedFiles.keys()].sort()).toEqual(
351+
['directory/strawberry.js', 'tomato.js'].map(normalize),
352+
);
353+
});
354+
355+
// The readdir mock is written with POSIX paths.
356+
(process.platform === 'win32' ? test.skip : test)(
357+
'crawls from a filesystem root without doubling separators',
358+
async () => {
359+
const fs = require('graceful-fs');
360+
nodeCrawl = require('../node').default;
361+
const ignore = jest.fn(pearMatcher);
362+
363+
const {changedFiles} = await nodeCrawl({
364+
console: global.console,
365+
previousState: {fileSystem: emptyFS},
366+
extensions: ['js'],
367+
ignore,
368+
rootDir,
369+
roots: ['/'],
370+
});
371+
372+
expect([...changedFiles.keys()].sort()).toEqual([
373+
'fruits/directory/strawberry.js',
374+
'fruits/tomato.js',
375+
]);
376+
expect(ignore).toHaveBeenCalledWith('/project');
377+
expect(fs.lstat).toHaveBeenCalledWith(
378+
'/project/fruits/tomato.js',
379+
expect.any(Function),
380+
);
381+
},
382+
);
277383
});

‎packages/metro-file-map/src/crawlers/node/index.js‎

Lines changed: 95 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -9,142 +9,132 @@
99
* @oncall react_native
1010
*/
1111

12-
import type {
13-
Console,
14-
CrawlerOptions,
15-
CrawlResult,
16-
FileData,
17-
IgnoreMatcher,
18-
} from '../../flow-types';
12+
import type {CrawlerOptions, CrawlResult, FileData} from '../../flow-types';
1913

2014
import {RootPathUtils} from '../../lib/RootPathUtils';
2115
import * as fs from 'graceful-fs';
2216
import * as path from 'node:path';
2317

24-
type Callback = (result: FileData) => void;
25-
26-
function find(
27-
roots: ReadonlyArray<string>,
28-
extensions: ReadonlyArray<string>,
29-
ignore: IgnoreMatcher,
30-
includeSymlinks: boolean,
31-
rootDir: string,
32-
console: Console,
33-
callback: Callback,
34-
): void {
18+
function find(options: CrawlerOptions): Promise<FileData> {
19+
const {console, extensions, ignore, includeSymlinks, rootDir, roots} =
20+
options;
3521
const result: FileData = new Map();
36-
let activeCalls = 0;
3722
const pathUtils = new RootPathUtils(rootDir);
23+
const exts = new Set(extensions);
3824

39-
function search(directory: string): void {
40-
activeCalls++;
41-
fs.readdir(directory, {withFileTypes: true}, (err, entries) => {
42-
activeCalls--;
43-
if (err) {
44-
console.warn(
45-
`Error "${err.code ?? err.message}" reading contents of "${directory}", skipping. Add this directory to your ignore list to exclude it.`,
46-
);
47-
} else {
48-
entries.forEach((entry: fs.Dirent) => {
49-
const file = path.join(directory, entry.name.toString());
50-
51-
if (ignore(file)) {
52-
return;
53-
}
25+
return new Promise(resolve => {
26+
let activeCalls = 0;
27+
const resolveIfDone = () => {
28+
if (activeCalls === 0) {
29+
resolve(result);
30+
}
31+
};
5432

55-
if (entry.isSymbolicLink() && !includeSymlinks) {
56-
return;
57-
}
33+
// `dirPrefix` is `directory` with a trailing separator, which only a root
34+
// may already have (a filesystem root, '/' or 'C:\\').
35+
function search(
36+
directory: string,
37+
dirPrefix: string,
38+
dirNormal: string,
39+
isWithinRoot: boolean,
40+
): void {
41+
activeCalls++;
42+
fs.readdir(directory, {withFileTypes: true}, (err, entries) => {
43+
activeCalls--;
44+
if (err) {
45+
console.warn(
46+
`Error "${err.code ?? err.message}" reading contents of "${directory}", skipping. Add this directory to your ignore list to exclude it.`,
47+
);
48+
} else {
49+
for (const entry of entries) {
50+
const name = entry.name.toString();
51+
const file = dirPrefix + name;
52+
53+
const isSymbolicLink = entry.isSymbolicLink();
54+
if (ignore(file) || (!includeSymlinks && isSymbolicLink)) {
55+
continue;
56+
}
5857

59-
if (entry.isDirectory()) {
60-
search(file);
61-
return;
62-
}
58+
// Within rootDir, a child's normal path is its parent's plus its
59+
// name. Outside rootDir, normal paths begin with '..' and can
60+
// collapse into rootDir, so derive them with absoluteToNormal
61+
// until the walk enters rootDir itself.
62+
const childNormal = !isWithinRoot
63+
? pathUtils.absoluteToNormal(file)
64+
: dirNormal === ''
65+
? name
66+
: dirNormal + path.sep + name;
67+
68+
if (entry.isDirectory()) {
69+
search(
70+
file,
71+
file + path.sep,
72+
childNormal,
73+
isWithinRoot || childNormal === '',
74+
);
75+
continue;
76+
}
6377

64-
activeCalls++;
78+
const ext = path.extname(name).substr(1);
79+
if (!isSymbolicLink && !exts.has(ext)) {
80+
continue;
81+
}
6582

66-
fs.lstat(file, (err, stat) => {
67-
activeCalls--;
83+
activeCalls++;
84+
fs.lstat(file, (err, stat) => {
85+
activeCalls--;
6886

69-
if (!err && stat) {
70-
const ext = path.extname(file).substr(1);
71-
if (stat.isSymbolicLink() || extensions.includes(ext)) {
72-
result.set(pathUtils.absoluteToNormal(file), [
87+
if (!err && stat) {
88+
result.set(childNormal, [
7389
stat.mtime.getTime(),
7490
stat.size,
7591
0,
7692
null,
77-
stat.isSymbolicLink() ? 1 : 0,
93+
isSymbolicLink ? 1 : 0,
7894
null,
7995
]);
8096
}
81-
}
82-
83-
if (activeCalls === 0) {
84-
callback(result);
85-
}
86-
});
87-
});
88-
}
89-
90-
if (activeCalls === 0) {
91-
callback(result);
92-
}
93-
});
94-
}
95-
96-
if (roots.length > 0) {
97-
roots.forEach(search);
98-
} else {
99-
callback(result);
100-
}
97+
resolveIfDone();
98+
});
99+
}
100+
}
101+
resolveIfDone();
102+
});
103+
}
104+
105+
for (const root of roots) {
106+
const rootNormal = pathUtils.absoluteToNormal(root);
107+
const isWithinRoot =
108+
rootNormal !== '..' && !rootNormal.startsWith('..' + path.sep);
109+
search(
110+
root,
111+
root.endsWith(path.sep) ? root : root + path.sep,
112+
rootNormal,
113+
isWithinRoot,
114+
);
115+
}
116+
// Resolve now if there were no roots to search.
117+
resolveIfDone();
118+
});
101119
}
102120

103121
export default async function nodeCrawl(
104122
options: CrawlerOptions,
105123
): Promise<CrawlResult> {
106-
const {
107-
console,
108-
previousState,
109-
extensions,
110-
ignore,
111-
rootDir,
112-
includeSymlinks,
113-
perfLogger,
114-
roots,
115-
abortSignal,
116-
subpath,
117-
} = options;
124+
const {abortSignal, perfLogger, previousState, subpath} = options;
118125

119126
abortSignal?.throwIfAborted();
120127

121128
perfLogger?.point('nodeCrawl_start');
122129

123-
return new Promise((resolve, reject) => {
124-
const callback: Callback = fileData => {
125-
const difference = previousState.fileSystem.getDifference(fileData, {
126-
subpath,
127-
});
130+
const fileData = await find(options);
128131

129-
perfLogger?.point('nodeCrawl_end');
130-
131-
try {
132-
// TODO: Use AbortSignal.reason directly when Flow supports it
133-
abortSignal?.throwIfAborted();
134-
} catch (e) {
135-
reject(e);
136-
}
137-
resolve(difference);
138-
};
132+
abortSignal?.throwIfAborted();
139133

140-
find(
141-
roots,
142-
extensions,
143-
ignore,
144-
includeSymlinks,
145-
rootDir,
146-
console,
147-
callback,
148-
);
134+
const difference = previousState.fileSystem.getDifference(fileData, {
135+
subpath,
149136
});
137+
138+
perfLogger?.point('nodeCrawl_end');
139+
return difference;
150140
}

0 commit comments

Comments
 (0)