Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions docs/Configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -682,6 +682,14 @@ Type: `boolean`

Enables `require.context()`, which imports all the files in a directory that match a pattern. Defaults to `false`.

#### `unstable_shareTransformResults` <div class="label experimental">Experimental</div>

Type: `boolean`

Whether graphs with equal transform options share one in-memory copy of each module's transform output. Each lazy bundle is served by its own graph, so without sharing, every module it has in common with its parent is held once per graph. Defaults to `true`.

A module's `output` and its dependencies' `data` are read-only, and with sharing on, every graph that uses a module holds the same objects. Code that reads modules, such as a [custom serializer](#customserializer), serializer plugins, or a wrapper around the bundler's `transformFile`, must not write to them: a write through one graph's module changes it in every other graph. Set this option to `false` if code you depend on writes to them.

#### `unstable_workerThreads` <div class="label experimental">Experimental</div>

Type: `boolean`
Expand Down
3 changes: 2 additions & 1 deletion packages/metro-config/API.md
Original file line number Diff line number Diff line change
Expand Up @@ -217,12 +217,13 @@ export type SymbolicatorConfigT = {
customizeStack: (symbolicatedStack: Array<IntermediateStackFrame>, extraData: unknown) => Array<IntermediateStackFrame> | Promise<Array<IntermediateStackFrame>>;
};

export type TransformerConfigT = Omit<JsTransformerConfig, 'getTransformOptions' | 'transformVariants' | 'publicPath' | 'unstable_workerThreads'> & {
export type TransformerConfigT = Omit<JsTransformerConfig, 'getTransformOptions' | 'transformVariants' | 'publicPath' | 'unstable_shareTransformResults' | 'unstable_workerThreads'> & {
getTransformOptions: GetTransformOptions;
transformVariants: {
readonly [name: string]: Partial<ExtraTransformOptions>;
};
publicPath: string;
unstable_shareTransformResults: boolean;
unstable_workerThreads: boolean;
};

Expand Down
1 change: 1 addition & 0 deletions packages/metro-config/src/defaults/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ const getDefaultValues = (projectRoot: ?string): ConfigT => ({
unstable_compactOutput: false,
unstable_disableInputAstCloning: true,
unstable_memoizeInlineRequires: false,
unstable_shareTransformResults: true,
unstable_workerThreads: false,
},
watcher: {
Expand Down
3 changes: 3 additions & 0 deletions packages/metro-config/src/types.js
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,9 @@ type TransformerConfigT = {
// TODO(moti): Remove this Meta-internal option from Metro's public config
transformVariants: {readonly [name: string]: Partial<ExtraTransformOptions>},
publicPath: string,
// Share one in-memory copy of each transform result between graphs with
// equal transform options. Off when `cacheStores` is empty. Experimental.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit · Config type comment still says sharing is off when cacheStores is empty

Commit 9927748 made sharing independent of cacheStores: sharesTransformResults now returns only config.transformer.unstable_shareTransformResults, and there's a new Transformer test for sharing with cacheStores: []. The comment on unstable_shareTransformResults in TransformerConfigT still says "Off when cacheStores is empty". Anyone who reads it and sets cacheStores: [] to turn sharing off would still get sharing. Suggest removing that sentence, which would also match docs/Configuration.md.

Evidence: grep cacheStores on the head: types.js:170 is the only remaining reference to the old gating. Transformer.js sharesTransformResults doesn't read cacheStores, and the test 'with transformer.unstable_shareTransformResults and empty cacheStores, transforms once and shares the result' passes. · id a3f0bb8e

unstable_shareTransformResults: boolean,
unstable_workerThreads: boolean,
};

Expand Down
9 changes: 9 additions & 0 deletions packages/metro/src/Bundler.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
*/

import type {TransformResultWithSource} from './DeltaBundler';
import type {ReleasedModules} from './DeltaBundler/types';
import type {TransformOptions} from './DeltaBundler/Worker';
import type {DynamicRoot} from './lib/dynamicRoots';
import type {ConfigT} from 'metro-config';
Expand Down Expand Up @@ -104,6 +105,14 @@ export default class Bundler {
);
}

/**
* Drops the shared transform results of released modules that no graph
* holds any more. See `transformer.unstable_shareTransformResults`.
*/
releaseSharedTransformResults(released: ReleasedModules<>): void {
this._transformer?.releaseSharedResults(released);
}

// Waits for the bundler to become ready.
async ready(): Promise<void> {
await this._initializedPromise;
Expand Down
60 changes: 58 additions & 2 deletions packages/metro/src/DeltaBundler.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,14 @@ import type {
MixedOutput,
Options,
ReadOnlyGraph,
ReleasedModules,
} from './DeltaBundler/types';
import type EventEmitter from 'node:events';

import DeltaCalculator from './DeltaBundler/DeltaCalculator';
import debugModule from 'debug';

const debug = debugModule('Metro:DeltaBundler');

export type {
DeltaResult,
Expand All @@ -40,12 +44,23 @@ export type {
* concurrent clients requesting their own deltas. This is done through the
* `clientId` param (which maps a client to a specific delta transformer).
*/
export type DeltaBundlerOptions<T> = Readonly<{
// Called after a graph releases modules or ends, with the outputs that live
// graphs still hold at the released paths.
onModulesReleased?: ?(released: ReleasedModules<T>) => void,
}>;

export default class DeltaBundler<T = MixedOutput> {
_changeEventSource: EventEmitter;
_deltaCalculators: Map<Graph<T>, DeltaCalculator<T>> = new Map();
readonly #onModulesReleased: ?(released: ReleasedModules<T>) => void;

constructor(changeEventSource: EventEmitter) {
constructor(
changeEventSource: EventEmitter,
options?: DeltaBundlerOptions<T>,
) {
this._changeEventSource = changeEventSource;
this.#onModulesReleased = options?.onModulesReleased;
}

end(): void {
Expand Down Expand Up @@ -109,7 +124,14 @@ export default class DeltaBundler<T = MixedOutput> {
throw new Error('Graph not found');
}

return await deltaCalculator.getDelta({reset, shallow});
const delta = await deltaCalculator.getDelta({reset, shallow});
if (delta.reset) {
// A reset delta doesn't say which modules the build released.
this.#notifyModulesReleased(null);
} else if (delta.deleted.size > 0) {
this.#notifyModulesReleased(Array.from(delta.deleted));
}
return delta;
}

listen(graph: Graph<T>, callback: () => Promise<void>): () => void {
Expand All @@ -136,5 +158,39 @@ export default class DeltaBundler<T = MixedOutput> {
deltaCalculator.end();

this._deltaCalculators.delete(graph);
this.#notifyModulesReleased(Array.from(graph.dependencies.keys()));
}

// Collects the outputs that live graphs hold at `paths`, or anywhere if
// `paths` is null, so the cost scales with what was released rather than
// with the size of every graph.
#notifyModulesReleased(paths: ?ReadonlyArray<string>): void {
const onModulesReleased = this.#onModulesReleased;
if (onModulesReleased == null) {
return;
}
const start = performance.now();
const heldOutputs = new Set<ReadonlyArray<T>>();
for (const graph of this._deltaCalculators.keys()) {
if (paths == null) {
for (const module of graph.dependencies.values()) {
heldOutputs.add(module.output);
}
} else {
for (const path of paths) {
const module = graph.dependencies.get(path);
if (module != null) {
heldOutputs.add(module.output);
}
}
}
}
onModulesReleased({paths, heldOutputs});
debug(
'Released %s modules, %d outputs still held, in %dms',
paths?.length ?? 'all',
heldOutputs.size,
Math.round(performance.now() - start),
);
}
}
143 changes: 143 additions & 0 deletions packages/metro/src/DeltaBundler/SharedTransformResults.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @flow strict-local
* @format
* @oncall react_native
*/

import type {ReleasedModules, TransformResult} from './types';

export type Computed<T> = Readonly<{
// The key the result belongs to, which differs from the requested key if
// the file changed between hashing and transforming it.
key: Buffer,
result: TransformResult<T>,
}>;

type Entry<T> = Readonly<{
...Computed<T>,
keyString: string,
path: string,
}>;

/**
* Shares transform results in memory between callers that request the same
* transform cache key, i.e. between graphs with equal transform options.
*
* Entries use the cache stores' key, so they can only be stale where a store
* hit would be, and need no invalidation of their own. Each module, i.e. each
* file and set of transform options, holds only its latest result, so editing
* a file replaces its entry. `release` drops the entries at released paths
* whose output no live graph holds, and the next request for a dropped entry
* reads the cache stores or transforms again. A graph that is still building is
* not yet live, so a release during its build can drop an entry that it is
* about to hold. Later graphs then get a second copy of that module, until the
* module changes or the building graph ends.
*
* Only graphs that `DeltaBundler` registers release their paths. Entries added
* by graphs it never registers, i.e. those built by `getDependencies` and a
* first build that fails, stay until a later result for the same module
* replaces them or a live graph releases their path. This keeps prepended
* scripts, which every graph's revision holds, shared between graphs.
*
* Callers must treat `output` and `dependencies` as read-only, as their types
* declare, since every graph holding a result sees a write to it.
*/
export default class SharedTransformResults<T> {
#inFlight: Map<string, Promise<Computed<T>>> = new Map();
#settled: Map<string, Entry<T>> = new Map();
// Module keys by path, so a release only checks the released modules.
#moduleKeysByPath: Map<string, Set<string>> = new Map();

/**
* `moduleKey` identifies the module, i.e. every input to the transform
* except the file's contents, `key` is the full transform cache key, and
* `path` is the module's path in graphs.
*/
async getOrCompute(
{
key,
moduleKey,
path,
}: Readonly<{key: Buffer, moduleKey: Buffer, path: string}>,
compute: () => Promise<Computed<T>>,
): Promise<Computed<T>> {
const keyString = toKeyString(key);
const moduleKeyString = toKeyString(moduleKey);
const entry = this.#settled.get(moduleKeyString);
if (entry != null && entry.keyString === keyString) {
return {key: entry.key, result: entry.result};
}

const pending = this.#inFlight.get(keyString);
if (pending != null) {
return pending;
}

const promise = compute().then(
computed => {
this.#inFlight.delete(keyString);
const result = {
dependencies: computed.result.dependencies,
output: computed.result.output,
};
this.#settled.set(moduleKeyString, {
key: computed.key,
keyString: toKeyString(computed.key),
path,
result,
});
let moduleKeys = this.#moduleKeysByPath.get(path);
if (moduleKeys == null) {
moduleKeys = new Set();
this.#moduleKeysByPath.set(path, moduleKeys);
}
moduleKeys.add(moduleKeyString);
return {key: computed.key, result};
},
error => {
this.#inFlight.delete(keyString);
throw error;
},
);
this.#inFlight.set(keyString, promise);
return promise;
}

/**
* Drops the entries at `paths`, or every entry if `paths` is null, whose
* output isn't in `heldOutputs`. Results still in flight are unaffected.
*/
release({paths, heldOutputs}: ReleasedModules<T>): void {
const moduleKeyStrings =
paths == null
? Array.from(this.#settled.keys())
: paths.flatMap(path =>
Array.from(this.#moduleKeysByPath.get(path) ?? []),
);
for (const moduleKeyString of moduleKeyStrings) {
const entry = this.#settled.get(moduleKeyString);
if (entry != null && !heldOutputs.has(entry.result.output)) {
this.#settled.delete(moduleKeyString);
const moduleKeys = this.#moduleKeysByPath.get(entry.path);
moduleKeys?.delete(moduleKeyString);
if (moduleKeys?.size === 0) {
this.#moduleKeysByPath.delete(entry.path);
}
}
}
}

countEntries(): number {
return this.#settled.size;
}
}

// 'binary' (latin1) is a lossless one-character-per-byte encoding.
function toKeyString(key: Buffer): string {
return key.toString('binary');
}
Loading
Loading