Skip to content

Commit af74737

Browse files
dabbottdabbott
committed
Compile in chunks
1 parent 3a1f6dd commit af74737

3 files changed

Lines changed: 227 additions & 87 deletions

File tree

‎packages/noya-compiler/src/compileProject.ts‎

Lines changed: 129 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1+
import { cartesianProduct } from '@noya-app/noya-utils';
12
import { path } from 'imfs';
23
import { loadDesignSystem } from 'noya-module-loader';
34
import { clean } from './clean';
45
import { sortFiles } from './common';
56
import { compileDesignSystem } from './compileDesignSystem';
7+
import { reduceIterator, reduceIteratorChunked } from './processWork';
68
import { CompilerConfiguration } from './types';
79
import { sanitizePackageName } from './validate';
810

@@ -33,78 +35,120 @@ const colorNames = [
3335

3436
const colorModes = ['light' as const, 'dark' as const];
3537

36-
export function compile(configuration: CompilerConfiguration) {
38+
export function compilePermutation(
39+
configuration: CompilerConfiguration,
40+
{
41+
colorMode,
42+
colorName,
43+
libraryName,
44+
}: { colorMode: 'light' | 'dark'; colorName: string; libraryName: string },
45+
) {
46+
const allDSFiles: Record<string, string> = {};
47+
48+
const {
49+
files: dsFiles,
50+
dependencies,
51+
devDependencies,
52+
allExportMap,
53+
} = compileDesignSystem({
54+
...configuration,
55+
ds: {
56+
...configuration.ds,
57+
config: {
58+
...configuration.ds.config,
59+
colorMode,
60+
colors: {
61+
...configuration.ds.config.colors,
62+
primary: colorName,
63+
},
64+
},
65+
},
66+
designSystemDefinition: configuration.resolvedDefinitions[libraryName],
67+
includeTailwindBase: libraryName === 'vanilla',
68+
spreadTheme: libraryName.endsWith('radix'),
69+
exportTypes:
70+
libraryName === 'vanilla'
71+
? [
72+
'html-css',
73+
'html-tailwind',
74+
'react-css',
75+
'react-css-modules',
76+
'react-tailwind',
77+
]
78+
: libraryName.endsWith('chakra') || libraryName.endsWith('radix')
79+
? ['react']
80+
: ['react-css', 'react-tailwind'],
81+
});
82+
83+
const basename = path.basename(libraryName);
84+
85+
Object.assign(
86+
allDSFiles,
87+
addPathPrefix(
88+
dsFiles,
89+
`src/app/${basename}/${colorName}/${colorMode}/components/`,
90+
),
91+
);
92+
93+
for (const [componentName, exportMap] of Object.entries(allExportMap)) {
94+
for (const [exportType, files] of Object.entries(exportMap)) {
95+
Object.assign(
96+
allDSFiles,
97+
addPathPrefix(
98+
files,
99+
`public/${basename}/${colorName}/${colorMode}/components/${componentName}/${exportType}/`,
100+
),
101+
);
102+
}
103+
}
104+
105+
return {
106+
files: allDSFiles,
107+
dependencies,
108+
devDependencies,
109+
};
110+
}
111+
112+
type ProgressCallback = (progress: {
113+
current: number;
114+
total: number;
115+
fileCount: number;
116+
}) => void;
117+
118+
export function* compileGenerator(
119+
configuration: CompilerConfiguration,
120+
onProgress?: ProgressCallback,
121+
) {
37122
const allDefinitions = Object.keys(configuration.resolvedDefinitions);
38123

39124
const allDSFiles: Record<string, string> = {};
40125
const allDependencies: Record<string, string> = {};
41126
const allDevDependencies: Record<string, string> = {};
42127

43-
for (const libraryName of allDefinitions) {
44-
for (const colorMode of colorModes) {
45-
for (const colorName of colorNames) {
46-
const {
47-
files: dsFiles,
48-
dependencies,
49-
devDependencies,
50-
allExportMap,
51-
} = compileDesignSystem({
52-
...configuration,
53-
ds: {
54-
...configuration.ds,
55-
config: {
56-
...configuration.ds.config,
57-
colorMode,
58-
colors: {
59-
...configuration.ds.config.colors,
60-
primary: colorName,
61-
},
62-
},
63-
},
64-
designSystemDefinition:
65-
configuration.resolvedDefinitions[libraryName],
66-
includeTailwindBase: libraryName === 'vanilla',
67-
spreadTheme: libraryName.endsWith('radix'),
68-
exportTypes:
69-
libraryName === 'vanilla'
70-
? [
71-
'html-css',
72-
'html-tailwind',
73-
'react-css',
74-
'react-css-modules',
75-
'react-tailwind',
76-
]
77-
: libraryName.endsWith('chakra') || libraryName.endsWith('radix')
78-
? ['react']
79-
: ['react-css', 'react-tailwind'],
80-
});
81-
82-
Object.assign(allDependencies, dependencies);
83-
Object.assign(allDevDependencies, devDependencies);
84-
85-
const basename = path.basename(libraryName);
86-
87-
Object.assign(
88-
allDSFiles,
89-
addPathPrefix(
90-
dsFiles,
91-
`src/app/${basename}/${colorName}/${colorMode}/components/`,
92-
),
93-
);
94-
95-
for (const [componentName, exportMap] of Object.entries(allExportMap)) {
96-
for (const [exportType, files] of Object.entries(exportMap)) {
97-
Object.assign(
98-
allDSFiles,
99-
addPathPrefix(
100-
files,
101-
`public/${basename}/${colorName}/${colorMode}/components/${componentName}/${exportType}/`,
102-
),
103-
);
104-
}
105-
}
106-
}
107-
}
128+
const allPermutations = cartesianProduct(
129+
allDefinitions,
130+
colorModes,
131+
colorNames,
132+
);
133+
134+
let fileCount = 0;
135+
136+
for (let i = 0; i < allPermutations.length; i++) {
137+
const [libraryName, colorMode, colorName] = allPermutations[i];
138+
const { files, dependencies, devDependencies } = compilePermutation(
139+
configuration,
140+
{ libraryName, colorMode, colorName },
141+
);
142+
143+
fileCount += Object.keys(files).length;
144+
145+
Object.assign(allDSFiles, files);
146+
Object.assign(allDependencies, dependencies);
147+
Object.assign(allDevDependencies, devDependencies);
148+
149+
onProgress?.({ current: i + 1, total: allPermutations.length, fileCount });
150+
151+
yield allDSFiles;
108152
}
109153

110154
const files = {
@@ -182,10 +226,21 @@ export default config
182226
return sortFiles(files);
183227
}
184228

229+
export function compile(
230+
configuration: CompilerConfiguration,
231+
): Record<string, string> {
232+
return reduceIterator(
233+
compileGenerator(configuration),
234+
(previous, current) => current,
235+
{},
236+
);
237+
}
238+
185239
export async function compileAsync(
186240
configuration: Omit<CompilerConfiguration, 'resolvedDefinitions'> & {
187241
definitions: string[];
188242
},
243+
onProgress?: ProgressCallback,
189244
) {
190245
const resolvedDefinitions = Object.fromEntries(
191246
await Promise.all(
@@ -196,10 +251,14 @@ export async function compileAsync(
196251
),
197252
);
198253

199-
return compile({
200-
...configuration,
201-
resolvedDefinitions,
202-
});
254+
const resolvedConfiguration = { ...configuration, resolvedDefinitions };
255+
256+
return await reduceIteratorChunked(
257+
compileGenerator(resolvedConfiguration, onProgress),
258+
1000,
259+
(previous, current) => current,
260+
{},
261+
);
203262
}
204263

205264
function addPathPrefix(

‎packages/noya-compiler/src/processWork.ts‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
export function reduceIterator<T, Result = T>(
2+
iterator: Iterator<T, T, unknown>,
3+
accumulator: (
4+
previousResult: Result,
5+
currentValue: T,
6+
isDone: boolean,
7+
) => Result,
8+
initialValue: Result,
9+
): Result {
10+
while (true) {
11+
const result = iterator.next();
12+
13+
initialValue = accumulator(initialValue, result.value, !!result.done);
14+
15+
if (result.done) break;
16+
}
17+
18+
return initialValue;
19+
}
20+
21+
export async function reduceIteratorChunked<T, Result = T>(
22+
iterator: Iterator<T, T, unknown>,
23+
targetDurationMs: number,
24+
accumulator: (
25+
previousResult: Result,
26+
currentValue: T,
27+
isDone: boolean,
28+
) => Result,
29+
initialValue: Result,
30+
): Promise<Result> {
31+
let chunkSize = 1; // Start with a single iteration per chunk
32+
let elapsedTime = 0;
33+
let finalValue = initialValue;
34+
let isDone = false;
35+
36+
while (!isDone) {
37+
let startTime = Date.now();
38+
39+
for (let i = 0; i < chunkSize; i++) {
40+
const result = iterator.next();
41+
42+
finalValue = accumulator(finalValue, result.value, !!result.done);
43+
44+
if (result.done) {
45+
isDone = true;
46+
break; // Exit the for loop immediately
47+
}
48+
49+
// If not done, process the yielded value as needed (omitted here)
50+
}
51+
52+
if (!isDone) {
53+
let currentTime = Date.now();
54+
elapsedTime += currentTime - startTime;
55+
56+
// Adjust chunk size based on the average time per iteration
57+
if (elapsedTime >= targetDurationMs) {
58+
chunkSize = Math.max(
59+
1,
60+
Math.round((chunkSize * targetDurationMs) / elapsedTime),
61+
);
62+
elapsedTime = 0;
63+
await new Promise((resolve) => setTimeout(resolve, 0)); // Yield control
64+
}
65+
}
66+
}
67+
68+
return finalValue;
69+
}

‎packages/site/src/dseditor/DSEditor.tsx‎

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -824,26 +824,34 @@ function DSGalleryCode({
824824
componentID?: string;
825825
}) {
826826
const [files, setFiles] = React.useState<Record<string, string>>();
827+
const [progress, setProgress] = React.useState<string | undefined>();
827828

828829
useEffect(() => {
829830
async function main() {
830-
const output = await compileAsync({
831-
name: 'Gallery',
832-
ds,
833-
definitions: componentID
834-
? [ds.source.name]
835-
: [
836-
'vanilla',
837-
'@noya-design-system/chakra',
838-
'@noya-design-system/antd',
839-
'@noya-design-system/mui',
840-
'@noya-design-system/radix',
841-
],
842-
...(componentID && {
843-
filterComponents: (component) =>
844-
component.componentID === componentID,
845-
}),
846-
});
831+
const output = await compileAsync(
832+
{
833+
name: 'Gallery',
834+
ds,
835+
definitions: componentID
836+
? [ds.source.name]
837+
: [
838+
'vanilla',
839+
'@noya-design-system/chakra',
840+
'@noya-design-system/antd',
841+
'@noya-design-system/mui',
842+
'@noya-design-system/radix',
843+
],
844+
...(componentID && {
845+
filterComponents: (component) =>
846+
component.componentID === componentID,
847+
}),
848+
},
849+
(progress) => {
850+
setProgress(
851+
`${progress.current} / ${progress.total} (${progress.fileCount} files)`,
852+
);
853+
},
854+
);
847855

848856
setFiles(output);
849857
}
@@ -853,6 +861,10 @@ function DSGalleryCode({
853861

854862
return (
855863
<Stack.V flex="1" gap="12px">
864+
<Stack.H gap="12px" position="absolute" top="10px">
865+
<span style={{ fontWeight: 500 }}>Status:</span>
866+
<span>{progress ?? 'Compiling...'}</span>
867+
</Stack.H>
856868
{files !== undefined && (
857869
<Playground
858870
files={files}

0 commit comments

Comments
 (0)