Skip to content

Commit 4292689

Browse files
nbbeekendurran
andauthored
chore(NODE-6186): add downloading to FLE build script (#5)
Co-authored-by: Durran Jordan <[email protected]>
1 parent e35bd59 commit 4292689

File tree

5 files changed

+236
-52
lines changed

5 files changed

+236
-52
lines changed

.github/docker/Dockerfile.glibc

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
ARG NODE_BUILD_IMAGE=node:16.20.1-bullseye
2+
FROM $NODE_BUILD_IMAGE AS build
3+
4+
WORKDIR /mongodb-client-encryption
5+
COPY . .
6+
7+
RUN node /mongodb-client-encryption/.github/scripts/libmongocrypt.mjs
8+
9+
FROM scratch
10+
11+
COPY --from=build /mongodb-client-encryption/prebuilds/ /

.github/scripts/libmongocrypt.mjs

Lines changed: 155 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -4,24 +4,42 @@ import fs from 'node:fs/promises';
44
import child_process from 'node:child_process';
55
import events from 'node:events';
66
import path from 'node:path';
7+
import https from 'node:https';
8+
import stream from 'node:stream/promises';
9+
import url from 'node:url';
10+
11+
const __dirname = path.dirname(url.fileURLToPath(import.meta.url));
12+
13+
/** Resolves to the root of this repository */
14+
function resolveRoot(...paths) {
15+
return path.resolve(__dirname, '..', '..', ...paths);
16+
}
17+
18+
async function exists(fsPath) {
19+
try {
20+
await fs.access(fsPath);
21+
return true;
22+
} catch {
23+
return false;
24+
}
25+
}
726

827
async function parseArguments() {
9-
const jsonImport = { [process.version.split('.').at(0) === 'v16' ? 'assert' : 'with']: { type: 'json' } };
10-
const pkg = (await import('../../package.json', jsonImport)).default;
11-
const libmongocryptVersion = pkg['mongodb:libmongocrypt'];
28+
const pkg = JSON.parse(await fs.readFile(resolveRoot('package.json'), 'utf8'));
1229

1330
const options = {
14-
url: { short: 'u', type: 'string', default: 'https://github.com/mongodb/libmongocrypt.git' },
15-
libversion: { short: 'l', type: 'string', default: libmongocryptVersion },
16-
clean: { short: 'c', type: 'boolean' },
17-
help: { short: 'h', type: 'boolean' }
31+
gitURL: { short: 'u', type: 'string', default: 'https://github.com/mongodb/libmongocrypt.git' },
32+
libVersion: { short: 'l', type: 'string', default: pkg['mongodb:libmongocrypt'] },
33+
clean: { short: 'c', type: 'boolean', default: false },
34+
build: { short: 'b', type: 'boolean', default: false },
35+
help: { short: 'h', type: 'boolean', default: false }
1836
};
1937

2038
const args = util.parseArgs({ args: process.argv.slice(2), options, allowPositionals: false });
2139

2240
if (args.values.help) {
2341
console.log(
24-
`${process.argv[1]} ${[...Object.keys(options)]
42+
`${path.basename(process.argv[1])} ${[...Object.keys(options)]
2543
.filter(k => k !== 'help')
2644
.map(k => `[--${k}=${options[k].type}]`)
2745
.join(' ')}`
@@ -30,46 +48,47 @@ async function parseArguments() {
3048
}
3149

3250
return {
33-
libmongocrypt: { url: args.values.url, ref: args.values.libversion },
34-
clean: args.values.clean
51+
libmongocrypt: { url: args.values.gitURL, ref: args.values.libVersion },
52+
clean: args.values.clean,
53+
build: args.values.build
3554
};
3655
}
3756

3857
/** `xtrace` style command runner, uses spawn so that stdio is inherited */
3958
async function run(command, args = [], options = {}) {
40-
console.error(`+ ${command} ${args.join(' ')}`, options.cwd ? `(in: ${options.cwd})` : '');
41-
await events.once(child_process.spawn(command, args, { stdio: 'inherit', ...options }), 'exit');
59+
const commandDetails = `+ ${command} ${args.join(' ')}${options.cwd ? ` (in: ${options.cwd})` : ''}`;
60+
console.error(commandDetails);
61+
const proc = child_process.spawn(command, args, {
62+
shell: process.platform === 'win32',
63+
stdio: 'inherit',
64+
cwd: resolveRoot('.'),
65+
...options
66+
});
67+
await events.once(proc, 'exit');
68+
69+
if (proc.exitCode != 0) throw new Error(`CRASH(${proc.exitCode}): ${commandDetails}`);
4270
}
4371

4472
/** CLI flag maker: `toFlags({a: 1, b: 2})` yields `['-a=1', '-b=2']` */
4573
function toFlags(object) {
4674
return Array.from(Object.entries(object)).map(([k, v]) => `-${k}=${v}`);
4775
}
4876

49-
const args = await parseArguments();
50-
const libmongocryptRoot = path.resolve('_libmongocrypt');
51-
52-
const currentLibMongoCryptBranch = await fs.readFile(path.join(libmongocryptRoot, '.git', 'HEAD'), 'utf8').catch(() => '')
53-
const libmongocryptAlreadyClonedAndCheckedOut = currentLibMongoCryptBranch.trim().endsWith(`r-${args.libmongocrypt.ref}`);
54-
55-
if (args.clean || !libmongocryptAlreadyClonedAndCheckedOut) {
56-
console.error('fetching libmongocrypt...', args.libmongocrypt);
77+
export async function cloneLibMongoCrypt(libmongocryptRoot, { url, ref }) {
78+
console.error('fetching libmongocrypt...', { url, ref });
5779
await fs.rm(libmongocryptRoot, { recursive: true, force: true });
58-
await run('git', ['clone', args.libmongocrypt.url, libmongocryptRoot]);
59-
await run('git', ['fetch', '--tags'], { cwd: libmongocryptRoot });
60-
await run('git', ['checkout', args.libmongocrypt.ref, '-b', `r-${args.libmongocrypt.ref}`], { cwd: libmongocryptRoot });
61-
} else {
62-
console.error('libmongocrypt already up to date...', args.libmongocrypt);
80+
await run('git', ['clone', url, libmongocryptRoot]);
81+
if (ref !== 'latest') {
82+
// Support "latest" as leaving the clone as-is so whatever the default branch name is works
83+
await run('git', ['fetch', '--tags'], { cwd: libmongocryptRoot });
84+
await run('git', ['checkout', ref, '-b', `r-${ref}`], { cwd: libmongocryptRoot });
85+
}
6386
}
6487

65-
const libmongocryptBuiltVersion = await fs.readFile(path.join(libmongocryptRoot, 'VERSION_CURRENT'), 'utf8').catch(() => '');
66-
const libmongocryptAlreadyBuilt = libmongocryptBuiltVersion.trim() === args.libmongocrypt.ref;
67-
68-
if (args.clean || !libmongocryptAlreadyBuilt) {
69-
console.error('building libmongocrypt...\n', args);
88+
export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot) {
89+
console.error('building libmongocrypt...');
7090

71-
const nodeDepsRoot = path.resolve('deps');
72-
const nodeBuildRoot = path.resolve(nodeDepsRoot, 'tmp', 'libmongocrypt-build');
91+
const nodeBuildRoot = resolveRoot(nodeDepsRoot, 'tmp', 'libmongocrypt-build');
7392

7493
await fs.rm(nodeBuildRoot, { recursive: true, force: true });
7594
await fs.mkdir(nodeBuildRoot, { recursive: true });
@@ -115,11 +134,109 @@ if (args.clean || !libmongocryptAlreadyBuilt) {
115134
? toFlags({ DCMAKE_OSX_DEPLOYMENT_TARGET: '10.12' })
116135
: [];
117136

118-
await run('cmake', [...CMAKE_FLAGS, ...WINDOWS_CMAKE_FLAGS, ...MACOS_CMAKE_FLAGS, libmongocryptRoot], { cwd: nodeBuildRoot });
119-
await run('cmake', ['--build', '.', '--target', 'install', '--config', 'RelWithDebInfo'], { cwd: nodeBuildRoot });
120-
} else {
121-
console.error('libmongocrypt already built...');
137+
await run(
138+
'cmake',
139+
[...CMAKE_FLAGS, ...WINDOWS_CMAKE_FLAGS, ...MACOS_CMAKE_FLAGS, libmongocryptRoot],
140+
{ cwd: nodeBuildRoot }
141+
);
142+
await run('cmake', ['--build', '.', '--target', 'install', '--config', 'RelWithDebInfo'], {
143+
cwd: nodeBuildRoot
144+
});
145+
}
146+
147+
export async function downloadLibMongoCrypt(nodeDepsRoot, { ref }) {
148+
const downloadURL =
149+
ref === 'latest'
150+
? 'https://mciuploads.s3.amazonaws.com/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz'
151+
: `https://mciuploads.s3.amazonaws.com/libmongocrypt/all/${ref}/libmongocrypt-all.tar.gz`;
152+
153+
console.error('downloading libmongocrypt...', downloadURL);
154+
const destination = resolveRoot(`_libmongocrypt-${ref}`);
155+
156+
await fs.rm(destination, { recursive: true, force: true });
157+
await fs.mkdir(destination);
158+
159+
const platformMatrix = {
160+
['darwin-arm64']: 'macos',
161+
['darwin-x64']: 'macos',
162+
['linux-ppc64']: 'rhel-71-ppc64el',
163+
['linux-s390x']: 'rhel72-zseries-test',
164+
['linux-arm64']: 'ubuntu1804-arm64',
165+
['linux-x64']: 'rhel-70-64-bit',
166+
['win32-x64']: 'windows-test'
167+
};
168+
169+
const detectedPlatform = `${process.platform}-${process.arch}`;
170+
const prebuild = platformMatrix[detectedPlatform];
171+
if (prebuild == null) throw new Error(`Unsupported: ${detectedPlatform}`);
172+
173+
console.error(`Platform: ${detectedPlatform} Prebuild: ${prebuild}`);
174+
175+
const unzipArgs = ['-xzv', '-C', `_libmongocrypt-${ref}`, `${prebuild}/nocrypto`];
176+
console.error(`+ tar ${unzipArgs.join(' ')}`);
177+
const unzip = child_process.spawn('tar', unzipArgs, {
178+
stdio: ['pipe', 'inherit'],
179+
cwd: resolveRoot('.')
180+
});
181+
182+
const [response] = await events.once(https.get(downloadURL), 'response');
183+
184+
const start = performance.now();
185+
await stream.pipeline(response, unzip.stdin);
186+
const end = performance.now();
187+
188+
console.error(`downloaded libmongocrypt in ${(end - start) / 1000} secs...`);
189+
190+
await fs.rm(nodeDepsRoot, { recursive: true, force: true });
191+
await fs.cp(resolveRoot(destination, prebuild, 'nocrypto'), nodeDepsRoot, { recursive: true });
192+
const currentPath = path.join(nodeDepsRoot, 'lib64');
193+
try {
194+
await fs.rename(currentPath, path.join(nodeDepsRoot, 'lib'));
195+
} catch (error) {
196+
console.error(`error renaming ${currentPath}: ${error.message}`);
197+
}
198+
}
199+
200+
async function main() {
201+
const { libmongocrypt, build, clean } = await parseArguments();
202+
203+
const nodeDepsDir = resolveRoot('deps');
204+
205+
if (build) {
206+
const libmongocryptCloneDir = resolveRoot('_libmongocrypt');
207+
208+
const currentLibMongoCryptBranch = await fs
209+
.readFile(path.join(libmongocryptCloneDir, '.git', 'HEAD'), 'utf8')
210+
.catch(() => '');
211+
const isClonedAndCheckedOut = currentLibMongoCryptBranch
212+
.trim()
213+
.endsWith(`r-${libmongocrypt.ref}`);
214+
215+
if (clean || !isClonedAndCheckedOut) {
216+
await cloneLibMongoCrypt(libmongocryptCloneDir, libmongocrypt);
217+
}
218+
219+
const libmongocryptBuiltVersion = await fs
220+
.readFile(path.join(libmongocryptCloneDir, 'VERSION_CURRENT'), 'utf8')
221+
.catch(() => '');
222+
const isBuilt = libmongocryptBuiltVersion.trim() === libmongocrypt.ref;
223+
224+
if (clean || !isBuilt) {
225+
await buildLibMongoCrypt(libmongocryptCloneDir, nodeDepsDir);
226+
}
227+
} else {
228+
// Download
229+
await downloadLibMongoCrypt(nodeDepsDir, libmongocrypt);
230+
}
231+
232+
await fs.rm(resolveRoot('build'), { force: true, recursive: true });
233+
await fs.rm(resolveRoot('prebuilds'), { force: true, recursive: true });
234+
235+
// install with "ignore-scripts" so that we don't attempt to download a prebuild
236+
await run('npm', ['install', '--ignore-scripts']);
237+
// The prebuild command will make both a .node file in `./build` (local and CI testing will run on current code)
238+
// it will also produce `./prebuilds/mongodb-client-encryption-vVERSION-napi-vNAPI_VERSION-OS-ARCH.tar.gz`.
239+
await run('npm', ['run', 'prebuild']);
122240
}
123241

124-
await run('npm', ['install', '--ignore-scripts']);
125-
await run('npm', ['run', 'rebuild'], { env: { ...process.env, BUILD_TYPE: 'static' } });
242+
await main();

.github/workflows/build.yml

Lines changed: 69 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,81 @@
11
on:
22
push:
33
branches: [main]
4+
pull_request:
5+
branches: [main]
46
workflow_dispatch: {}
57

68
name: build
79

810
jobs:
9-
build:
11+
host_builds:
12+
strategy:
13+
matrix:
14+
os: [macos-11, macos-latest, windows-2019]
15+
runs-on: ${{ matrix.os }}
16+
steps:
17+
- uses: actions/checkout@v4
18+
19+
- name: Build ${{ matrix.os }} Prebuild
20+
run: node .github/scripts/libmongocrypt.mjs ${{ runner.os == 'Windows' && '--build' || '' }}
21+
shell: bash
22+
23+
- id: upload
24+
name: Upload prebuild
25+
uses: actions/upload-artifact@v4
26+
with:
27+
name: build-${{ matrix.os }}
28+
path: prebuilds/
29+
if-no-files-found: 'error'
30+
retention-days: 1
31+
compression-level: 0
32+
33+
container_builds:
34+
outputs:
35+
artifact_id: ${{ steps.upload.outputs.artifact-id }}
1036
runs-on: ubuntu-latest
1137
strategy:
12-
matrix:
13-
node: ['20.x'] # '16.x', '18.x',
14-
name: Node.js ${{ matrix.node }} build
38+
matrix:
39+
linux_arch: [s390x, arm64, amd64]
1540
steps:
16-
- uses: actions/setup-node@v4
41+
- uses: actions/checkout@v4
42+
43+
- name: Set up QEMU
44+
uses: docker/setup-qemu-action@v3
45+
46+
- name: Set up Docker Buildx
47+
uses: docker/setup-buildx-action@v3
48+
49+
- name: Run Buildx
50+
run: |
51+
docker buildx create --name builder --bootstrap --use
52+
docker buildx build --platform linux/${{ matrix.linux_arch }} --output type=local,dest=./prebuilds,platform-split=false -f ./.github/docker/Dockerfile.glibc .
53+
54+
- id: upload
55+
name: Upload prebuild
56+
uses: actions/upload-artifact@v4
1757
with:
18-
node-version: ${{ matrix.node }}
19-
cache: 'npm'
20-
registry-url: 'https://registry.npmjs.org'
21-
- run: npm install -g npm@latest
22-
shell: bash
23-
- run: node .github/scripts/libmongocrypt.mjs
24-
shell: bash
58+
name: build-linux-${{ matrix.linux_arch }}
59+
path: prebuilds/
60+
if-no-files-found: 'error'
61+
retention-days: 1
62+
compression-level: 0
63+
64+
collect:
65+
needs: [host_builds, container_builds]
66+
runs-on: ubunutu-latest
67+
steps:
68+
- uses: actions/download-artifact@v4
69+
70+
- name: Display structure of downloaded files
71+
run: ls -R
72+
73+
- id: upload
74+
name: Upload all prebuilds
75+
uses: actions/upload-artifact@v4
76+
with:
77+
name: all-build
78+
path: '*.tar.gz'
79+
if-no-files-found: 'error'
80+
retention-days: 1
81+
compression-level: 0

.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,4 @@ xunit.xml
2525
lib
2626
prebuilds
2727

28-
_libmongocrypt/
28+
_libmongocrypt*

package.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
"check:clang-format": "clang-format --style=file:.clang-format --dry-run --Werror addon/*",
2323
"test": "mocha test",
2424
"prepare": "tsc",
25-
"rebuild": "prebuild --compile",
2625
"prebuild": "prebuild --runtime napi --strip --verbose --all"
2726
},
2827
"author": {

0 commit comments

Comments
 (0)