Skip to content

Commit f4106c2

Browse files
Merge branch 'main' into NODE-6185
2 parents 74f96c8 + 4292689 commit f4106c2

File tree

14 files changed

+798
-183
lines changed

14 files changed

+798
-183
lines changed

.github/docker/Dockerfile.glibc

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
ARG NODE_BUILD_IMAGE=node:16.20.1-bullseye
2+
FROM $NODE_BUILD_IMAGE AS build
3+
4+
WORKDIR /mongodb-client-encryption
5+
COPY . .
6+
7+
RUN node /mongodb-client-encryption/.github/scripts/libmongocrypt.mjs
8+
9+
FROM scratch
10+
11+
COPY --from=build /mongodb-client-encryption/prebuilds/ /

.github/pull_request_template.md

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
### Description
2+
3+
#### What is changing?
4+
5+
##### Is there new documentation needed for these changes?
6+
7+
#### What is the motivation for this change?
8+
9+
<!-- If this is a bug, it helps to describe the current behavior and a clear outline of the expected behavior -->
10+
<!-- If this is a feature, it helps to describe the new use case enabled by this change -->
11+
12+
<!--
13+
Contributors!
14+
First of all, thank you so much!!
15+
If you haven't already, it would greatly help the team review this work in a timely manner if you create a JIRA ticket to track this PR.
16+
You can do that here: https://jira.mongodb.org/projects/NODE
17+
-->
18+
19+
### Release Highlight
20+
21+
<!-- RELEASE_HIGHLIGHT_START -->
22+
23+
### Fill in title or leave empty for no highlight
24+
25+
<!-- RELEASE_HIGHLIGHT_END -->
26+
27+
### Double check the following
28+
29+
- [ ] Ran `npm run check:lint` script
30+
- [ ] Self-review completed using the [steps outlined here](https://github.com/mongodb/node-mongodb-native/blob/HEAD/CONTRIBUTING.md#reviewer-guidelines)
31+
- [ ] PR title follows the [correct format](https://www.conventionalcommits.org/en/v1.0.0/): `type(NODE-xxxx)[!]: description`
32+
- Example: `feat(NODE-1234)!: rewriting everything in coffeescript`
33+
- [ ] Changes are covered by tests
34+
- [ ] New TODOs have a related JIRA ticket

.github/scripts/libmongocrypt.mjs

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
import util from 'node:util';
2+
import process from 'node:process';
3+
import fs from 'node:fs/promises';
4+
import child_process from 'node:child_process';
5+
import events from 'node:events';
6+
import path from 'node:path';
7+
import https from 'node:https';
8+
import stream from 'node:stream/promises';
9+
import url from 'node:url';
10+
11+
const __dirname = path.dirname(url.fileURLToPath(import.meta.url));
12+
13+
/** Resolves to the root of this repository */
14+
function resolveRoot(...paths) {
15+
return path.resolve(__dirname, '..', '..', ...paths);
16+
}
17+
18+
async function exists(fsPath) {
19+
try {
20+
await fs.access(fsPath);
21+
return true;
22+
} catch {
23+
return false;
24+
}
25+
}
26+
27+
async function parseArguments() {
28+
const pkg = JSON.parse(await fs.readFile(resolveRoot('package.json'), 'utf8'));
29+
30+
const options = {
31+
gitURL: { short: 'u', type: 'string', default: 'https://github.com/mongodb/libmongocrypt.git' },
32+
libVersion: { short: 'l', type: 'string', default: pkg['mongodb:libmongocrypt'] },
33+
clean: { short: 'c', type: 'boolean', default: false },
34+
build: { short: 'b', type: 'boolean', default: false },
35+
help: { short: 'h', type: 'boolean', default: false }
36+
};
37+
38+
const args = util.parseArgs({ args: process.argv.slice(2), options, allowPositionals: false });
39+
40+
if (args.values.help) {
41+
console.log(
42+
`${path.basename(process.argv[1])} ${[...Object.keys(options)]
43+
.filter(k => k !== 'help')
44+
.map(k => `[--${k}=${options[k].type}]`)
45+
.join(' ')}`
46+
);
47+
process.exit(0);
48+
}
49+
50+
return {
51+
libmongocrypt: { url: args.values.gitURL, ref: args.values.libVersion },
52+
clean: args.values.clean,
53+
build: args.values.build
54+
};
55+
}
56+
57+
/** `xtrace` style command runner, uses spawn so that stdio is inherited */
58+
async function run(command, args = [], options = {}) {
59+
const commandDetails = `+ ${command} ${args.join(' ')}${options.cwd ? ` (in: ${options.cwd})` : ''}`;
60+
console.error(commandDetails);
61+
const proc = child_process.spawn(command, args, {
62+
shell: process.platform === 'win32',
63+
stdio: 'inherit',
64+
cwd: resolveRoot('.'),
65+
...options
66+
});
67+
await events.once(proc, 'exit');
68+
69+
if (proc.exitCode != 0) throw new Error(`CRASH(${proc.exitCode}): ${commandDetails}`);
70+
}
71+
72+
/** CLI flag maker: `toFlags({a: 1, b: 2})` yields `['-a=1', '-b=2']` */
73+
function toFlags(object) {
74+
return Array.from(Object.entries(object)).map(([k, v]) => `-${k}=${v}`);
75+
}
76+
77+
export async function cloneLibMongoCrypt(libmongocryptRoot, { url, ref }) {
78+
console.error('fetching libmongocrypt...', { url, ref });
79+
await fs.rm(libmongocryptRoot, { recursive: true, force: true });
80+
await run('git', ['clone', url, libmongocryptRoot]);
81+
if (ref !== 'latest') {
82+
// Support "latest" as leaving the clone as-is so whatever the default branch name is works
83+
await run('git', ['fetch', '--tags'], { cwd: libmongocryptRoot });
84+
await run('git', ['checkout', ref, '-b', `r-${ref}`], { cwd: libmongocryptRoot });
85+
}
86+
}
87+
88+
export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot) {
89+
console.error('building libmongocrypt...');
90+
91+
const nodeBuildRoot = resolveRoot(nodeDepsRoot, 'tmp', 'libmongocrypt-build');
92+
93+
await fs.rm(nodeBuildRoot, { recursive: true, force: true });
94+
await fs.mkdir(nodeBuildRoot, { recursive: true });
95+
96+
const CMAKE_FLAGS = toFlags({
97+
/**
98+
* We provide crypto hooks from Node.js binding to openssl (so disable system crypto)
99+
* TODO: NODE-5455
100+
*
101+
* One thing that is not obvious from the build instructions for libmongocrypt
102+
* and the Node.js bindings is that the Node.js driver uses libmongocrypt in
103+
* DISABLE_NATIVE_CRYPTO aka nocrypto mode, that is, instead of using native
104+
* system libraries for crypto operations, it provides callbacks to libmongocrypt
105+
* which, in the Node.js addon case, call JS functions that in turn call built-in
106+
* Node.js crypto methods.
107+
*
108+
* That’s way more convoluted than it needs to be, considering that we always
109+
* have a copy of OpenSSL available directly, but for now it seems to make sense
110+
* to stick with what the Node.js addon does here.
111+
*/
112+
DDISABLE_NATIVE_CRYPTO: '1',
113+
/** A consistent name for the output "library" directory */
114+
DCMAKE_INSTALL_LIBDIR: 'lib',
115+
/** No warnings allowed */
116+
DENABLE_MORE_WARNINGS_AS_ERRORS: 'ON',
117+
/** Where to build libmongocrypt */
118+
DCMAKE_PREFIX_PATH: nodeDepsRoot,
119+
/**
120+
* Where to install libmongocrypt
121+
* Note that `binding.gyp` will set `./deps/include`
122+
* as an include path if BUILD_TYPE=static
123+
*/
124+
DCMAKE_INSTALL_PREFIX: nodeDepsRoot
125+
});
126+
127+
const WINDOWS_CMAKE_FLAGS =
128+
process.platform === 'win32' // Windows is still called "win32" when it is 64-bit
129+
? toFlags({ Thost: 'x64', A: 'x64', DENABLE_WINDOWS_STATIC_RUNTIME: 'ON' })
130+
: [];
131+
132+
const MACOS_CMAKE_FLAGS =
133+
process.platform === 'darwin' // The minimum macos target version we want for
134+
? toFlags({ DCMAKE_OSX_DEPLOYMENT_TARGET: '10.12' })
135+
: [];
136+
137+
await run(
138+
'cmake',
139+
[...CMAKE_FLAGS, ...WINDOWS_CMAKE_FLAGS, ...MACOS_CMAKE_FLAGS, libmongocryptRoot],
140+
{ cwd: nodeBuildRoot }
141+
);
142+
await run('cmake', ['--build', '.', '--target', 'install', '--config', 'RelWithDebInfo'], {
143+
cwd: nodeBuildRoot
144+
});
145+
}
146+
147+
export async function downloadLibMongoCrypt(nodeDepsRoot, { ref }) {
148+
const downloadURL =
149+
ref === 'latest'
150+
? 'https://mciuploads.s3.amazonaws.com/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz'
151+
: `https://mciuploads.s3.amazonaws.com/libmongocrypt/all/${ref}/libmongocrypt-all.tar.gz`;
152+
153+
console.error('downloading libmongocrypt...', downloadURL);
154+
const destination = resolveRoot(`_libmongocrypt-${ref}`);
155+
156+
await fs.rm(destination, { recursive: true, force: true });
157+
await fs.mkdir(destination);
158+
159+
const platformMatrix = {
160+
['darwin-arm64']: 'macos',
161+
['darwin-x64']: 'macos',
162+
['linux-ppc64']: 'rhel-71-ppc64el',
163+
['linux-s390x']: 'rhel72-zseries-test',
164+
['linux-arm64']: 'ubuntu1804-arm64',
165+
['linux-x64']: 'rhel-70-64-bit',
166+
['win32-x64']: 'windows-test'
167+
};
168+
169+
const detectedPlatform = `${process.platform}-${process.arch}`;
170+
const prebuild = platformMatrix[detectedPlatform];
171+
if (prebuild == null) throw new Error(`Unsupported: ${detectedPlatform}`);
172+
173+
console.error(`Platform: ${detectedPlatform} Prebuild: ${prebuild}`);
174+
175+
const unzipArgs = ['-xzv', '-C', `_libmongocrypt-${ref}`, `${prebuild}/nocrypto`];
176+
console.error(`+ tar ${unzipArgs.join(' ')}`);
177+
const unzip = child_process.spawn('tar', unzipArgs, {
178+
stdio: ['pipe', 'inherit'],
179+
cwd: resolveRoot('.')
180+
});
181+
182+
const [response] = await events.once(https.get(downloadURL), 'response');
183+
184+
const start = performance.now();
185+
await stream.pipeline(response, unzip.stdin);
186+
const end = performance.now();
187+
188+
console.error(`downloaded libmongocrypt in ${(end - start) / 1000} secs...`);
189+
190+
await fs.rm(nodeDepsRoot, { recursive: true, force: true });
191+
await fs.cp(resolveRoot(destination, prebuild, 'nocrypto'), nodeDepsRoot, { recursive: true });
192+
const currentPath = path.join(nodeDepsRoot, 'lib64');
193+
try {
194+
await fs.rename(currentPath, path.join(nodeDepsRoot, 'lib'));
195+
} catch (error) {
196+
console.error(`error renaming ${currentPath}: ${error.message}`);
197+
}
198+
}
199+
200+
async function main() {
201+
const { libmongocrypt, build, clean } = await parseArguments();
202+
203+
const nodeDepsDir = resolveRoot('deps');
204+
205+
if (build) {
206+
const libmongocryptCloneDir = resolveRoot('_libmongocrypt');
207+
208+
const currentLibMongoCryptBranch = await fs
209+
.readFile(path.join(libmongocryptCloneDir, '.git', 'HEAD'), 'utf8')
210+
.catch(() => '');
211+
const isClonedAndCheckedOut = currentLibMongoCryptBranch
212+
.trim()
213+
.endsWith(`r-${libmongocrypt.ref}`);
214+
215+
if (clean || !isClonedAndCheckedOut) {
216+
await cloneLibMongoCrypt(libmongocryptCloneDir, libmongocrypt);
217+
}
218+
219+
const libmongocryptBuiltVersion = await fs
220+
.readFile(path.join(libmongocryptCloneDir, 'VERSION_CURRENT'), 'utf8')
221+
.catch(() => '');
222+
const isBuilt = libmongocryptBuiltVersion.trim() === libmongocrypt.ref;
223+
224+
if (clean || !isBuilt) {
225+
await buildLibMongoCrypt(libmongocryptCloneDir, nodeDepsDir);
226+
}
227+
} else {
228+
// Download
229+
await downloadLibMongoCrypt(nodeDepsDir, libmongocrypt);
230+
}
231+
232+
await fs.rm(resolveRoot('build'), { force: true, recursive: true });
233+
await fs.rm(resolveRoot('prebuilds'), { force: true, recursive: true });
234+
235+
// install with "ignore-scripts" so that we don't attempt to download a prebuild
236+
await run('npm', ['install', '--ignore-scripts']);
237+
// The prebuild command will make both a .node file in `./build` (local and CI testing will run on current code)
238+
// it will also produce `./prebuilds/mongodb-client-encryption-vVERSION-napi-vNAPI_VERSION-OS-ARCH.tar.gz`.
239+
await run('npm', ['run', 'prebuild']);
240+
}
241+
242+
await main();

.github/workflows/build.yml

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
on:
2+
push:
3+
branches: [main]
4+
pull_request:
5+
branches: [main]
6+
workflow_dispatch: {}
7+
8+
name: build
9+
10+
jobs:
11+
host_builds:
12+
strategy:
13+
matrix:
14+
os: [macos-11, macos-latest, windows-2019]
15+
runs-on: ${{ matrix.os }}
16+
steps:
17+
- uses: actions/checkout@v4
18+
19+
- name: Build ${{ matrix.os }} Prebuild
20+
run: node .github/scripts/libmongocrypt.mjs ${{ runner.os == 'Windows' && '--build' || '' }}
21+
shell: bash
22+
23+
- id: upload
24+
name: Upload prebuild
25+
uses: actions/upload-artifact@v4
26+
with:
27+
name: build-${{ matrix.os }}
28+
path: prebuilds/
29+
if-no-files-found: 'error'
30+
retention-days: 1
31+
compression-level: 0
32+
33+
container_builds:
34+
outputs:
35+
artifact_id: ${{ steps.upload.outputs.artifact-id }}
36+
runs-on: ubuntu-latest
37+
strategy:
38+
matrix:
39+
linux_arch: [s390x, arm64, amd64]
40+
steps:
41+
- uses: actions/checkout@v4
42+
43+
- name: Set up QEMU
44+
uses: docker/setup-qemu-action@v3
45+
46+
- name: Set up Docker Buildx
47+
uses: docker/setup-buildx-action@v3
48+
49+
- name: Run Buildx
50+
run: |
51+
docker buildx create --name builder --bootstrap --use
52+
docker buildx build --platform linux/${{ matrix.linux_arch }} --output type=local,dest=./prebuilds,platform-split=false -f ./.github/docker/Dockerfile.glibc .
53+
54+
- id: upload
55+
name: Upload prebuild
56+
uses: actions/upload-artifact@v4
57+
with:
58+
name: build-linux-${{ matrix.linux_arch }}
59+
path: prebuilds/
60+
if-no-files-found: 'error'
61+
retention-days: 1
62+
compression-level: 0
63+
64+
collect:
65+
needs: [host_builds, container_builds]
66+
runs-on: ubunutu-latest
67+
steps:
68+
- uses: actions/download-artifact@v4
69+
70+
- name: Display structure of downloaded files
71+
run: ls -R
72+
73+
- id: upload
74+
name: Upload all prebuilds
75+
uses: actions/upload-artifact@v4
76+
with:
77+
name: all-build
78+
path: '*.tar.gz'
79+
if-no-files-found: 'error'
80+
retention-days: 1
81+
compression-level: 0

0 commit comments

Comments
 (0)