Skip to content

Commit 241f57e

Browse files
kuhekuhe
committed
chore(scripts): stage oxc build dependencies
1 parent 7aa3c7a commit 241f57e

5 files changed

Lines changed: 768 additions & 0 deletions

File tree

‎package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@
117117
"kill-port": "^2.0.1",
118118
"lerna": "5.5.2",
119119
"lint-staged": "^10.0.1",
120+
"oxc-parser": "^0.138.0",
121+
"oxc-transform": "^0.138.0",
120122
"prettier": "2.8.5",
121123
"rimraf": "5.0.10",
122124
"ts-jest": "29.1.1",

‎scripts/compilation/build-es-batch.js‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
/**
2+
* Batch ES module transpilation using oxc-transform for multiple packages.
3+
*
4+
* Usage:
5+
* node scripts/compilation/build-es-batch.js client-s3 client-dynamodb core
6+
* node scripts/compilation/build-es-batch.js --all
7+
* node scripts/compilation/build-es-batch.js --concurrency 8 client-s3 client-dynamodb
8+
*/
9+
10+
const path = require("node:path");
11+
const fs = require("node:fs");
12+
const { transformSync } = require("oxc-transform");
13+
const { parseSync } = require("oxc-parser");
14+
const { listFolders } = require("../utils/list-folders");
15+
16+
const root = path.join(__dirname, "..", "..");
17+
18+
const args = process.argv.slice(2);
19+
const concurrency = (() => {
20+
const idx = args.indexOf("--concurrency");
21+
if (idx !== -1) {
22+
const val = parseInt(args[idx + 1], 10);
23+
args.splice(idx, 2);
24+
return val;
25+
}
26+
return 6;
27+
})();
28+
29+
const all = args.includes("--all");
30+
if (all) args.splice(args.indexOf("--all"), 1);
31+
32+
function getAllPackages() {
33+
const packages = [];
34+
for (const pkg of listFolders(path.join(root, "packages"))) packages.push(pkg);
35+
for (const pkg of listFolders(path.join(root, "packages-internal"))) packages.push(pkg);
36+
for (const lib of listFolders(path.join(root, "lib"))) packages.push(`lib-${lib.replace(/^lib-/, "")}`);
37+
for (const pkg of listFolders(path.join(root, "clients"))) packages.push(pkg);
38+
return packages;
39+
}
40+
41+
function resolvePackageDir(pkg) {
42+
const candidates = [
43+
path.join(root, "clients", pkg),
44+
path.join(root, "packages", pkg),
45+
path.join(root, "packages-internal", pkg),
46+
path.join(root, "lib", pkg),
47+
path.join(root, "lib", `lib-${pkg}`),
48+
];
49+
for (const dir of candidates) {
50+
if (fs.existsSync(dir)) return dir;
51+
}
52+
return null;
53+
}
54+
55+
/**
56+
* Remove comments using oxc-parser's AST comment positions.
57+
*/
58+
function stripComments(code) {
59+
const { comments } = parseSync("file.js", code);
60+
if (!comments.length) return code;
61+
let result = "";
62+
let last = 0;
63+
for (const { start, end } of comments) {
64+
result += code.slice(last, start);
65+
last = end;
66+
}
67+
result += code.slice(last);
68+
return result.replace(/^\s*\n/gm, "");
69+
}
70+
71+
function processDir(srcDir, outDir) {
72+
let count = 0;
73+
const entries = fs.readdirSync(srcDir, { withFileTypes: true });
74+
for (const entry of entries) {
75+
const fullPath = path.join(srcDir, entry.name);
76+
if (entry.isDirectory()) {
77+
count += processDir(fullPath, path.join(outDir, entry.name));
78+
} else if (
79+
entry.name.endsWith(".ts") &&
80+
!entry.name.endsWith(".d.ts") &&
81+
!entry.name.endsWith(".spec.ts") &&
82+
!entry.name.startsWith("vitest.")
83+
) {
84+
const outPath = path.join(outDir, entry.name.replace(/\.ts$/, ".js"));
85+
fs.mkdirSync(path.dirname(outPath), { recursive: true });
86+
const source = fs.readFileSync(fullPath, "utf-8");
87+
const { code, errors } = transformSync(fullPath, source, { sourcemap: false });
88+
if (errors.length) {
89+
console.error(`Errors in ${fullPath}:`, errors);
90+
process.exit(1);
91+
}
92+
fs.writeFileSync(outPath, stripComments(code));
93+
count++;
94+
}
95+
}
96+
return count;
97+
}
98+
99+
function buildPackage(pkg) {
100+
const packageDir = resolvePackageDir(pkg);
101+
if (!packageDir) {
102+
throw new Error(`Package not found: ${pkg}`);
103+
}
104+
const srcDir = path.join(packageDir, "src");
105+
const outDir = path.join(packageDir, "dist-es");
106+
107+
if (!fs.existsSync(srcDir)) {
108+
throw new Error(`No src/ directory in ${packageDir}`);
109+
}
110+
111+
// Clean dist-es
112+
fs.rmSync(outDir, { recursive: true, force: true });
113+
114+
const fileCount = processDir(srcDir, outDir);
115+
return fileCount;
116+
}
117+
118+
async function runBatch(packages, concurrency) {
119+
const total = packages.length;
120+
let completed = 0;
121+
const start = Date.now();
122+
123+
async function process(pkg) {
124+
const t0 = Date.now();
125+
const fileCount = buildPackage(pkg);
126+
completed++;
127+
console.log(`[${completed}/${total}] ${pkg} (${fileCount} files, ${Date.now() - t0}ms)`);
128+
}
129+
130+
// Process with bounded concurrency.
131+
const queue = [...packages];
132+
const workers = Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
133+
while (queue.length > 0) {
134+
const pkg = queue.shift();
135+
await process(pkg);
136+
}
137+
});
138+
139+
await Promise.all(workers);
140+
console.log(`\nDone: ${total} packages in ${((Date.now() - start) / 1000).toFixed(1)}s (concurrency=${concurrency})`);
141+
}
142+
143+
const packages = all ? getAllPackages() : args;
144+
145+
if (packages.length === 0) {
146+
console.error("Usage: node build-es-batch.js [--concurrency N] [--all] pkg1 pkg2 ...");
147+
process.exit(1);
148+
}
149+
150+
runBatch(packages, concurrency);

‎scripts/compilation/build-es.js‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/**
2+
* Fast ES module transpilation using oxc-transform.
3+
* Replaces `tsc -p tsconfig.es.json` (which uses noCheck, i.e. transpile-only).
4+
*
5+
* Usage: node ../../scripts/compilation/build-es.js
6+
* (run from a package directory)
7+
*/
8+
const { transformSync } = require("oxc-transform");
9+
const { parseSync } = require("oxc-parser");
10+
const { readdirSync, readFileSync, writeFileSync, mkdirSync } = require("node:fs");
11+
const path = require("node:path");
12+
13+
const packageDir = process.cwd();
14+
const srcDir = path.join(packageDir, "src");
15+
const outDir = path.join(packageDir, "dist-es");
16+
17+
/**
18+
* Remove comments using oxc-parser's AST comment positions.
19+
*/
20+
function stripComments(code) {
21+
const { comments } = parseSync("file.js", code);
22+
if (!comments.length) return code;
23+
let result = "";
24+
let last = 0;
25+
for (const { start, end } of comments) {
26+
result += code.slice(last, start);
27+
last = end;
28+
}
29+
result += code.slice(last);
30+
return result.replace(/^\s*\n/gm, "");
31+
}
32+
33+
function processDir(dir) {
34+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
35+
const fullPath = path.join(dir, entry.name);
36+
if (entry.isDirectory()) {
37+
processDir(fullPath);
38+
} else if (
39+
entry.name.endsWith(".ts") &&
40+
!entry.name.endsWith(".d.ts") &&
41+
!entry.name.endsWith(".spec.ts") &&
42+
!entry.name.startsWith("vitest.")
43+
) {
44+
const relPath = path.relative(srcDir, fullPath);
45+
const outPath = path.join(outDir, relPath.replace(/\.ts$/, ".js"));
46+
mkdirSync(path.dirname(outPath), { recursive: true });
47+
const source = readFileSync(fullPath, "utf-8");
48+
const { code, errors } = transformSync(fullPath, source, { sourcemap: false });
49+
if (errors.length) {
50+
console.error(`Errors in ${fullPath}:`, errors);
51+
process.exit(1);
52+
}
53+
writeFileSync(outPath, stripComments(code));
54+
}
55+
}
56+
}
57+
58+
processDir(srcDir);

‎scripts/compilation/inline-batch.js‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/**
2+
* Batch inline multiple packages in a single process.
3+
*
4+
* Usage:
5+
* node scripts/compilation/inline-batch.js client-s3 client-dynamodb core
6+
* node scripts/compilation/inline-batch.js --all
7+
* node scripts/compilation/inline-batch.js --concurrency 4 client-s3 client-dynamodb
8+
*/
9+
10+
const path = require("node:path");
11+
const Inliner = require("./Inliner");
12+
const { listFolders } = require("../utils/list-folders");
13+
14+
const root = path.join(__dirname, "..", "..");
15+
16+
const args = process.argv.slice(2);
17+
const concurrency = (() => {
18+
const idx = args.indexOf("--concurrency");
19+
if (idx !== -1) {
20+
const val = parseInt(args[idx + 1], 10);
21+
args.splice(idx, 2);
22+
return val;
23+
}
24+
return 6;
25+
})();
26+
27+
const all = args.includes("--all");
28+
if (all) args.splice(args.indexOf("--all"), 1);
29+
30+
function getAllPackages() {
31+
const packages = [];
32+
for (const pkg of listFolders(path.join(root, "packages"))) packages.push(pkg);
33+
for (const pkg of listFolders(path.join(root, "packages-internal"))) packages.push(pkg);
34+
for (const lib of ["dynamodb", "storage"]) packages.push(`lib-${lib}`);
35+
for (const pkg of listFolders(path.join(root, "clients"))) packages.push(pkg);
36+
return packages;
37+
}
38+
39+
async function runBatch(packages, concurrency) {
40+
const total = packages.length;
41+
let completed = 0;
42+
const start = Date.now();
43+
44+
async function process(pkg) {
45+
const t0 = Date.now();
46+
const inliner = new Inliner(pkg);
47+
await inliner.clean();
48+
await inliner.bundle();
49+
completed++;
50+
console.log(`[${completed}/${total}] ${pkg} (${Date.now() - t0}ms)`);
51+
}
52+
53+
// Process with bounded concurrency.
54+
const queue = [...packages];
55+
const workers = Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
56+
while (queue.length > 0) {
57+
const pkg = queue.shift();
58+
await process(pkg);
59+
}
60+
});
61+
62+
await Promise.all(workers);
63+
console.log(`\nDone: ${total} packages in ${((Date.now() - start) / 1000).toFixed(1)}s (concurrency=${concurrency})`);
64+
}
65+
66+
const packages = all ? getAllPackages() : args;
67+
68+
if (packages.length === 0) {
69+
console.error("Usage: node inline-batch.js [--concurrency N] [--all] pkg1 pkg2 ...");
70+
process.exit(1);
71+
}
72+
73+
runBatch(packages, concurrency);

0 commit comments

Comments
 (0)