From fc63b5699314f96fd541488001050b93d814fefe Mon Sep 17 00:00:00 2001 From: Alexis Williams Date: Thu, 23 Jul 2026 19:15:07 -0700 Subject: [PATCH 01/13] fix(bun2nix): dispatch lockfile entries on resolution, not tuple arity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bun does not keep entry kinds and tuple arities in one-to-one correspondence: github entries can carry an integrity hash (arity 4) and remote or vendored tarball entries can carry inline metadata (arity 3), so dispatching on arity misroutes both — a github dep lands in the npm parser and a pkg.pr.new tarball lands in the git parser, failing with MissingGitRef. Dispatch on the identifier's resolution instead (github:/git+/http(s)://, npm for bare versions at arity 4, file paths otherwise). The identifier is split at the '@' that ends the package name — the second '@' for scoped names — which also fixes resolutions that contain '@' themselves, like https://pkg.pr.new/@scope/pkg@sha tarball URLs. Vendored tarball paths with no file:/./ prefix (e.g. "vendor/pkg-1.0.0.tgz") are now accepted as file packages. --- .../src/lockfile/package_deserializer.rs | 161 +++++++++++------- 1 file changed, 104 insertions(+), 57 deletions(-) diff --git a/programs/bun2nix/src/lockfile/package_deserializer.rs b/programs/bun2nix/src/lockfile/package_deserializer.rs index ccc7108..28b8ec5 100644 --- a/programs/bun2nix/src/lockfile/package_deserializer.rs +++ b/programs/bun2nix/src/lockfile/package_deserializer.rs @@ -26,16 +26,41 @@ impl PackageDeserializer { /// # Deserialize package /// /// Deserialize a given package from it's lockfile representation + /// + /// Entries are dispatched on the identifier's resolution (the part after + /// the package name), not on tuple arity: bun emits github entries with + /// an integrity hash (arity 4) and remote/vendored tarball entries with + /// inline metadata (arity 3), so arity alone cannot tell entry kinds + /// apart. pub fn deserialize_package(name: String, values: Values) -> Result { let arity = values.len(); let deserializer = Self { name, values }; - match arity { - 1 => deserializer.deserialize_workspace_package(), - 2 => deserializer.deserialize_tarball_or_file_package(), - 3 => deserializer.deserialize_tarball_git_or_github_package(), - 4 => deserializer.deserialize_npm_package(), - x => Err(Error::UnexpectedPackageEntryLength(x)), + if arity == 1 { + return deserializer.deserialize_workspace_package(); + } + if !(2..=4).contains(&arity) { + return Err(Error::UnexpectedPackageEntryLength(arity)); + } + + let resolution = deserializer + .values + .first() + .and_then(|v| v.as_str()) + .map(str::to_owned) + .and_then(drain_package_specifier) + .ok_or(Error::NoAtInPackageIdentifier)?; + + if resolution.starts_with("github:") { + Self::deserialize_github_package(resolution) + } else if resolution.starts_with("git+") { + Self::deserialize_git_package(resolution) + } else if resolution.starts_with("http://") || resolution.starts_with("https://") { + Self::deserialize_tarball_package(resolution) + } else if arity == 4 { + deserializer.deserialize_npm_package() + } else { + Self::deserialize_file_package(deserializer.name, resolution) } } @@ -80,34 +105,10 @@ impl PackageDeserializer { Ok(Package::new(npm_identifier_raw, fetcher)) } - /// # Deserialize a Tarball, Git or Github Package - /// - /// Deserialize a tarball, git or github package from it's bun - /// lockfile representation - /// - /// These are grouped together as all three lockfile - /// representations are a tuple of arity 3, hence the - /// specifier prefix decides between them - `http` is a - /// tarball (bun records an integrity hash for these), `github:` - /// is a github package, and anything else is a git package - pub fn deserialize_tarball_git_or_github_package(mut self) -> Result { - let id = swap_remove_value(&mut self.values, 0); - let specifier = drain_package_specifier(id).ok_or(Error::NoAtInPackageIdentifier)?; - - if specifier.starts_with("http") { - Self::deserialize_tarball_package(specifier) - } else if specifier.starts_with("github:") { - Self::deserialize_github_package(specifier) - } else { - Self::deserialize_git_package(specifier) - } - } - /// # Deserialize a Github Package /// - /// Deserialize a github package from it's bun lockfile representation - /// - /// This is found in the source as a tuple of arity 3 + /// Deserialize a github package from its `github:owner/repo#rev` + /// resolution pub fn deserialize_github_package(id: String) -> Result { let (url, rev) = split_once_owned(id, '#').ok_or(Error::MissingGitRef)?; @@ -131,9 +132,7 @@ impl PackageDeserializer { /// # Deserialize a Git Package /// - /// Deserialize a git package from it's bun lockfile representation - /// - /// This is found in the source as a tuple of arity 3 + /// Deserialize a git package from its `git+#` resolution pub fn deserialize_git_package(id: String) -> Result { let git_url = drop_prefix(id, "git+"); let (url, rev) = split_once_owned(git_url, '#').ok_or(Error::MissingGitRef)?; @@ -152,26 +151,6 @@ impl PackageDeserializer { Ok(Package::new(id_with_rev, fetcher)) } - /// # Deserialize a tarball or file package - /// - /// Deserialize a tarball or file package from it's bun - /// lockfile representation - /// - /// These are grouped together as both lockfile - /// representations are a tupe of arity 2, hence - /// paths starting with `http` are considered - /// tarballs - pub fn deserialize_tarball_or_file_package(mut self) -> Result { - let id = swap_remove_value(&mut self.values, 0); - let path = drain_package_specifier(id).ok_or(Error::NoAtInPackageIdentifier)?; - - if path.starts_with("http") { - Self::deserialize_tarball_package(path) - } else { - Self::deserialize_file_package(self.name, path) - } - } - /// # Deserialize a file package /// /// Deserialize a file package from it's bun lockfile representation @@ -191,11 +170,13 @@ impl PackageDeserializer { "File path can never contain http, because then it would be a tarball" ); - // Strip prefix: explicit "file:" or implicit "./" (Bun strips file: for local tarballs) + // Strip prefix: explicit "file:" or implicit "./" (Bun strips file: for + // local tarballs). Vendored tarballs appear as bare relative paths + // (e.g. "vendor/pkg-1.0.0.tgz") with no prefix at all. let path = path .strip_prefix("file:") .or_else(|| path.strip_prefix("./")) - .ok_or(Error::MissingFileSpecifier)?; + .unwrap_or(&path); Ok(Package::new( name, @@ -364,3 +345,69 @@ pub fn drop_prefix(mut input: String, prefix: &str) -> String { input } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + const SHA: &str = "sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ=="; + + // A plain npm entry (arity 4, bare version resolution) still routes to the + // npm deserializer. + #[test] + fn npm_entry_dispatches_to_npm_package() { + let values = vec![ + json!("react-dom@19.2.7"), + json!(""), + json!({ "dependencies": { "scheduler": "^0.27.0" } }), + json!(SHA), + ]; + + let pkg = PackageDeserializer::deserialize_package("react-dom".into(), values).unwrap(); + assert!( + matches!(pkg.fetcher, Fetcher::FetchUrl { ref url, .. } + if url == "https://registry.npmjs.org/react-dom/-/react-dom-19.2.7.tgz"), + "expected FetchUrl, got {:?}", + pkg.fetcher + ); + } + + // Vendored tarballs are arity-3 entries whose resolution is a bare + // relative path: [id, meta, integrity]. + #[test] + fn vendored_tarball_dispatches_to_file_package() { + let values = vec![ + json!("@opencode-ai/client@vendor/opencode-ai-client-1.17.13.tgz"), + json!({}), + json!(SHA), + ]; + + let pkg = PackageDeserializer::deserialize_package( + "@opencode-ai/app/@opencode-ai/client".into(), + values, + ) + .unwrap(); + + assert!( + matches!(pkg.fetcher, Fetcher::CopyToStore { ref path } + if path == "vendor/opencode-ai-client-1.17.13.tgz"), + "expected CopyToStore, got {:?}", + pkg.fetcher + ); + } + + // file: and ./ prefixes are still stripped from file-package paths. + #[test] + fn prefixed_file_paths_are_stripped() { + for id in ["local-pkg@file:local/pkg.tgz", "local-pkg@./local/pkg.tgz"] { + let values = vec![json!(id), json!(SHA)]; + let pkg = PackageDeserializer::deserialize_package("local-pkg".into(), values).unwrap(); + assert!( + matches!(pkg.fetcher, Fetcher::CopyToStore { ref path } if path == "local/pkg.tgz"), + "expected stripped CopyToStore for {id}, got {:?}", + pkg.fetcher + ); + } + } +} From aab1304b92ab5bd5dc4a1a5fc6ba108d166509da Mon Sep 17 00:00:00 2001 From: Alexis Williams Date: Thu, 23 Jul 2026 19:15:15 -0700 Subject: [PATCH 02/13] fix(bun2nix): resolve nested file-dep paths against their workspace dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A "/" lockfile entry records its file-dependency path relative to that workspace's directory, but bun.nix paths resolve from the project root — so a vendored tarball under packages/app rendered as ./vendor/pkg.tgz (missing) and a sibling workspace's ../app/vendor/pkg.tgz escaped the root entirely. Factor package building out of convert_lockfile_to_nix_expression into build_packages, and rewrite CopyToStore paths for entries nested under a workspace (longest workspace-name prefix wins) with lexical ././.. normalization. --- programs/bun2nix/src/lib.rs | 106 +++++++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 1 deletion(-) diff --git a/programs/bun2nix/src/lib.rs b/programs/bun2nix/src/lib.rs index c4df1ba..ef23d44 100644 --- a/programs/bun2nix/src/lib.rs +++ b/programs/bun2nix/src/lib.rs @@ -24,15 +24,119 @@ use wasm_bindgen::prelude::*; #[cfg_attr(target_arch = "wasm32", wasm_bindgen)] #[cfg_attr(target_arch = "wasm32", no_mangle)] pub fn convert_lockfile_to_nix_expression(contents: String, options: Options) -> Result { + let packages = build_packages(&contents)?; + + NixExpression::new(packages)?.render_with_options(options) +} + +/// # Build Packages from a Lockfile +/// +/// Parses a bun lockfile and produces the sorted, de-duplicated list of +/// [`Package`]s it describes. +pub fn build_packages(contents: &str) -> Result> { let lockfile = contents.parse::()?; if lockfile.lockfile_version != 1 { return Err(Error::UnsupportedLockfileVersion(lockfile.lockfile_version)); }; + // Workspace name → directory, for resolving nested file-dependency paths: + // bun records a `/` entry's path relative to that + // workspace's directory, but `bun.nix` paths resolve from the project root. + let workspace_dirs: Vec<(String, String)> = lockfile + .workspaces + .iter() + .filter_map(|(dir, ws)| ws.name.clone().map(|name| (name, dir.clone()))) + .collect(); + let mut packages = lockfile.packages(); packages.sort(); packages.dedup_by(|a, b| a.name == b.name); - NixExpression::new(packages)?.render_with_options(options) + for package in &mut packages { + if let package::Fetcher::CopyToStore { path } = &mut package.fetcher { + // Longest matching `/` prefix wins; entries whose + // key IS a workspace name (the workspaces themselves) don't match. + let parent_dir = workspace_dirs + .iter() + .filter(|(name, _)| { + !name.is_empty() + && package.name.len() > name.len() + 1 + && package.name.starts_with(name.as_str()) + && package.name.as_bytes()[name.len()] == b'/' + }) + .max_by_key(|(name, _)| name.len()) + .map(|(_, dir)| dir.as_str()); + if let Some(dir) = parent_dir + && !dir.is_empty() + { + *path = normalize_path(&format!("{dir}/{path}")); + } + } + } + + Ok(packages) +} + +/// Collapse `.` and `..` segments lexically (`a/b/../c` → `a/c`). +fn normalize_path(path: &str) -> String { + let mut parts: Vec<&str> = Vec::new(); + for seg in path.split('/') { + match seg { + "" | "." => {} + ".." => { + if parts.last().is_none_or(|last| *last == "..") { + parts.push(".."); + } else { + parts.pop(); + } + } + s => parts.push(s), + } + } + parts.join("/") +} + +#[cfg(test)] +mod tests { + use super::*; + + // A vendored tarball nested under a workspace ("/") is + // recorded relative to the workspace dir; bun.nix needs it root-relative. + #[test] + fn nested_file_dep_paths_resolve_against_workspace_dir() { + let lock = r#"{ + "lockfileVersion": 1, + "workspaces": { + "": { "name": "root" }, + "packages/app": { "name": "@oc/app" }, + "packages/ui": { "name": "@oc/ui" } + }, + "packages": { + "@oc/app": ["@oc/app@workspace:packages/app"], + "@oc/ui": ["@oc/ui@workspace:packages/ui"], + "@oc/app/@oc/client": ["@oc/client@vendor/client-1.0.0.tgz", {}, "sha512-AAAA"], + "@oc/ui/@oc/client": ["@oc/client@../app/vendor/client-1.0.0.tgz", {}, "sha512-AAAA"], + } +}"#; + let pkgs = build_packages(lock).unwrap(); + let path_of = |name: &str| { + let p = pkgs.iter().find(|p| p.name == name).unwrap(); + match &p.fetcher { + package::Fetcher::CopyToStore { path } => path.clone(), + other => panic!("expected CopyToStore, got {other:?}"), + } + }; + + assert_eq!( + path_of("@oc/app/@oc/client"), + "packages/app/vendor/client-1.0.0.tgz" + ); + assert_eq!( + path_of("@oc/ui/@oc/client"), + "packages/app/vendor/client-1.0.0.tgz" + ); + // Workspace entries themselves stay untouched. + assert_eq!(path_of("@oc/app"), "packages/app"); + } } From 493276d74a43df4374354af55548fc1892f37b94 Mon Sep 17 00:00:00 2001 From: Alexis Williams Date: Thu, 23 Jul 2026 20:19:43 -0700 Subject: [PATCH 03/13] fix(nix): fail fast on lockfile drift instead of letting bun hit the network MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two hook fixes for failure modes found installing opencode offline. Both stem from the same bun behavior: any drift between package.json and bun.lock makes bun re-resolve the affected dependencies, and re-resolving a git/github/remote-tarball dependency downloads it unconditionally — fatal in the sandbox no matter how complete the cache is. - Detect trustedDependencies / patchedDependencies drift between the root package.json and bun.lock and fail with an actionable message (refresh bun.lock, regenerate bun.nix). Projects routinely commit a lockfile whose copies of these sections lag package.json (opencode's does); without the check the drift surfaces as a wall of ConnectionRefused errors at resolve time with no hint of the cause. - Leave catalog: references pointing at non-registry specs (github:, git+, tarball URLs, file:) unrewritten. bun resolves those natively from the lockfile's catalog section; rewriting them to their resolution registered as a changed spec and forced a re-resolve. The hook now runs the prep script whenever bun.lock exists (previously only when it contained catalog: refs), so the drift check covers every project. --- nix/mk-derivation/hook.sh | 18 ++-- nix/mk-derivation/resolve-catalog.ts | 127 ++++++++++++++++++++++----- 2 files changed, 113 insertions(+), 32 deletions(-) diff --git a/nix/mk-derivation/hook.sh b/nix/mk-derivation/hook.sh index 3e4f7f1..01f41c6 100644 --- a/nix/mk-derivation/hook.sh +++ b/nix/mk-derivation/hook.sh @@ -83,14 +83,14 @@ function bunPatchPhase { runHook postBunPatchPhase } -# bun re-resolves `catalog:` dependency specifiers against the npm registry on -# every `bun install`, even with a fully populated cache. In the Nix sandbox -# this fails. The lockfile already records the exact resolved version for -# every package, so rewrite every `catalog:` reference (in bun.lock's -# `workspaces` section and in every workspace package.json) to that exact -# version (or `workspace:*` for workspace packages) before `bun install`. -function bunResolveCatalogRefs { - if ! [ -f bun.lock ] || ! grep -q '"catalog:' bun.lock 2>/dev/null; then +# Prepare the project for offline install: rewrite `catalog:` references to +# the exact versions recorded in bun.lock (bun re-resolves them against the +# registry otherwise, which fails in the sandbox), and fail fast if bun.lock +# has drifted from package.json (drift makes bun re-resolve, with the same +# result). Runs whenever a lockfile exists — the drift check applies to every +# project, not just those using catalogs. +function bunPrepareOfflineInstall { + if ! [ -f bun.lock ]; then return 0 fi # --config=/dev/null: ignore the project's bunfig.toml, which may remap @@ -103,7 +103,7 @@ function bunNodeModulesInstallPhase { pushd "$bunRoot" || exit 1 runHook preBunNodeModulesInstallPhase - bunResolveCatalogRefs + bunPrepareOfflineInstall # Remove patchedDependencies from package.json and bun.lock since we # pre-patch packages during the Nix build. This ensures bun looks for diff --git a/nix/mk-derivation/resolve-catalog.ts b/nix/mk-derivation/resolve-catalog.ts index fd44293..9421073 100644 --- a/nix/mk-derivation/resolve-catalog.ts +++ b/nix/mk-derivation/resolve-catalog.ts @@ -1,12 +1,21 @@ -// bun2nix: resolve `catalog:` specifiers to exact versions for offline install. +// bun2nix: prepare the project for offline install. // -// bun re-resolves `catalog:` dependency specifiers against the npm registry on -// every `bun install`, even with a fully populated cache and -// `--frozen-lockfile` / `--offline`. In the Nix sandbox this fails. The -// lockfile already records the exact resolved version for every package, so -// rewrite every `catalog:` reference (in bun.lock's `workspaces` section and -// in every workspace package.json) to that exact version (or `workspace:*` -// for workspace packages) before `bun install` runs. +// Both steps exist because bun re-resolves any dependency whose recorded +// state drifted from package.json — and re-resolving git/github/ +// remote-tarball deps downloads them unconditionally, which fails in the Nix +// sandbox no matter how complete the cache is: +// +// 1. Resolve `catalog:` specifiers to exact versions (older bun re-resolves +// them against the registry on every `bun install`). Non-registry catalog +// values (github:/git+/tarball URLs) are left as `catalog:` — bun resolves +// those natively from the lockfile's catalog section, and rewriting them +// would itself register as a changed spec and force a re-resolve. +// 2. Detect `trustedDependencies` / `patchedDependencies` drift between the +// root package.json and bun.lock, and fail with an actionable message. +// Projects routinely commit a bun.lock whose copies of these sections lag +// package.json; any mismatch makes bun distrust the lockfile mapping +// wholesale, and the resulting re-resolution surfaces as a wall of +// ConnectionRefused errors long after the actual cause. // // Invoked as: bun resolve-catalog.ts @@ -29,6 +38,8 @@ interface BunLock { catalog?: Deps; catalogs?: Record; packages?: Record; + trustedDependencies?: string[]; + patchedDependencies?: Deps; } const depSections = [ @@ -42,7 +53,7 @@ const root = process.argv[2] ?? "."; const lockPath = join(root, "bun.lock"); if (!existsSync(lockPath)) process.exit(0); -if (!readFileSync(lockPath, "utf8").includes('"catalog:')) process.exit(0); +const hasCatalogRefs = readFileSync(lockPath, "utf8").includes('"catalog:'); // bun.lock is JSON-with-trailing-commas. Bun's module loader has a built-in // JSONC parser (used for tsconfig.json / bun.lock) that we can reach via @@ -71,6 +82,20 @@ for (const [name, entry] of Object.entries(packages)) { resolved[name] = spec.slice(prefix.length); } +// A spec whose resolution is not a plain registry version. Rewriting a +// `catalog:` reference to one of these would change the dependency's spec +// string and force bun to re-resolve (= re-download) it; bun resolves these +// natively from the lockfile's catalog section, so leave them alone. +function isNonRegistrySpec(spec: string): boolean { + return ( + spec.startsWith("github:") || + spec.startsWith("git+") || + spec.startsWith("http://") || + spec.startsWith("https://") || + spec.startsWith("file:") + ); +} + function cresolve(name: string, spec: string): string { const cname = spec.slice("catalog:".length); const table = cname === "" ? catalog : (catalogs[cname] ?? {}); @@ -79,6 +104,8 @@ function cresolve(name: string, spec: string): string { if (typeof cv === "string" && cv.startsWith("workspace:")) return cv; if (typeof rv === "string" && rv.startsWith("workspace:")) return "workspace:*"; + if (typeof cv === "string" && isNonRegistrySpec(cv)) return spec; + if (typeof rv === "string" && isNonRegistrySpec(rv)) return spec; if (typeof rv === "string") return rv; if (typeof cv === "string") return cv; return spec; @@ -99,25 +126,79 @@ function rewriteDeps(holder: DepHolder): boolean { return changed; } -console.log("bun2nix: resolving catalog: specifiers from bun.lock"); - -// Rewrite the lockfile's workspaces section. let lockChanged = false; -for (const ws of Object.values(workspaces)) { - if (rewriteDeps(ws)) lockChanged = true; + +if (hasCatalogRefs) { + console.log("bun2nix: resolving catalog: specifiers from bun.lock"); + + // Rewrite the lockfile's workspaces section. + for (const ws of Object.values(workspaces)) { + if (rewriteDeps(ws)) lockChanged = true; + } + + // Rewrite every workspace package.json (root "" + each workspace dir). + for (const wsDir of Object.keys(workspaces)) { + const pkgJson = join(root, wsDir, "package.json"); + if (!existsSync(pkgJson)) continue; + const text = readFileSync(pkgJson, "utf8"); + if (!text.includes('"catalog:')) continue; + const pkg = JSON.parse(text) as DepHolder; + if (rewriteDeps(pkg)) { + writeFileSync(pkgJson, JSON.stringify(pkg, null, 2) + "\n"); + } + } } + if (lockChanged) { writeFileSync(lockPath, JSON.stringify(lock, null, 2) + "\n"); } -// Rewrite every workspace package.json (root "" + each workspace dir). -for (const wsDir of Object.keys(workspaces)) { - const pkgJson = join(root, wsDir, "package.json"); - if (!existsSync(pkgJson)) continue; - const text = readFileSync(pkgJson, "utf8"); - if (!text.includes('"catalog:')) continue; - const pkg = JSON.parse(text) as DepHolder; - if (rewriteDeps(pkg)) { - writeFileSync(pkgJson, JSON.stringify(pkg, null, 2) + "\n"); +// Fail fast on trustedDependencies / patchedDependencies drift between the +// root package.json and bun.lock. Compared as a set / as key-value pairs so +// pure ordering differences don't trip the check. +const rootPkgPath = join(root, "package.json"); +if (existsSync(rootPkgPath)) { + const rootPkg = JSON.parse(readFileSync(rootPkgPath, "utf8")) as { + trustedDependencies?: string[]; + patchedDependencies?: Deps; + }; + + const drift: string[] = []; + + const pkgTrusted = [...(rootPkg.trustedDependencies ?? [])].sort(); + const lockTrusted = [...(lock.trustedDependencies ?? [])].sort(); + if (JSON.stringify(pkgTrusted) !== JSON.stringify(lockTrusted)) { + const missing = pkgTrusted.filter((n) => !lockTrusted.includes(n)); + const extra = lockTrusted.filter((n) => !pkgTrusted.includes(n)); + drift.push( + `trustedDependencies differ` + + (missing.length + ? `; missing from bun.lock: ${missing.join(", ")}` + : "") + + (extra.length ? `; only in bun.lock: ${extra.join(", ")}` : ""), + ); + } + + const pkgPatched = rootPkg.patchedDependencies ?? {}; + const lockPatched = lock.patchedDependencies ?? {}; + const patchKeys = [ + ...new Set([...Object.keys(pkgPatched), ...Object.keys(lockPatched)]), + ].sort(); + const patchDiffs = patchKeys.filter((k) => pkgPatched[k] !== lockPatched[k]); + if (patchDiffs.length) { + drift.push(`patchedDependencies differ for: ${patchDiffs.join(", ")}`); + } + + if (drift.length) { + console.error(` +bun2nix: error: bun.lock is out of sync with package.json: +${drift.map((d) => ` - ${d}`).join("\n")} + +bun re-resolves dependencies when these sections drift, and re-resolving +git/github/tarball dependencies requires network access, which is not +available in the Nix sandbox. Run \`bun install\` to refresh bun.lock, +commit the result, and regenerate bun.nix. +`); + process.exit(1); } } From 947db0cf334cc73ec166e7fb90524fbaa7d2744e Mon Sep 17 00:00:00 2001 From: Alexis Williams Date: Sat, 25 Jul 2026 17:26:00 -0700 Subject: [PATCH 04/13] test(nix): guard lockfile drift detection and catalog rewrite behavior Runs resolve-catalog.ts against three inline fixtures: drifted trustedDependencies/patchedDependencies must fail with a diagnostic naming the drifted entries, identical-but-reordered sections must pass, and catalog: refs must be rewritten to exact versions except when they resolve to non-registry specs. The drifted fixture has no catalog: refs, pinning that the check runs for plain projects too. --- nix/checks/lockfile-drift-detection.nix | 161 ++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 nix/checks/lockfile-drift-detection.nix diff --git a/nix/checks/lockfile-drift-detection.nix b/nix/checks/lockfile-drift-detection.nix new file mode 100644 index 0000000..a2d7c68 --- /dev/null +++ b/nix/checks/lockfile-drift-detection.nix @@ -0,0 +1,161 @@ +# Guards the hook's offline-install prep script (resolve-catalog.ts): +# +# 1. A bun.lock whose trustedDependencies / patchedDependencies drifted from +# package.json must fail fast with a diagnostic naming the drifted +# entries. Without the check, bun silently re-resolves the affected +# dependencies at install time and the drift surfaces as a wall of +# ConnectionRefused errors in the sandbox with no hint of the cause. +# 2. Sections that merely list the same entries in a different order must +# NOT trip the check. +# 3. `catalog:` references resolving to registry versions are rewritten to +# the exact version, but references resolving to non-registry specs +# (github:, git+, tarball URLs, file:) must be left alone — rewriting +# them registers as a changed spec and forces the same re-resolve. +# +# The drifted fixture deliberately contains no `catalog:` refs, pinning that +# the drift check runs for plain projects too, not only catalog users. +_: { + perSystem = + { pkgs, ... }: + { + checks.lockfileDriftDetection = pkgs.stdenv.mkDerivation { + name = "lockfile-drift-detection"; + + dontUnpack = true; + + nativeBuildInputs = [ pkgs.bun ]; + + buildPhase = '' + export HOME="$TMPDIR" + script=${../mk-derivation/resolve-catalog.ts} + + run() { + rc=0 + bun --config=/dev/null --no-install "$script" "$1" >log.txt 2>&1 || rc=$? + } + + expect() { + if ! grep -qF "$1" log.txt; then + echo "expected output to contain: $1" + cat log.txt + exit 1 + fi + } + + # --- drifted: both sections disagree with package.json --- + mkdir drifted + cat > drifted/package.json <<'EOF' + { + "name": "fixture", + "version": "1.0.0", + "trustedDependencies": ["node-pty"], + "patchedDependencies": { "left-pad@1.3.0": "patches/left-pad.patch" } + } + EOF + cat > drifted/bun.lock <<'EOF' + { + "lockfileVersion": 1, + "workspaces": { "": { "name": "fixture" } }, + "trustedDependencies": ["esbuild"], + "packages": {} + } + EOF + + run drifted + if [ "$rc" -eq 0 ]; then + echo "drifted fixture: expected failure, got exit 0" + cat log.txt + exit 1 + fi + expect "bun.lock is out of sync with package.json" + expect "missing from bun.lock: node-pty" + expect "only in bun.lock: esbuild" + expect "patchedDependencies differ for: left-pad@1.3.0" + echo "drifted fixture: failed with diagnostic, as intended" + + # --- synced: same entries, different order --- + mkdir synced + cat > synced/package.json <<'EOF' + { + "name": "fixture", + "version": "1.0.0", + "trustedDependencies": ["b-pkg", "a-pkg"], + "patchedDependencies": { "x@1.0.0": "patches/x.patch" } + } + EOF + cat > synced/bun.lock <<'EOF' + { + "lockfileVersion": 1, + "workspaces": { "": { "name": "fixture" } }, + "trustedDependencies": ["a-pkg", "b-pkg"], + "patchedDependencies": { "x@1.0.0": "patches/x.patch" }, + "packages": {} + } + EOF + + run synced + if [ "$rc" -ne 0 ]; then + echo "synced fixture: expected success, got exit $rc" + cat log.txt + exit 1 + fi + echo "synced fixture: passed, ordering ignored" + + # --- catalog: registry ref rewritten, non-registry ref preserved --- + mkdir catalog + cat > catalog/package.json <<'EOF' + { + "name": "fixture", + "version": "1.0.0", + "dependencies": { + "bar": "catalog:", + "foo": "catalog:" + } + } + EOF + cat > catalog/bun.lock <<'EOF' + { + "lockfileVersion": 1, + "workspaces": { + "": { + "name": "fixture", + "dependencies": { "bar": "catalog:", "foo": "catalog:" } + } + }, + "catalog": { + "bar": "^1.0.0", + "foo": "github:user/repo#abcdef" + }, + "packages": { + "bar": ["bar@1.2.3", "", {}, "sha512-aaa"], + "foo": ["foo@github:user/repo#abcdef", {}, "abcdef"] + } + } + EOF + + run catalog + if [ "$rc" -ne 0 ]; then + echo "catalog fixture: expected success, got exit $rc" + cat log.txt + exit 1 + fi + if ! grep -qF '"bar": "1.2.3"' catalog/package.json; then + echo "catalog fixture: registry ref not rewritten to exact version" + cat catalog/package.json + exit 1 + fi + if ! grep -qF '"foo": "catalog:"' catalog/package.json; then + echo "catalog fixture: non-registry ref was rewritten; must stay catalog:" + cat catalog/package.json + exit 1 + fi + echo "catalog fixture: rewrite behavior correct" + ''; + + installPhase = '' + mkdir -p "$out" + echo "lockfileDriftDetection: PASS" > "$out/result" + ''; + }; + }; +} From 1f14bda91ed66dd1f225acb690cfe00f0be32c5d Mon Sep 17 00:00:00 2001 From: Alexis Williams Date: Thu, 23 Jul 2026 18:12:51 -0700 Subject: [PATCH 05/13] feat(core): cargo workspace with bun2nix-core and Rust cache-entry-creator Introduce a programs/ cargo workspace with a shared bun2nix-core crate: vendored bun Wyhash11, the bun cache-folder-name port, and verbatim structs + serializer + multi-version builder for bun's binary .npm manifest cache format (bun-npm-manifest-cache-v0.0.7). Rewrite cache-entry-creator from Zig to Rust on top of it, keeping the symlink behavior and adding a manifest mode that turns EntryMeta JSON into .npm cache files. Nix package builds updated to build from the workspace. --- nix/bun2nix.nix | 12 +- nix/bun2nix/bun2nix-js.nix | 10 +- nix/cargo-toml.nix | 12 +- nix/fetch-bun-deps/build-package.nix | 2 +- nix/fetch-bun-deps/cache-entry-creator.nix | 58 +- programs/{bun2nix => }/Cargo.lock | 221 +++++--- programs/Cargo.toml | 16 + programs/bun2nix-core/Cargo.toml | 9 + programs/bun2nix-core/src/cache_name.rs | 250 +++++++++ programs/bun2nix-core/src/lib.rs | 3 + programs/bun2nix-core/src/manifest/build.rs | 400 +++++++++++++ programs/bun2nix-core/src/manifest/layout.rs | 528 ++++++++++++++++++ programs/bun2nix-core/src/manifest/meta.rs | 69 +++ programs/bun2nix-core/src/manifest/mod.rs | 441 +++++++++++++++ .../bun2nix-core/src/manifest/serialize.rs | 125 +++++ programs/bun2nix-core/src/wyhash.rs | 416 ++++++++++++++ programs/bun2nix-core/tests/fixtures/ms.json | 31 + .../tests/fixtures/neoconfetti.json | 20 + programs/bun2nix-core/tests/golden.rs | 294 ++++++++++ programs/bun2nix/Cargo.toml | 9 +- programs/cache-entry-creator/Cargo.toml | 14 + programs/cache-entry-creator/build.zig | 40 -- programs/cache-entry-creator/build.zig.zon | 48 -- programs/cache-entry-creator/deps.nix | 14 - programs/cache-entry-creator/src/main.rs | 344 ++++++++++++ programs/cache-entry-creator/src/main.zig | 355 ------------ programs/cache-entry-creator/src/wyhash.zig | 181 ------ 27 files changed, 3171 insertions(+), 751 deletions(-) rename programs/{bun2nix => }/Cargo.lock (69%) create mode 100644 programs/Cargo.toml create mode 100644 programs/bun2nix-core/Cargo.toml create mode 100644 programs/bun2nix-core/src/cache_name.rs create mode 100644 programs/bun2nix-core/src/lib.rs create mode 100644 programs/bun2nix-core/src/manifest/build.rs create mode 100644 programs/bun2nix-core/src/manifest/layout.rs create mode 100644 programs/bun2nix-core/src/manifest/meta.rs create mode 100644 programs/bun2nix-core/src/manifest/mod.rs create mode 100644 programs/bun2nix-core/src/manifest/serialize.rs create mode 100644 programs/bun2nix-core/src/wyhash.rs create mode 100644 programs/bun2nix-core/tests/fixtures/ms.json create mode 100644 programs/bun2nix-core/tests/fixtures/neoconfetti.json create mode 100644 programs/bun2nix-core/tests/golden.rs create mode 100644 programs/cache-entry-creator/Cargo.toml delete mode 100644 programs/cache-entry-creator/build.zig delete mode 100644 programs/cache-entry-creator/build.zig.zon delete mode 100644 programs/cache-entry-creator/deps.nix create mode 100644 programs/cache-entry-creator/src/main.rs delete mode 100644 programs/cache-entry-creator/src/main.zig delete mode 100644 programs/cache-entry-creator/src/wyhash.zig diff --git a/nix/bun2nix.nix b/nix/bun2nix.nix index ca2f82b..5b554d2 100644 --- a/nix/bun2nix.nix +++ b/nix/bun2nix.nix @@ -22,12 +22,22 @@ in pname = pkgInfo.name; inherit (pkgInfo) version; - src = ../programs/bun2nix; + src = ../programs; cargoLock = { lockFile = finalAttrs.src + "/Cargo.lock"; }; + cargoBuildFlags = [ + "-p" + "bun2nix" + ]; + + cargoTestFlags = [ + "-p" + "bun2nix" + ]; + passthru = with config; { inherit (mkDerivation) hook; inherit writeBunScriptBin writeBunApplication; diff --git a/nix/bun2nix/bun2nix-js.nix b/nix/bun2nix/bun2nix-js.nix index d02b90d..ef6a205 100644 --- a/nix/bun2nix/bun2nix-js.nix +++ b/nix/bun2nix/bun2nix-js.nix @@ -11,14 +11,19 @@ pname = "bun2nix-js"; inherit (config.cargoTOML.package) version; - src = ../../programs/bun2nix; + src = ../../programs; cargoLock = { lockFile = finalAttrs.src + "/Cargo.lock"; }; + # The `bun2nix` crate path-depends on the sibling `bun2nix-core` + # crate, so the build source must be the workspace root. The actual + # wasm/js build still runs inside the `bun2nix` crate directory. + bunRoot = "bun2nix"; + bunDeps = final.bun2nix.fetchBunDeps { - bunNix = finalAttrs.src + "/bun.nix"; + bunNix = finalAttrs.src + "/bun2nix/bun.nix"; }; nativeBuildInputs = with final; [ @@ -34,6 +39,7 @@ buildPhase = '' runHook preBuild + cd "$bunRoot" bun run build runHook postBuild diff --git a/nix/cargo-toml.nix b/nix/cargo-toml.nix index 471167b..3118652 100644 --- a/nix/cargo-toml.nix +++ b/nix/cargo-toml.nix @@ -8,5 +8,15 @@ in type = types.raw; }; - config.cargoTOML = builtins.fromTOML (builtins.readFile "${self}/programs/bun2nix/Cargo.toml"); + config.cargoTOML = + let + crate = builtins.fromTOML (builtins.readFile "${self}/programs/bun2nix/Cargo.toml"); + ws = builtins.fromTOML (builtins.readFile "${self}/programs/Cargo.toml"); + in + crate + // { + package = crate.package // { + inherit (ws.workspace.package) version; + }; + }; } diff --git a/nix/fetch-bun-deps/build-package.nix b/nix/fetch-bun-deps/build-package.nix index bcad311..1b1d7e7 100644 --- a/nix/fetch-bun-deps/build-package.nix +++ b/nix/fetch-bun-deps/build-package.nix @@ -136,7 +136,7 @@ in cacheEntryPhase = '' runHook preCacheEntry - "${lib.getExe self'.packages.cacheEntryCreator}" \ + "${lib.getExe self'.packages.cacheEntryCreator}" symlink \ --out "$out/share/bun-cache" \ --name "${name}" \ --package "$out/share/bun-packages/${name}" \ diff --git a/nix/fetch-bun-deps/cache-entry-creator.nix b/nix/fetch-bun-deps/cache-entry-creator.nix index 7df9db9..22b1aca 100644 --- a/nix/fetch-bun-deps/cache-entry-creator.nix +++ b/nix/fetch-bun-deps/cache-entry-creator.nix @@ -3,43 +3,39 @@ perSystem = { pkgs, ... }: { - packages.cacheEntryCreator = pkgs.stdenvNoCC.mkDerivation ( - finalAttrs: - let - depsNix = finalAttrs.src + "/deps.nix"; - in - { - pname = "bun2nix-cache-entry-creator"; - inherit (config.cargoTOML.package) version; + packages.cacheEntryCreator = pkgs.rustPlatform.buildRustPackage (finalAttrs: { + pname = "bun2nix-cache-entry-creator"; + inherit (config.cargoTOML.package) version; - src = ../../programs/cache-entry-creator; + src = ../../programs; - nativeBuildInputs = with pkgs; [ - zig_0_15.hook - ]; + cargoLock = { + lockFile = finalAttrs.src + "/Cargo.lock"; + }; - postConfigure = '' - ln -s ${pkgs.callPackage depsNix { }} $ZIG_GLOBAL_CACHE_DIR/p - ''; + cargoBuildFlags = [ + "-p" + "cache-entry-creator" + ]; - zigBuildFlags = [ - "--release=fast" - ]; + cargoTestFlags = [ + "-p" + "cache-entry-creator" + ]; - doCheck = true; + doCheck = true; - meta = { - description = "Cache entry creator for bun packages"; - longDescription = '' - Uses bun's specific `wyhash` implementation to calculate - the correct location in which to place a cache entry for - a given package after the tarball has been downloaded and - extracted. - ''; - mainProgram = "cache_entry_creator"; - }; - } - ); + meta = { + description = "Cache entry creator for bun packages"; + longDescription = '' + Uses bun's specific `wyhash` implementation to calculate + the correct location in which to place a cache entry for + a given package after the tarball has been downloaded and + extracted. + ''; + mainProgram = "cache_entry_creator"; + }; + }); }; } diff --git a/programs/bun2nix/Cargo.lock b/programs/Cargo.lock similarity index 69% rename from programs/bun2nix/Cargo.lock rename to programs/Cargo.lock index 39a4ea3..be7c1b6 100644 --- a/programs/bun2nix/Cargo.lock +++ b/programs/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -28,15 +28,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -112,17 +112,24 @@ dependencies = [ "serde", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bumpalo" -version = "3.19.0" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bun2nix" version = "2.1.2" dependencies = [ "askama", + "bun2nix-core", "cfg-if", "clap", "env_logger", @@ -135,6 +142,23 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "bun2nix-core" +version = "2.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "cache-entry-creator" +version = "2.1.0" +dependencies = [ + "bun2nix-core", + "clap", + "serde_json", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -143,9 +167,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "clap" -version = "4.5.53" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -153,9 +177,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.53" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", @@ -165,9 +189,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.49" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck", "proc-macro2", @@ -177,27 +201,59 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.6" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "colorchoice" -version = "1.0.4" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "defmt" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e524506490a1953d237cb87b1cfc1e46f88c18f10a22dfe0f507dc6bfc7f7f" +dependencies = [ + "bitflags", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a27770e9c8f719a79d8b638281f4d828f77d8fd61e0bd94451b9b85e576a0b" +dependencies = [ + "defmt-parser", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror", +] [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "env_filter" -version = "0.1.4" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" dependencies = [ "log", "regex", @@ -205,9 +261,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.8" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" dependencies = [ "anstream", "anstyle", @@ -239,16 +295,17 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.15" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.16" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35" +checksum = "34f877a98676d2fb664698d74cc6a51ce6c484ce8c770f05d0108ec9090aeb46" dependencies = [ + "defmt", "jiff-static", "log", "portable-atomic", @@ -258,9 +315,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.16" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" +checksum = "0666b5ab5ecaca213fc2a85b8c0083d9004e84ee2d5f9a7e0017aaf50986f25f" dependencies = [ "proc-macro2", "quote", @@ -278,21 +335,21 @@ dependencies = [ [[package]] name = "log" -version = "0.4.28" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -308,42 +365,64 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "portable-atomic" -version = "1.11.1" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.4" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ "portable-atomic", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.42" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] [[package]] name = "regex" -version = "1.12.2" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -353,9 +432,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -364,15 +443,15 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "rustversion" @@ -380,12 +459,6 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" -[[package]] -name = "ryu" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" - [[package]] name = "serde" version = "1.0.228" @@ -418,15 +491,15 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.145" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", - "ryu", "serde", "serde_core", + "zmij", ] [[package]] @@ -437,9 +510,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.111" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -448,18 +521,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", @@ -468,9 +541,9 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "utf8parse" @@ -554,9 +627,15 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.13" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" dependencies = [ "memchr", ] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/programs/Cargo.toml b/programs/Cargo.toml new file mode 100644 index 0000000..79bd198 --- /dev/null +++ b/programs/Cargo.toml @@ -0,0 +1,16 @@ +[workspace] +resolver = "2" +members = ["bun2nix", "bun2nix-core", "cache-entry-creator"] + +[workspace.package] +version = "2.1.2" +edition = "2024" +license = "MIT" + +[workspace.dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[profile.release] +lto = true +codegen-units = 1 diff --git a/programs/bun2nix-core/Cargo.toml b/programs/bun2nix-core/Cargo.toml new file mode 100644 index 0000000..40ad5c1 --- /dev/null +++ b/programs/bun2nix-core/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "bun2nix-core" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } diff --git a/programs/bun2nix-core/src/cache_name.rs b/programs/bun2nix-core/src/cache_name.rs new file mode 100644 index 0000000..59d6f28 --- /dev/null +++ b/programs/bun2nix-core/src/cache_name.rs @@ -0,0 +1,250 @@ +//! Cache-folder naming logic ported from `programs/cache-entry-creator/src/main.zig`. +//! +//! Produces the bun on-disk cache directory name for a given package identifier. +//! See bun's `PackageManagerDirectories.zig` for the original source. + +use crate::wyhash::wyhash11; + +const WYHASH_SEED: u64 = 0; + +/// Dispatch: select the correct naming function based on the identifier prefix. +/// +/// Mirrors `cachedFolderPrintBasename` from the Zig implementation. +pub fn cached_folder_print_basename(input: &str, registry: Option<&str>) -> String { + if input.starts_with("tarball:") { + cached_tarball_folder_print_basename(input) + } else if input.starts_with("github:") { + cached_github_folder_print_basename(input) + } else if input.starts_with("git:") { + cached_git_folder_print_basename(input) + } else { + cached_npm_package_folder_print_basename(input, registry) + } +} + +/// Produce a correct bun cache folder name for a given npm identifier. +/// +/// Ported from `cachedNpmPackageFolderPrintBasename` in `main.zig`. +/// +/// When a non-default registry is used, the format includes the registry hostname: +/// e.g., `@scope/pkg@1.0.0@@npm.pkg.github.com@@@1`. +/// +/// Pre-release components (after `-`) are hashed with wyhash11, formatted **lowercase**. +/// Build-metadata components (after `+`) are hashed with wyhash11, formatted **uppercase**. +pub fn cached_npm_package_folder_print_basename(pkg: &str, registry: Option<&str>) -> String { + // Suffix is "@@{registry}@@@1" for non-default registries, or "@@@1" for default. + let suffix = if let Some(reg) = registry { + format!("@@{}@@@1", reg) + } else { + "@@@1".to_string() + }; + + // Find the last '@' to split name from version (handles scoped packages like @scope/pkg@ver). + let Some(version_start) = pkg.rfind('@') else { + return format!("{}{}", pkg, suffix); + }; + let name = &pkg[..version_start]; + let ver = &pkg[version_start..]; // includes leading '@' + + // Handle pre-release: ver contains '-' before any '+' + if let Some(pre_idx) = ver.find('-') { + let version = &ver[..pre_idx]; // e.g. "@1.2.3" + let pre_and_build = &ver[pre_idx + 1..]; // e.g. "beta.1+build.123" + + if let Some(build_idx) = pre_and_build.find('+') { + let pre = &pre_and_build[..build_idx]; + let build = &pre_and_build[build_idx + 1..]; + // pre-release: lowercase; build-metadata: uppercase — match Zig {x:0>16}/{X:0>16} + return format!( + "{}{}-{:016x}+{:016X}{}", + name, + version, + wyhash11(WYHASH_SEED, pre.as_bytes()), + wyhash11(WYHASH_SEED, build.as_bytes()), + suffix, + ); + } + + return format!( + "{}{}-{:016x}{}", + name, + version, + wyhash11(WYHASH_SEED, pre_and_build.as_bytes()), + suffix, + ); + } + + // Handle build-metadata only (no pre-release '-') + if let Some(build_idx) = ver.find('+') { + let version = &ver[..build_idx]; // e.g. "@1.2.3" + let build = &ver[build_idx + 1..]; // e.g. "build.123" + // build-metadata: uppercase + return format!( + "{}{}+{:016X}{}", + name, + version, + wyhash11(WYHASH_SEED, build.as_bytes()), + suffix, + ); + } + + // Plain version — no hashing needed. + format!("{}{}", pkg, suffix) +} + +/// Produce a correct bun cache folder name for a given tarball dependency. +/// +/// Ported from `cachedTarballFolderPrintBasename` in `main.zig`. +pub fn cached_tarball_folder_print_basename(url: &str) -> String { + let without_pre = &url["tarball:".len()..]; + format!( + "@T@{:016x}@@@1", + wyhash11(WYHASH_SEED, without_pre.as_bytes()) + ) +} + +/// Produce a correct bun cache folder name for a given github dependency. +/// +/// Ported from `cachedGithubFolderPrintBasename` in `main.zig`. +pub fn cached_github_folder_print_basename(url: &str) -> String { + let without_pre = &url["github:".len()..]; + format!("@GH@{}@@@1", without_pre) +} + +/// Produce a correct bun cache folder name for a given git dependency. +/// +/// Ported from `cachedGitFolderPrintBasename` in `main.zig`. +pub fn cached_git_folder_print_basename(url: &str) -> String { + let without_pre = &url["git:".len()..]; + format!("@G@{}", without_pre) +} + +#[cfg(test)] +mod tests { + use super::*; + + // ── Verbatim Zig test vectors ──────────────────────────────────────────── + // Copied from `programs/cache-entry-creator/src/main.zig`, converted to + // Rust `#[test]` form. These are the fidelity gate — casing must match exactly. + + #[test] + fn cached_npm_package_folder_print_basename_fn() { + let cases: &[(&str, Option<&str>, &str)] = &[ + // Without registry (default npm registry) + ( + "react@1.2.3-beta.1+build.123", + None, + "react@1.2.3-c0734e9369ab610d+F48F05ED5AABC3A0@@@1", + ), + ( + "tailwindcss@4.0.0-beta.9", + None, + "tailwindcss@4.0.0-73c5c46324e78b9b@@@1", + ), + ( + "react@1.2.3+build.123", + None, + "react@1.2.3+F48F05ED5AABC3A0@@@1", + ), + ("react@1.2.3", None, "react@1.2.3@@@1"), + ("undici-types@6.20.0", None, "undici-types@6.20.0@@@1"), + ( + "@types/react-dom@19.0.4", + None, + "@types/react-dom@19.0.4@@@1", + ), + ( + "react-compiler-runtime@19.0.0-beta-e552027-20250112", + None, + "react-compiler-runtime@19.0.0-0f3fc645a5103715@@@1", + ), + // With registry (non-default registry like GitHub Packages) + ( + "@scope/package@1.0.0", + Some("npm.pkg.github.com"), + "@scope/package@1.0.0@@npm.pkg.github.com@@@1", + ), + ( + "private-pkg@2.0.0", + Some("my.registry.com"), + "private-pkg@2.0.0@@my.registry.com@@@1", + ), + // With registry and pre-release version + ( + "@scope/pkg@1.0.0-beta.1", + Some("npm.pkg.github.com"), + "@scope/pkg@1.0.0-c0734e9369ab610d@@npm.pkg.github.com@@@1", + ), + // With registry and build metadata + ( + "@scope/pkg@1.0.0+build.123", + Some("npm.pkg.github.com"), + "@scope/pkg@1.0.0+F48F05ED5AABC3A0@@npm.pkg.github.com@@@1", + ), + ]; + for &(input, registry, expected) in cases { + let result = cached_npm_package_folder_print_basename(input, registry); + assert_eq!( + result, expected, + "npm basename mismatch: input={:?} registry={:?}", + input, registry + ); + } + } + + #[test] + fn cached_tarball_folder_print_basename_fn() { + let cases: &[(&str, &str)] = &[( + "tarball:https://registry.npmjs.org/zod/-/zod-3.21.4.tgz", + "@T@3be02e19198e30ee@@@1", + )]; + for &(input, expected) in cases { + let result = cached_tarball_folder_print_basename(input); + assert_eq!( + result, expected, + "tarball basename mismatch: input={:?}", + input + ); + } + } + + #[test] + fn cached_github_folder_print_basename_fn() { + let cases: &[(&str, &str)] = &[( + "github:colinhacks-zod-f9bbb50", + "@GH@colinhacks-zod-f9bbb50@@@1", + )]; + for &(input, expected) in cases { + let result = cached_github_folder_print_basename(input); + assert_eq!( + result, expected, + "github basename mismatch: input={:?}", + input + ); + } + } + + #[test] + fn cached_git_folder_print_basename_fn() { + let cases: &[(&str, &str)] = &[( + "git:ee100d81f12ae315a81c2a664979a6cc1bce99a2", + "@G@ee100d81f12ae315a81c2a664979a6cc1bce99a2", + )]; + for &(input, expected) in cases { + let result = cached_git_folder_print_basename(input); + assert_eq!(result, expected, "git basename mismatch: input={:?}", input); + } + } + + #[test] + fn dispatcher_routes_correctly() { + // Verify the dispatcher selects the right function for each prefix. + assert!( + cached_folder_print_basename("tarball:https://foo.com/pkg.tgz", None) + .starts_with("@T@") + ); + assert!(cached_folder_print_basename("github:owner-repo-sha", None).starts_with("@GH@")); + assert!(cached_folder_print_basename("git:abc123", None).starts_with("@G@")); + assert!(cached_folder_print_basename("react@1.0.0", None).ends_with("@@@1")); + } +} diff --git a/programs/bun2nix-core/src/lib.rs b/programs/bun2nix-core/src/lib.rs new file mode 100644 index 0000000..336bfce --- /dev/null +++ b/programs/bun2nix-core/src/lib.rs @@ -0,0 +1,3 @@ +pub mod cache_name; +pub mod manifest; +pub mod wyhash; diff --git a/programs/bun2nix-core/src/manifest/build.rs b/programs/bun2nix-core/src/manifest/build.rs new file mode 100644 index 0000000..5b93017 --- /dev/null +++ b/programs/bun2nix-core/src/manifest/build.rs @@ -0,0 +1,400 @@ +//! General multi-version `.npm` manifest builder. +//! +//! [`build_manifest`] converts a [`meta::PackageMeta`] (which may contain +//! multiple versions) into a [`serialize::PackageManifest`] suitable for +//! writing with [`serialize::write`]. +//! +//! The implementation reuses [`super::StringArena`] and +//! [`super::build_dep_group`] — the same interning primitives used by +//! [`super::build_single_version`] — so string-buffer logic lives in one place. + +use super::{ + StringArena, build_dep_group, + layout::{ + Architecture, Bin, BinValue, DistTagMap, ExternVersionMap, ExternalString, + ExternalStringList, Integrity, IntegrityTag, NpmPackage, OperatingSystem, PackageVersion, + PackageVersionList, SemverString, SemverVersion, VersionSlice, + }, + meta::{PackageMeta, VersionMeta}, + serialize::PackageManifest, +}; + +// ────────────────────────────────────────────────────────────────────────── +// Public entry point +// ────────────────────────────────────────────────────────────────────────── + +/// Build an in-memory [`PackageManifest`] for **all** versions in `pkg`. +/// +/// Versions are sorted deterministically (ascending semver) before being +/// written, so the resulting binary is independent of input order. +/// +/// The builder only handles registry packages (name + versions with a semver +/// string and a tarball URL). Git/tarball/workspace packages must not be +/// passed here. +/// +/// `NpmPackage.public_max_age` is set to [`u32::MAX`] so bun never considers +/// the manifest stale when running in offline mode. +pub fn build_manifest(pkg: &PackageMeta) -> PackageManifest { + let mut arena = StringArena::default(); + + // Shared external-string buffers, grown monotonically so slice offsets + // recorded in earlier versions remain stable. + let mut names: Vec = Vec::new(); + let mut values: Vec = Vec::new(); + let mut bin_entries: Vec = Vec::new(); + + // Package name (inlined if ≤ 8 bytes, stored in the arena otherwise). + let name_ext = arena.intern(&pkg.name); + + // Sort versions ascending (deterministic output regardless of JSON order). + let mut sorted: Vec<&VersionMeta> = pkg.versions.iter().collect(); + sorted.sort_by_key(|v| parse_semver(&v.version)); + + let mut semver_versions: Vec = Vec::with_capacity(sorted.len()); + let mut package_versions: Vec = Vec::with_capacity(sorted.len()); + + for vm in &sorted { + let (major, minor, patch) = parse_semver(&vm.version); + semver_versions.push(SemverVersion { + major, + minor, + patch, + ..SemverVersion::default() + }); + + let pv = build_one_version(vm, &mut arena, &mut names, &mut values, &mut bin_entries); + package_versions.push(pv); + } + + let n = sorted.len() as u32; + let n_names = names.len() as u32; + let external_strings = names.into_boxed_slice(); + let external_strings_for_versions = values.into_boxed_slice(); + let extern_strings_bin_entries = bin_entries.into_boxed_slice(); + + let mut pkg_struct = NpmPackage { + name: name_ext, + // Never expire — so bun treats this manifest as fresh indefinitely + // under BUN_MANIFEST_CACHE=2 (verified in the Task 3 spike). + public_max_age: u32::MAX, + ..NpmPackage::default() + }; + + // releases: keys = entire versions array, values = entire package_versions array. + pkg_struct.releases = ExternVersionMap { + keys: VersionSlice::new(0, n), + values: PackageVersionList::new(0, n), + }; + pkg_struct.prereleases = ExternVersionMap::default(); + // dist_tags: left empty (v1 scope — see brief "Leave dist_tags empty"). + pkg_struct.dist_tags = DistTagMap::default(); + pkg_struct.versions_buf = VersionSlice::new(0, n); + pkg_struct.string_lists_buf = ExternalStringList::new(0, n_names); + + PackageManifest { + pkg: pkg_struct, + string_buf: arena.buf.into_boxed_slice(), + versions: semver_versions.into_boxed_slice(), + external_strings, + external_strings_for_versions, + package_versions: package_versions.into_boxed_slice(), + extern_strings_bin_entries, + bundled_deps_buf: Box::new([]), + } +} + +// ────────────────────────────────────────────────────────────────────────── +// Per-version helper +// ────────────────────────────────────────────────────────────────────────── + +/// Build one [`PackageVersion`] from a [`VersionMeta`], appending strings to +/// the shared arenas. This is the per-version helper called in a loop by +/// [`build_manifest`], factored out so that the interning/dep-group/integrity +/// logic is not duplicated. +fn build_one_version( + vm: &VersionMeta, + arena: &mut StringArena, + names: &mut Vec, + values: &mut Vec, + bin_entries: &mut Vec, +) -> PackageVersion { + // Dependency groups (three groups: deps / optional / peer). + let dependencies = build_dep_group( + arena, + names, + values, + vm.dependencies.iter().map(|(k, v)| (k.clone(), v.clone())), + ); + let optional_dependencies = build_dep_group( + arena, + names, + values, + vm.optional_dependencies.iter().map(|(k, v)| (k.clone(), v.clone())), + ); + + // Peer dependencies: bun's ABI places **optional** peers at the FRONT of + // the `peer_dependencies` array and stores the count of optional peers in + // `non_optional_peer_dependencies_start` (i.e. the index where non-optional + // peers begin). Indices [0, start) are optional; [start, len) are required. + // Source: npm.rs:688 comment + lockfile/Package.rs:841 reader condition. + let optional_peer_set: std::collections::BTreeSet<&str> = + vm.optional_peers.iter().map(|s| s.as_str()).collect(); + + // Split peers: optional first, then non-optional (required). + let opt_peers = vm + .peer_dependencies + .iter() + .filter(|(k, _)| optional_peer_set.contains(k.as_str())); + let non_opt_peers = vm + .peer_dependencies + .iter() + .filter(|(k, _)| !optional_peer_set.contains(k.as_str())); + + // `non_optional_peer_dependencies_start` = number of optional peers + // (= the index at which non-optional peers begin). + let opt_count = + vm.peer_dependencies.keys().filter(|k| optional_peer_set.contains(k.as_str())).count() + as u32; + + // Build the combined peer group in one pass (optional then non-optional). + let peer_dependencies = build_dep_group( + arena, + names, + values, + opt_peers.chain(non_opt_peers).map(|(k, v)| (k.clone(), v.clone())), + ); + + // Integrity: decode SRI string → raw tag + digest bytes. + let integrity = parse_integrity(&vm.integrity); + + // Tarball URL: intern the full URL string; bun reads it directly from the + // PackageVersion when scheduling tarball downloads. + let tarball_url = if vm.tarball_url.is_empty() { + ExternalString::default() + } else { + arena.intern(&vm.tarball_url) + }; + + // OS and CPU bit-flag fields. + let os = parse_os(&vm.os); + let cpu = parse_cpu(&vm.cpu); + + // Bin entries. + let bin = build_bin(vm, arena, bin_entries); + + PackageVersion { + integrity, + dependencies, + optional_dependencies, + peer_dependencies, + non_optional_peer_dependencies_start: opt_count, + tarball_url, + os, + cpu, + has_install_script: vm.has_install_script, + bin, + ..PackageVersion::default() + } +} + +// ────────────────────────────────────────────────────────────────────────── +// Integrity (SRI → raw bytes) +// ────────────────────────────────────────────────────────────────────────── + +fn parse_integrity(sri: &str) -> Integrity { + if sri.is_empty() { + return Integrity::default(); + } + let (tag, b64) = if let Some(rest) = sri.strip_prefix("sha512-") { + (IntegrityTag::SHA512, rest) + } else if let Some(rest) = sri.strip_prefix("sha384-") { + (IntegrityTag::SHA384, rest) + } else if let Some(rest) = sri.strip_prefix("sha256-") { + (IntegrityTag::SHA256, rest) + } else if let Some(rest) = sri.strip_prefix("sha1-") { + (IntegrityTag::SHA1, rest) + } else { + return Integrity::default(); + }; + + let Some(decoded) = decode_base64(b64) else { + return Integrity::default(); + }; + + let mut value = [0u8; 64]; + let len = decoded.len().min(64); + value[..len].copy_from_slice(&decoded[..len]); + + Integrity { tag, value } +} + +/// Minimal standard-alphabet base64 decoder (no URL-safe; handles `=` padding). +fn decode_base64(input: &str) -> Option> { + let input = input.trim_end_matches('='); + let bytes = input.as_bytes(); + let mut out = Vec::with_capacity(bytes.len() * 3 / 4); + let mut i = 0; + while i < bytes.len() { + let remaining = bytes.len() - i; + let a = b64_val(bytes[i])?; + let b = if remaining > 1 { b64_val(bytes[i + 1])? } else { return None }; + out.push((a << 2) | (b >> 4)); + if remaining > 2 { + let c = b64_val(bytes[i + 2])?; + out.push((b << 4) | (c >> 2)); + if remaining > 3 { + let d = b64_val(bytes[i + 3])?; + out.push((c << 6) | d); + } + } + i += 4; + } + Some(out) +} + +#[inline] +fn b64_val(b: u8) -> Option { + match b { + b'A'..=b'Z' => Some(b - b'A'), + b'a'..=b'z' => Some(b - b'a' + 26), + b'0'..=b'9' => Some(b - b'0' + 52), + b'+' => Some(62), + b'/' => Some(63), + _ => None, + } +} + +// ────────────────────────────────────────────────────────────────────────── +// OS / CPU bit-flags +// ────────────────────────────────────────────────────────────────────────── + +/// Parse an npm `"os"` array into [`OperatingSystem`] bitflags. +/// An empty list means "all OSes" (`OperatingSystem::ALL`). +fn parse_os(os: &[String]) -> OperatingSystem { + if os.is_empty() { + return OperatingSystem::ALL; + } + let mut flags: u16 = 0; + for s in os { + flags |= match s.as_str() { + "aix" => OperatingSystem::AIX, + "darwin" | "macos" => OperatingSystem::DARWIN, + "freebsd" => OperatingSystem::FREEBSD, + "linux" => OperatingSystem::LINUX, + "openbsd" => OperatingSystem::OPENBSD, + "sunos" => OperatingSystem::SUNOS, + "win32" => OperatingSystem::WIN32, + "android" => OperatingSystem::ANDROID, + _ => 0, + }; + } + OperatingSystem(flags) +} + +/// Parse an npm `"cpu"` array into [`Architecture`] bitflags. +/// An empty list means "all architectures" (`Architecture::ALL`). +fn parse_cpu(cpu: &[String]) -> Architecture { + if cpu.is_empty() { + return Architecture::ALL; + } + let mut flags: u16 = 0; + for s in cpu { + flags |= match s.as_str() { + "arm" => Architecture::ARM, + "arm64" => Architecture::ARM64, + "ia32" => Architecture::IA32, + "mips" => Architecture::MIPS, + "mipsel" => Architecture::MIPSEL, + "ppc" => Architecture::PPC, + "ppc64" => Architecture::PPC64, + "s390" => Architecture::S390, + "s390x" => Architecture::S390X, + "x32" => Architecture::X32, + "x64" => Architecture::X64, + _ => 0, + }; + } + Architecture(flags) +} + +// ────────────────────────────────────────────────────────────────────────── +// Bin +// ────────────────────────────────────────────────────────────────────────── + +/// Build the [`Bin`] entry for one version. +/// +/// Tag encoding (mirrors bun's `Bin.Tag`): +/// - `None (0)` — no bin field. +/// - `NamedFile (2)` — a single-entry map; value is two packed [`SemverString`]s +/// (key at `raw[0..1]`, path at `raw[2..3]`). +/// - `Map (4)` — multi-entry; `raw[0]` = offset into `bin_entries`, +/// `raw[1]` = count of `ExternalString` entries (2 × number of pairs). +fn build_bin( + vm: &VersionMeta, + arena: &mut StringArena, + bin_entries: &mut Vec, +) -> Bin { + match vm.bin.len() { + 0 => Bin::default(), + 1 => { + let (key, val) = vm.bin.iter().next().expect("len==1"); + let k_ss = arena.intern(key).value; + let v_ss = arena.intern(val).value; + Bin { + tag: 2, // NamedFile + _padding_tag: [0; 3], + value: BinValue { + raw: [ + ss_lo(&k_ss), + ss_hi(&k_ss), + ss_lo(&v_ss), + ss_hi(&v_ss), + ], + }, + } + } + _ => { + let off = bin_entries.len() as u32; + let mut count: u32 = 0; + for (key, val) in &vm.bin { + bin_entries.push(arena.intern(key)); + bin_entries.push(arena.intern(val)); + count += 2; + } + Bin { + tag: 4, // Map + _padding_tag: [0; 3], + value: BinValue { raw: [off, count, 0, 0] }, + } + } + } +} + +/// Low 32 bits of a [`SemverString`]'s bytes (as native-endian `u32`). +#[inline] +fn ss_lo(ss: &SemverString) -> u32 { + u32::from_ne_bytes(ss.bytes[..4].try_into().unwrap()) +} + +/// High 32 bits of a [`SemverString`]'s bytes (as native-endian `u32`). +#[inline] +fn ss_hi(ss: &SemverString) -> u32 { + u32::from_ne_bytes(ss.bytes[4..].try_into().unwrap()) +} + +// ────────────────────────────────────────────────────────────────────────── +// Semver helpers +// ────────────────────────────────────────────────────────────────────────── + +/// Parse `"major.minor.patch"` (optionally with a pre-release suffix) into a +/// sortable `(major, minor, patch)` triple. Unknown components default to 0. +pub(crate) fn parse_semver(s: &str) -> (u64, u64, u64) { + let mut parts = s.splitn(3, '.'); + let major: u64 = parts.next().and_then(|p| p.parse().ok()).unwrap_or(0); + let minor: u64 = parts.next().and_then(|p| p.parse().ok()).unwrap_or(0); + let patch: u64 = parts + .next() + .and_then(|p| p.split('-').next()?.parse().ok()) + .unwrap_or(0); + (major, minor, patch) +} diff --git a/programs/bun2nix-core/src/manifest/layout.rs b/programs/bun2nix-core/src/manifest/layout.rs new file mode 100644 index 0000000..159588e --- /dev/null +++ b/programs/bun2nix-core/src/manifest/layout.rs @@ -0,0 +1,528 @@ +//! POD structs ported **verbatim** from bun's `bun-npm-manifest-cache-v0.0.7` +//! on-disk format. Layout is an ABI contract: these structs are reinterpreted +//! as raw bytes by the serializer, so size/alignment/field-offsets must match +//! bun exactly. Every `const _: () = { … }` block below is a transcription +//! guard — if one fails to compile, a layout was copied wrong. +//! +//! Sources (bun @ 5621c5de90): +//! - `DistTagMap`, `ExternVersionMap`, `PackageVersion`, `NpmPackage` — `src/install/npm.rs` +//! - `ExternalSlice`, `ExternalStringMap`, OS/CPU/Libc — `src/install_types/resolver_hooks.rs` +//! - `ExternalString`, `SemverString` (`String`), `Pointer` — `src/semver/lib.rs` +//! - `VersionType` (`Semver::Version`), `Tag` — `src/semver/Version.rs` +//! - `Integrity` (+ `Tag`) — `src/install/integrity.rs` +//! - `Bin` (+ `Value` union, `Tag`) — `src/install/bin.rs` + +use core::marker::PhantomData; +use core::mem::{offset_of, size_of}; + +// ────────────────────────────────────────────────────────────────────────── +// SemverString (`Semver::semver_string::String`) — src/semver/lib.rs:238 +// +// 8 raw bytes. Either an inline string (final bit of byte 7 clear, NUL +// terminated when < 8 bytes) or an external `Pointer` (final bit set, low 63 +// bits are the `{off,len}` pair into the string buffer). +// ────────────────────────────────────────────────────────────────────────── + +#[repr(C)] +#[derive(Copy, Clone, PartialEq, Eq, Default)] +pub struct SemverString { + pub bytes: [u8; SemverString::MAX_INLINE_LEN], +} + +impl SemverString { + pub const MAX_INLINE_LEN: usize = 8; + + pub const EMPTY: SemverString = SemverString { bytes: [0; 8] }; + + /// Pointers are truncated to 63 bits via this mask (bun: `MAX_ADDRESSABLE_SPACE_MASK`). + const MAX_ADDRESSABLE_SPACE_MASK: u64 = (1u64 << 63) - 1; + + #[inline] + pub fn can_inline(buf: &[u8]) -> bool { + match buf.len() { + 0..=7 => true, + 8 => buf[7] & 0x80 == 0, + _ => false, + } + } + + /// Build an inline string (`String::init_inline`). Caller guarantees + /// `can_inline(in_)`. + #[inline] + pub fn init_inline(in_: &[u8]) -> SemverString { + debug_assert!(Self::can_inline(in_)); + let mut bytes = [0u8; 8]; + bytes[..in_.len()].copy_from_slice(in_); + SemverString { bytes } + } + + /// Build an external string referencing `string_buf[off..off+len]`. Mirrors + /// the pointer-packing path of `String::init`: `off` in the low 4 bytes, + /// `len` in the high 4 bytes, with bit 63 set to mark "external". + #[inline] + pub fn init_external(off: u32, len: u32) -> SemverString { + let bits = (off as u64) | ((len as u64) << 32); + let packed = (bits & Self::MAX_ADDRESSABLE_SPACE_MASK) | (1u64 << 63); + SemverString { + bytes: packed.to_ne_bytes(), + } + } + + /// Construct the right representation for `in_` placed at `off` in the + /// string buffer. + #[inline] + pub fn init(off: u32, in_: &[u8]) -> SemverString { + if Self::can_inline(in_) { + Self::init_inline(in_) + } else { + Self::init_external(off, in_.len() as u32) + } + } +} + +// ────────────────────────────────────────────────────────────────────────── +// String.Pointer — src/semver/lib.rs:803 +// ────────────────────────────────────────────────────────────────────────── + +#[repr(C)] +#[derive(Copy, Clone, Default)] +pub struct Pointer { + pub off: u32, + pub len: u32, +} + +// ────────────────────────────────────────────────────────────────────────── +// ExternalString — src/semver/lib.rs:165 +// ────────────────────────────────────────────────────────────────────────── + +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub struct ExternalString { + pub value: SemverString, + pub hash: u64, +} + +const _: () = assert!(size_of::() == 16); + +// ────────────────────────────────────────────────────────────────────────── +// Semver::Version == VersionType — src/semver/Version.rs:54 +// +// v0.0.7 uses u64 for major/minor/patch, so `_tag_padding` is `[u8; 0]` and is +// omitted here. `tag` sits at offset 24, total size 56. +// ────────────────────────────────────────────────────────────────────────── + +#[repr(C)] +#[derive(Copy, Clone, Default)] +pub struct Tag { + pub pre: ExternalString, + pub build: ExternalString, +} + +const _: () = { + assert!(size_of::() == 32); + assert!(align_of::() == 8); +}; + +#[repr(C)] +#[derive(Copy, Clone, Default)] +pub struct SemverVersion { + pub major: u64, + pub minor: u64, + pub patch: u64, + pub tag: Tag, +} + +// Layout is load-bearing (lockfile/manifest binary format). +const _: () = { + assert!(size_of::() == 56); + assert!(offset_of!(SemverVersion, tag) == 24); +}; + +// ────────────────────────────────────────────────────────────────────────── +// ExternalSlice + aliases — src/install_types/resolver_hooks.rs:41 +// +// An `(off, len)` index pair into a flat backing buffer. Manual trait impls +// (matching bun) so a non-`Copy` element type would not gate the `(off,len)` +// pair's own copyability. +// ────────────────────────────────────────────────────────────────────────── + +#[repr(C)] +pub struct ExternalSlice { + pub off: u32, + pub len: u32, + _marker: PhantomData, +} + +impl Copy for ExternalSlice {} +impl Clone for ExternalSlice { + #[inline] + fn clone(&self) -> Self { + *self + } +} +impl Default for ExternalSlice { + #[inline] + fn default() -> Self { + Self { + off: 0, + len: 0, + _marker: PhantomData, + } + } +} + +impl ExternalSlice { + #[inline] + pub const fn new(off: u32, len: u32) -> Self { + Self { + off, + len, + _marker: PhantomData, + } + } + + pub const INVALID: Self = Self { + off: u32::MAX, + len: u32::MAX, + _marker: PhantomData, + }; + + #[inline] + pub fn is_invalid(self) -> bool { + self.off == u32::MAX && self.len == u32::MAX + } +} + +pub type PackageNameHash = u64; +pub type ExternalStringList = ExternalSlice; +pub type ExternalPackageNameHashList = ExternalSlice; +pub type VersionSlice = ExternalSlice; +pub type PackageVersionList = ExternalSlice; + +const _: () = assert!(size_of::() == 8); + +#[repr(C)] +#[derive(Copy, Clone, Default)] +pub struct ExternalStringMap { + pub name: ExternalStringList, + pub value: ExternalStringList, +} + +const _: () = assert!(size_of::() == 16); + +// ────────────────────────────────────────────────────────────────────────── +// OperatingSystem / Architecture / Libc — src/install_types/resolver_hooks.rs +// transparent newtypes over u16/u16/u8. +// ────────────────────────────────────────────────────────────────────────── + +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq, Default)] +pub struct OperatingSystem(pub u16); + +impl OperatingSystem { + pub const NONE: Self = Self(0); + pub const AIX: u16 = 1 << 1; + pub const DARWIN: u16 = 1 << 2; + pub const FREEBSD: u16 = 1 << 3; + pub const LINUX: u16 = 1 << 4; + pub const OPENBSD: u16 = 1 << 5; + pub const SUNOS: u16 = 1 << 6; + pub const WIN32: u16 = 1 << 7; + pub const ANDROID: u16 = 1 << 8; + pub const ALL_VALUE: u16 = Self::AIX + | Self::DARWIN + | Self::FREEBSD + | Self::LINUX + | Self::OPENBSD + | Self::SUNOS + | Self::WIN32 + | Self::ANDROID; + pub const ALL: Self = Self(Self::ALL_VALUE); +} + +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq, Default)] +pub struct Architecture(pub u16); + +impl Architecture { + pub const NONE: Self = Self(0); + pub const ARM: u16 = 1 << 1; + pub const ARM64: u16 = 1 << 2; + pub const IA32: u16 = 1 << 3; + pub const MIPS: u16 = 1 << 4; + pub const MIPSEL: u16 = 1 << 5; + pub const PPC: u16 = 1 << 6; + pub const PPC64: u16 = 1 << 7; + pub const S390: u16 = 1 << 8; + pub const S390X: u16 = 1 << 9; + pub const X32: u16 = 1 << 10; + pub const X64: u16 = 1 << 11; + pub const ALL_VALUE: u16 = Self::ARM + | Self::ARM64 + | Self::IA32 + | Self::MIPS + | Self::MIPSEL + | Self::PPC + | Self::PPC64 + | Self::S390 + | Self::S390X + | Self::X32 + | Self::X64; + pub const ALL: Self = Self(Self::ALL_VALUE); +} + +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq, Default)] +pub struct Libc(pub u8); + +impl Libc { + pub const NONE: Self = Self(0); + pub const GLIBC: u8 = 1 << 1; + pub const MUSL: u8 = 1 << 2; + pub const ALL_VALUE: u8 = Self::GLIBC | Self::MUSL; + pub const ALL: Self = Self(Self::ALL_VALUE); +} + +// ────────────────────────────────────────────────────────────────────────── +// Integrity — src/install/integrity.rs:15 (size 65, align 1) +// ────────────────────────────────────────────────────────────────────────── + +/// `#[repr(transparent)]` newtype over `u8` (any byte is a valid tag). +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct IntegrityTag(pub u8); + +impl IntegrityTag { + pub const UNKNOWN: IntegrityTag = IntegrityTag(0); + pub const SHA1: IntegrityTag = IntegrityTag(1); + pub const SHA256: IntegrityTag = IntegrityTag(2); + pub const SHA384: IntegrityTag = IntegrityTag(3); + pub const SHA512: IntegrityTag = IntegrityTag(4); +} + +impl Default for IntegrityTag { + fn default() -> Self { + IntegrityTag::UNKNOWN + } +} + +/// `max(SHA1=20, SHA256=32, SHA384=48, SHA512=64) = 64`. +pub const DIGEST_BUF_LEN: usize = 64; + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct Integrity { + pub tag: IntegrityTag, + pub value: [u8; DIGEST_BUF_LEN], +} + +impl Default for Integrity { + fn default() -> Self { + Self { + tag: IntegrityTag::UNKNOWN, + value: [0u8; DIGEST_BUF_LEN], + } + } +} + +const _: () = assert!(size_of::() == 65); +const _: () = assert!(align_of::() == 1); + +// ────────────────────────────────────────────────────────────────────────── +// Bin — src/install/bin.rs:40 (tag + 3 padding + 16-byte union value) +// +// The real `Value` is a `#[repr(C)] union` whose largest member is +// `[SemverString; 2]` (16 bytes) and whose alignment (4) comes from +// `ExternalStringList`. Modelled here as a 16-byte, align-4 POD so `Bin` has +// size 20 / align 4, matching bun. We only ever emit `Tag::None` (zeroed). +// ────────────────────────────────────────────────────────────────────────── + +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub struct BinValue { + pub raw: [u32; 4], +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct Bin { + pub tag: u8, + pub _padding_tag: [u8; 3], + pub value: BinValue, +} + +impl Bin { + pub const TAG_NONE: u8 = 0; +} + +impl Default for Bin { + fn default() -> Self { + Bin { + tag: Bin::TAG_NONE, + _padding_tag: [0; 3], + value: BinValue::default(), + } + } +} + +const _: () = { + assert!(size_of::() == 20); + assert!(align_of::() == 4); +}; + +// ────────────────────────────────────────────────────────────────────────── +// DistTagMap — src/install/npm.rs:586 +// ────────────────────────────────────────────────────────────────────────── + +#[repr(C)] +#[derive(Default, Clone, Copy)] +pub struct DistTagMap { + pub tags: ExternalStringList, + pub versions: VersionSlice, +} + +// ────────────────────────────────────────────────────────────────────────── +// ExternVersionMap — src/install/npm.rs:595 +// ────────────────────────────────────────────────────────────────────────── + +#[repr(C)] +#[derive(Default, Clone, Copy)] +pub struct ExternVersionMap { + pub keys: VersionSlice, + pub values: PackageVersionList, +} + +// ────────────────────────────────────────────────────────────────────────── +// PackageVersion (240 B) — src/install/npm.rs:650 +// ────────────────────────────────────────────────────────────────────────── + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct PackageVersion { + pub integrity: Integrity, + pub _padding_after_integrity: [u8; 3], + pub dependencies: ExternalStringMap, + pub optional_dependencies: ExternalStringMap, + pub peer_dependencies: ExternalStringMap, + pub dev_dependencies: ExternalStringMap, + pub bundled_dependencies: ExternalPackageNameHashList, + pub bin: Bin, + pub engines: ExternalStringMap, + pub non_optional_peer_dependencies_start: u32, + pub _padding_before_man_dir: [u8; 4], + pub man_dir: ExternalString, + pub tarball_url: ExternalString, + pub unpacked_size: u32, + pub file_count: u32, + pub os: OperatingSystem, + pub cpu: Architecture, + pub libc: Libc, + pub has_install_script: bool, + pub _padding_tail: [u8; 2], + pub publish_timestamp_ms: f64, +} + +impl Default for PackageVersion { + fn default() -> Self { + Self { + integrity: Integrity::default(), + _padding_after_integrity: [0; 3], + dependencies: ExternalStringMap::default(), + optional_dependencies: ExternalStringMap::default(), + peer_dependencies: ExternalStringMap::default(), + dev_dependencies: ExternalStringMap::default(), + bundled_dependencies: ExternalPackageNameHashList::default(), + bin: Bin::default(), + engines: ExternalStringMap::default(), + non_optional_peer_dependencies_start: 0, + _padding_before_man_dir: [0; 4], + man_dir: ExternalString::default(), + tarball_url: ExternalString::default(), + unpacked_size: 0, + file_count: 0, + os: OperatingSystem::ALL, + cpu: Architecture::ALL, + libc: Libc::NONE, + has_install_script: false, + _padding_tail: [0; 2], + publish_timestamp_ms: 0.0, + } + } +} + +const _: () = assert!( + size_of::() == 240, + "Npm.PackageVersion layout drifted from bun spec (expected 240 bytes)" +); + +const _: () = { + // gap between `integrity` (size 65, align 1) and `dependencies` (align 4 → 68) + assert!( + offset_of!(PackageVersion, _padding_after_integrity) + == offset_of!(PackageVersion, integrity) + size_of::() + ); + assert!( + offset_of!(PackageVersion, dependencies) + == offset_of!(PackageVersion, _padding_after_integrity) + 3 + ); + // gap between `non_optional_peer_dependencies_start` (ends at 180) and `man_dir` (align 8 → 184) + assert!( + offset_of!(PackageVersion, _padding_before_man_dir) + == offset_of!(PackageVersion, non_optional_peer_dependencies_start) + size_of::() + ); + assert!( + offset_of!(PackageVersion, man_dir) + == offset_of!(PackageVersion, _padding_before_man_dir) + 4 + ); + // gap between `has_install_script` (ends at 230) and `publish_timestamp_ms` (align 8 → 232) + assert!( + offset_of!(PackageVersion, _padding_tail) + == offset_of!(PackageVersion, has_install_script) + size_of::() + ); + assert!( + offset_of!(PackageVersion, publish_timestamp_ms) + == offset_of!(PackageVersion, _padding_tail) + 2 + ); + // anchor a few absolute offsets to catch drift in the middle of the struct + assert!(offset_of!(PackageVersion, peer_dependencies) == 100); + assert!(offset_of!(PackageVersion, man_dir) == 184); + assert!(offset_of!(PackageVersion, publish_timestamp_ms) == 232); +}; + +// ────────────────────────────────────────────────────────────────────────── +// NpmPackage (120 B) — src/install/npm.rs:812 +// ────────────────────────────────────────────────────────────────────────── + +#[repr(C)] +#[derive(Default, Clone, Copy)] +pub struct NpmPackage { + pub last_modified: SemverString, + pub etag: SemverString, + pub modified: SemverString, + pub public_max_age: u32, + pub _padding_after_max_age: [u8; 4], + pub name: ExternalString, + pub releases: ExternVersionMap, + pub prereleases: ExternVersionMap, + pub dist_tags: DistTagMap, + pub versions_buf: VersionSlice, + pub string_lists_buf: ExternalStringList, + pub has_extended_manifest: bool, + pub _padding_tail: [u8; 7], +} + +const _: () = { + // gap between `public_max_age` (ends at 28) and `name` (align 8 → 32) + assert!( + offset_of!(NpmPackage, _padding_after_max_age) + == offset_of!(NpmPackage, public_max_age) + size_of::() + ); + assert!(offset_of!(NpmPackage, name) == offset_of!(NpmPackage, _padding_after_max_age) + 4); + // tail gap after `has_extended_manifest` (bool, at 112) → struct end (120) + assert!( + offset_of!(NpmPackage, _padding_tail) + == offset_of!(NpmPackage, has_extended_manifest) + size_of::() + ); + assert!(offset_of!(NpmPackage, _padding_tail) + 7 == size_of::()); + assert!(size_of::() == 120); +}; diff --git a/programs/bun2nix-core/src/manifest/meta.rs b/programs/bun2nix-core/src/manifest/meta.rs new file mode 100644 index 0000000..e7c1fd6 --- /dev/null +++ b/programs/bun2nix-core/src/manifest/meta.rs @@ -0,0 +1,69 @@ +//! Serde-serialisable metadata types that flow through the bun2nix pipeline: +//! +//! * [`PackageMeta`] — the registry view of a package (name + all versions). +//! * [`VersionMeta`] — per-version fields extracted from the npm registry manifest. +//! * [`EntryMeta`] — the per-entry record written into `bun.nix` and consumed +//! by the manifest tool (Task 7) to assemble offline `.npm` cache files. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +/// The subset of an npm registry document needed to build a bun `.npm` manifest +/// cache entry. Contains one or more versions of a single package. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PackageMeta { + /// Bare npm package name, e.g. `"@neoconfetti/svelte"` or `"ms"`. + pub name: String, + /// At least one version; caller must sort or the builder will sort for you. + pub versions: Vec, +} + +/// Per-version metadata extracted from the npm registry `vnd.npm.install-v1` +/// abbreviated manifest. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VersionMeta { + /// Semver version string, e.g. `"2.2.2"`. + pub version: String, + /// Full tarball URL, e.g. + /// `"https://registry.npmjs.org/@neoconfetti/svelte/-/svelte-2.2.2.tgz"`. + pub tarball_url: String, + /// SRI integrity string, e.g. `"sha512-"`. The npm client (Task 5) + /// may leave this empty; the manifest tool (Task 7) fills it from + /// [`EntryMeta::hash`] before calling the builder. + pub integrity: String, + /// `"dependencies"` map from the registry manifest. + pub dependencies: BTreeMap, + /// `"peerDependencies"` map. + pub peer_dependencies: BTreeMap, + /// `"optionalDependencies"` map. + pub optional_dependencies: BTreeMap, + /// Names of peer dependencies that are marked optional in + /// `"peerDependenciesMeta"`. + pub optional_peers: Vec, + /// `"bin"` map (command → relative path). + pub bin: BTreeMap, + /// `"os"` list from the registry manifest (`[]` → all OSes). + pub os: Vec, + /// `"cpu"` list from the registry manifest (`[]` → all architectures). + pub cpu: Vec, + /// `true` when the package has a non-empty `install` / `preinstall` / + /// `postinstall` script, or when the registry sets `"hasInstallScript"`. + pub has_install_script: bool, +} + +/// The per-`bun.nix`-entry record materialised by the Nix layer (Task 8) and +/// consumed by the manifest tool (Task 7) to write `.npm` cache files. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EntryMeta { + /// `"@"`, the unique key used to look up this entry. + pub name_version: String, + /// Nix store hash of the fetched tarball (used as the SRI `sha512` value + /// after base64-encoding when `manifest.integrity` is empty). + pub hash: String, + /// Registry base URL (without trailing slash). `None` → use the default + /// `https://registry.npmjs.org` registry. + pub registry: Option, + /// Full version metadata for this entry. + pub manifest: VersionMeta, +} diff --git a/programs/bun2nix-core/src/manifest/mod.rs b/programs/bun2nix-core/src/manifest/mod.rs new file mode 100644 index 0000000..6ecbe8b --- /dev/null +++ b/programs/bun2nix-core/src/manifest/mod.rs @@ -0,0 +1,441 @@ +//! bun npm `.npm` manifest-cache (`bun-npm-manifest-cache-v0.0.7`) support: +//! verbatim layout structs ([`layout`]), the serializer ([`serialize`]), a +//! minimal single-version builder, a round-trip reader, serde metadata types +//! ([`meta`]), and the general multi-version builder ([`build`]). + +pub mod build; +pub mod layout; +pub mod meta; +pub mod serialize; + +use std::collections::BTreeMap; + +use crate::wyhash::wyhash11; +use layout::{ + DistTagMap, ExternVersionMap, ExternalString, ExternalStringList, ExternalStringMap, Integrity, + IntegrityTag, NpmPackage, PackageVersion, PackageVersionList, SemverString, SemverVersion, + VersionSlice, +}; +use serialize::PackageManifest; + +/// bun's default registry URL (`Registry::DEFAULT_URL`), with trailing slash. +pub const DEFAULT_REGISTRY_URL: &str = "https://registry.npmjs.org/"; + +/// `wyhash11(0, "https://registry.npmjs.org")` — bun's `DEFAULT_URL_HASH`. +/// +/// Hardcoded from the literal observed in real `.npm` headers (bytes 49..57) +/// and verified at test time against [`default_url_hash`]. +pub const DEFAULT_URL_HASH: u64 = 0x9c1e_4d1f_1eff_5fcd; + +/// Length of the default registry URL with the trailing slash removed +/// (`len("https://registry.npmjs.org") == 26`). This is the value bun stores in +/// the `.npm` header right after `url_hash`. +pub const DEFAULT_REGISTRY_HREF_LEN: u64 = 26; + +/// wyhash11 of the default registry URL **without** the trailing slash — matches +/// bun's `DEFAULT_URL_HASH`. +pub fn default_url_hash() -> u64 { + wyhash11(0, DEFAULT_REGISTRY_URL.trim_end_matches('/').as_bytes()) +} + +/// `.npm` filename for a package on the default registry (`.npm`). +pub fn manifest_file_name(name: &str) -> String { + format!("{:016x}.npm", wyhash11(0, name.as_bytes())) +} + +// ────────────────────────────────────────────────────────────────────────── +// Minimal single-version builder (spike quality; Task 4 generalizes it). +// ────────────────────────────────────────────────────────────────────────── + +/// A single (name, version-range) dependency pair, e.g. +/// `("svelte", "^3.0.0 || ^4.0.0 || ^5.0.0")`. +#[derive(Clone)] +pub struct Dep { + pub name: String, + pub range: String, +} + +/// Inputs for [`build_single_version`]. +pub struct SingleVersionInput<'a> { + pub name: &'a str, + pub version: (u64, u64, u64), + /// Optional SHA-512 digest (raw 64 bytes) for the version's integrity field. + pub sha512: Option<[u8; 64]>, + pub dependencies: Vec, + pub optional_dependencies: Vec, + pub peer_dependencies: Vec, +} + +/// Helper that accumulates the string buffer and the two external-string arrays +/// while building a manifest. +#[derive(Default)] +pub(crate) struct StringArena { + pub(crate) buf: Vec, +} + +impl StringArena { + /// Append `s` to the buffer if it cannot be stored inline, returning the + /// `ExternalString` handle (inline or external) for it. + pub(crate) fn intern(&mut self, s: &str) -> ExternalString { + let bytes = s.as_bytes(); + let hash = wyhash11(0, bytes); + let value = if SemverString::can_inline(bytes) { + SemverString::init_inline(bytes) + } else { + let off = self.buf.len() as u32; + self.buf.extend_from_slice(bytes); + SemverString::init_external(off, bytes.len() as u32) + }; + ExternalString { value, hash } + } +} + +/// Append a dependency group (name → version-range pairs) to the shared +/// `names` and `values` vectors, returning the `ExternalStringMap` slice that +/// indexes them. Both vectors grow monotonically across calls, so offsets from +/// earlier calls remain stable. +pub(crate) fn build_dep_group( + arena: &mut StringArena, + names: &mut Vec, + values: &mut Vec, + deps: impl Iterator, +) -> ExternalStringMap { + let name_off = names.len() as u32; + let value_off = values.len() as u32; + let mut count: u32 = 0; + for (name, range) in deps { + names.push(arena.intern(&name)); + values.push(arena.intern(&range)); + count += 1; + } + if count == 0 { + return ExternalStringMap::default(); + } + ExternalStringMap { + name: ExternalStringList::new(name_off, count), + value: ExternalStringList::new(value_off, count), + } +} + +/// Build an in-memory [`PackageManifest`] for a single version of an +/// npm-registry package, populating just the fields bun reads for +/// (peer-)dependency resolution: name, the `releases` map, and each dependency +/// group of the one `PackageVersion`. +/// +/// For multi-version packages, prefer [`build::build_manifest`] with a +/// [`meta::PackageMeta`] instead. +pub fn build_single_version(input: &SingleVersionInput) -> PackageManifest { + let mut arena = StringArena::default(); + + // Package name. + let name_ext = arena.intern(input.name); + + // Dependency groups: names go into `external_strings`, values (ranges) go + // into `external_strings_for_versions` — matching bun's two-buffer split. + let mut names: Vec = Vec::new(); + let mut values: Vec = Vec::new(); + + let dependencies = build_dep_group( + &mut arena, + &mut names, + &mut values, + input.dependencies.iter().map(|d| (d.name.clone(), d.range.clone())), + ); + let optional_dependencies = build_dep_group( + &mut arena, + &mut names, + &mut values, + input.optional_dependencies.iter().map(|d| (d.name.clone(), d.range.clone())), + ); + let peer_dependencies = build_dep_group( + &mut arena, + &mut names, + &mut values, + input.peer_dependencies.iter().map(|d| (d.name.clone(), d.range.clone())), + ); + + let integrity = match input.sha512 { + Some(value) => Integrity { + tag: IntegrityTag::SHA512, + value, + }, + None => Integrity::default(), + }; + + let pv = PackageVersion { + integrity, + dependencies, + optional_dependencies, + peer_dependencies, + ..PackageVersion::default() + }; + + let version = SemverVersion { + major: input.version.0, + minor: input.version.1, + patch: input.version.2, + ..SemverVersion::default() + }; + + let versions = vec![version].into_boxed_slice(); + let package_versions = vec![pv].into_boxed_slice(); + let n_names = names.len() as u32; + let external_strings = names.into_boxed_slice(); + let external_strings_for_versions = values.into_boxed_slice(); + + let mut pkg = NpmPackage { + name: name_ext, + // Far-future expiry so bun treats the manifest as "fresh" under + // `BUN_MANIFEST_CACHE=2` (cache-control on) and never revalidates over + // the network: `by_name_hash` only returns a non-expired manifest when + // `public_max_age > timestamp_for_manifest_cache_control` (current time). + public_max_age: u32::MAX, + ..NpmPackage::default() + }; + // releases: keys index the `versions` buffer, values index `package_versions`. + pkg.releases = ExternVersionMap { + keys: VersionSlice::new(0, 1), + values: PackageVersionList::new(0, 1), + }; + pkg.prereleases = ExternVersionMap::default(); + pkg.dist_tags = DistTagMap::default(); + pkg.versions_buf = VersionSlice::new(0, 1); + pkg.string_lists_buf = ExternalStringList::new(0, n_names); + + PackageManifest { + pkg, + string_buf: arena.buf.into_boxed_slice(), + versions, + external_strings, + external_strings_for_versions, + package_versions, + extern_strings_bin_entries: Box::new([]), + bundled_deps_buf: Box::new([]), + } +} + +// ────────────────────────────────────────────────────────────────────────── +// Round-trip reader (ports `Serializer::read_array` + `read_all`). +// ────────────────────────────────────────────────────────────────────────── + +/// A deserialized manifest with the same buffers as [`PackageManifest`]. +#[derive(Default)] +pub struct ReadManifest { + pub url_hash: u64, + pub href_len: u64, + pub pkg: NpmPackage, + pub string_buf: Vec, + pub versions: Vec, + pub external_strings: Vec, + pub external_strings_for_versions: Vec, + pub package_versions: Vec, +} + +struct Reader<'a> { + bytes: &'a [u8], + pos: usize, +} + +impl<'a> Reader<'a> { + fn read_u64(&mut self) -> u64 { + let v = u64::from_le_bytes(self.bytes[self.pos..self.pos + 8].try_into().unwrap()); + self.pos += 8; + v + } + + fn align_to(&mut self, align: usize) { + self.pos = self.pos.next_multiple_of(align); + } + + /// Mirror of `read_struct::` after aligning. + fn read_struct(&mut self) -> T { + self.align_to(std::mem::align_of::()); + // SAFETY: bytes were produced by our own serializer for a POD `T`. + let v = unsafe { std::ptr::read_unaligned(self.bytes[self.pos..].as_ptr().cast::()) }; + self.pos += std::mem::size_of::(); + v + } + + /// Mirror of `Serializer::read_array`. + fn read_array(&mut self) -> Vec { + let byte_len = self.read_u64() as usize; + if byte_len == 0 { + return Vec::new(); + } + self.align_to(std::mem::align_of::()); + let region = &self.bytes[self.pos..self.pos + byte_len]; + let n = byte_len / std::mem::size_of::(); + let mut out = Vec::with_capacity(n); + for i in 0..n { + // SAFETY: region is `byte_len` long, n elements of size_of::. + let v = unsafe { + std::ptr::read_unaligned(region.as_ptr().add(i * std::mem::size_of::()).cast::()) + }; + out.push(v); + } + self.pos += byte_len; + out + } +} + +/// Deserialize a `.npm` byte buffer back into a [`ReadManifest`]. Returns `None` +/// if the header does not match. +pub fn read(bytes: &[u8]) -> Option { + if bytes.len() < serialize::HEADER.len() || &bytes[..serialize::HEADER.len()] != serialize::HEADER + { + return None; + } + let mut r = Reader { + bytes, + pos: serialize::HEADER.len(), + }; + let url_hash = r.read_u64(); + let href_len = r.read_u64(); + let pkg: NpmPackage = r.read_struct(); + let string_buf = r.read_array::(); + let versions = r.read_array::(); + let external_strings = r.read_array::(); + let external_strings_for_versions = r.read_array::(); + let package_versions = r.read_array::(); + let _bin = r.read_array::(); + let _bundled = r.read_array::(); + + Some(ReadManifest { + url_hash, + href_len, + pkg, + string_buf, + versions, + external_strings, + external_strings_for_versions, + package_versions, + }) +} + +/// Resolve an [`ExternalString`]'s text against `string_buf` (inline or external). +pub fn resolve_str<'a>(s: &'a ExternalString, string_buf: &'a [u8]) -> Vec { + let bytes = &s.value.bytes; + if bytes[SemverString::MAX_INLINE_LEN - 1] & 0x80 == 0 { + // inline: bytes up to first NUL (or all 8) + let end = bytes.iter().position(|&b| b == 0).unwrap_or(8); + bytes[..end].to_vec() + } else { + let bits = u64::from_ne_bytes(*bytes) & ((1u64 << 63) - 1); + let off = (bits & 0xffff_ffff) as usize; + let len = (bits >> 32) as usize; + string_buf[off..off + len].to_vec() + } +} + +// ────────────────────────────────────────────────────────────────────────── +// ReadManifest higher-level accessors (used by tests and downstream crates). +// ────────────────────────────────────────────────────────────────────────── + +/// A resolved view of one `PackageVersion` within a deserialized manifest. +pub struct ReadPackageVersion<'a> { + manifest: &'a ReadManifest, + pub pv: PackageVersion, +} + +impl<'a> ReadPackageVersion<'a> { + /// All peer dependencies as a `BTreeMap`. + pub fn peer_dependencies(&self) -> BTreeMap { + self.dep_map(self.pv.peer_dependencies) + } + + /// All regular (non-optional) dependencies as a `BTreeMap`. + pub fn dependencies(&self) -> BTreeMap { + self.dep_map(self.pv.dependencies) + } + + /// All optional dependencies as a `BTreeMap`. + pub fn optional_dependencies(&self) -> BTreeMap { + self.dep_map(self.pv.optional_dependencies) + } + + /// Full tarball URL stored in this `PackageVersion`, or an empty string if + /// the URL was not stored (bun infers it from the registry in that case). + pub fn tarball_url(&self) -> String { + let bytes = resolve_str(&self.pv.tarball_url, &self.manifest.string_buf); + String::from_utf8_lossy(&bytes).into_owned() + } + + /// Resolve the name of the peer dependency at position `i` within the + /// `peer_dependencies` array. Used by tests that validate the bun ABI + /// ordering (optional peers first, non-optional peers after + /// `non_optional_peer_dependencies_start`). + pub fn peer_dep_name_at(&self, i: usize) -> String { + let n_off = self.pv.peer_dependencies.name.off as usize; + let bytes = resolve_str( + &self.manifest.external_strings[n_off + i], + &self.manifest.string_buf, + ); + String::from_utf8_lossy(&bytes).into_owned() + } + + fn dep_map(&self, map: ExternalStringMap) -> BTreeMap { + let mut result = BTreeMap::new(); + let n_off = map.name.off as usize; + let v_off = map.value.off as usize; + for i in 0..map.name.len as usize { + let name_bytes = + resolve_str(&self.manifest.external_strings[n_off + i], &self.manifest.string_buf); + let val_bytes = resolve_str( + &self.manifest.external_strings_for_versions[v_off + i], + &self.manifest.string_buf, + ); + result.insert( + String::from_utf8_lossy(&name_bytes).into_owned(), + String::from_utf8_lossy(&val_bytes).into_owned(), + ); + } + result + } +} + +impl ReadManifest { + /// Package name as raw UTF-8 bytes. + pub fn name(&self) -> Vec { + resolve_str(&self.pkg.name, &self.string_buf) + } + + /// Cache-control max-age (seconds). `u32::MAX` means "never expires". + pub fn public_max_age(&self) -> u32 { + self.pkg.public_max_age + } + + /// Look up a release version by its semver string (e.g. `"2.2.2"`). + /// + /// Returns `None` if the version is not present in `releases`. Pre-release + /// versions (tag present) are not searched. + pub fn find_version(&self, version_str: &str) -> Option> { + let (major, minor, patch) = parse_semver_str(version_str)?; + + let keys_slice = &self.versions[self.pkg.releases.keys.off as usize + ..(self.pkg.releases.keys.off + self.pkg.releases.keys.len) as usize]; + let pvs_slice = &self.package_versions[self.pkg.releases.values.off as usize + ..(self.pkg.releases.values.off + self.pkg.releases.values.len) as usize]; + + for (i, v) in keys_slice.iter().enumerate() { + if v.major == major && v.minor == minor && v.patch == patch { + return Some(ReadPackageVersion { + manifest: self, + pv: pvs_slice[i], + }); + } + } + None + } +} + +/// Parse `"major.minor.patch"` into `(major, minor, patch)`. Returns `None` +/// if the string cannot be parsed. +fn parse_semver_str(s: &str) -> Option<(u64, u64, u64)> { + let mut parts = s.splitn(3, '.'); + let major: u64 = parts.next()?.parse().ok()?; + let minor: u64 = parts.next()?.parse().ok()?; + // Strip any pre-release suffix after the patch number. + let patch_str = parts.next()?; + let patch: u64 = patch_str.split('-').next().unwrap_or("").parse().ok()?; + Some((major, minor, patch)) +} diff --git a/programs/bun2nix-core/src/manifest/serialize.rs b/programs/bun2nix-core/src/manifest/serialize.rs new file mode 100644 index 0000000..f58b7d2 --- /dev/null +++ b/programs/bun2nix-core/src/manifest/serialize.rs @@ -0,0 +1,125 @@ +//! Serializer for bun's `bun-npm-manifest-cache-v0.0.7` `.npm` files. +//! +//! Mirrors `PackageManifest` + `Serializer::write` / `write_array` / `Aligner` +//! from `src/install/npm.rs` (~865, ~926-1025) and `src/install/lib.rs` (~1040). +//! The on-disk framing is: +//! 1. 49-byte ASCII header +//! 2. `u64` LE registry url_hash +//! 3. `u64` LE registry href length (without trailing slash) +//! 4. `pkg: NpmPackage` (aligned to align_of::()) +//! 5. seven arrays, each: `u64` LE byte-length, alignment padding, raw bytes. +//! (a zero-length array is just the `u64` 0, with no padding.) + +use std::io::{self, Write}; + +use super::layout::{ExternalString, NpmPackage, PackageVersion, SemverVersion}; + +/// `#!/usr/bin/env bun\nbun-npm-manifest-cache-v0.0.7\n` — exactly 49 bytes. +/// (The length is not serialized, so it must be fixed.) +pub const HEADER: &[u8] = b"#!/usr/bin/env bun\nbun-npm-manifest-cache-v0.0.7\n"; + +pub const HEADER_LEN_ASSERT: () = assert!(HEADER.len() == 49); + +/// In-memory manifest mirroring `npm.rs:865`'s `PackageManifest`. +/// +/// `pkg` holds the fixed-size header struct; the seven boxed slices are the +/// flat backing buffers that the slices/offsets inside `pkg` (and inside each +/// `PackageVersion`) index into. +#[derive(Default, Clone)] +pub struct PackageManifest { + pub pkg: NpmPackage, + pub string_buf: Box<[u8]>, + pub versions: Box<[SemverVersion]>, + pub external_strings: Box<[ExternalString]>, + /// Backing buffer for dependency *values* (version ranges) and version tag + /// strings. Kept separate so contiguous identical versions can be deduped. + pub external_strings_for_versions: Box<[ExternalString]>, + pub package_versions: Box<[PackageVersion]>, + pub extern_strings_bin_entries: Box<[ExternalString]>, + pub bundled_deps_buf: Box<[u64]>, +} + +/// Number of zero bytes needed to advance `pos` to the next multiple of `align`. +/// Mirrors `Aligner::skip_amount_with_align`. +#[inline] +fn skip_amount(align: usize, pos: u64) -> usize { + (pos as usize).next_multiple_of(align) - (pos as usize) +} + +/// Write alignment padding (zero bytes) for `align`, returning the count. +/// Mirrors `Aligner::write`. +fn write_alignment(w: &mut impl Write, align: usize, pos: u64) -> io::Result { + let to_write = skip_amount(align, pos); + if to_write > 0 { + // bun repeats from a 144-byte zero buffer; equivalent to writing zeros. + const ZEROS: [u8; 144] = [0u8; 144]; + let mut remaining = to_write; + while remaining > 0 { + let n = remaining.min(ZEROS.len()); + w.write_all(&ZEROS[..n])?; + remaining -= n; + } + } + Ok(to_write) +} + +/// Reinterpret a `&[T]` of POD as raw bytes (the structs have explicit padding, +/// so every byte is initialized). +fn as_bytes(array: &[T]) -> &[u8] { + // SAFETY: T is a `#[repr(C)]` POD with no implicit padding (guarded by the + // layout asserts in `layout.rs`), so all bytes are initialized. + unsafe { std::slice::from_raw_parts(array.as_ptr().cast::(), std::mem::size_of_val(array)) } +} + +/// Mirror of `Serializer::write_array`. +fn write_array(w: &mut impl Write, array: &[T], pos: &mut u64) -> io::Result<()> { + let bytes = as_bytes(array); + if bytes.is_empty() { + w.write_all(&0u64.to_le_bytes())?; + *pos += 8; + return Ok(()); + } + w.write_all(&(bytes.len() as u64).to_le_bytes())?; + *pos += 8; + *pos += write_alignment(w, std::mem::align_of::(), *pos)? as u64; + w.write_all(bytes)?; + *pos += bytes.len() as u64; + Ok(()) +} + +/// Serialize `m` into bun's `.npm` binary format. `url_hash` and +/// `registry_href_len` are the registry scope fields (for the default registry, +/// `super::DEFAULT_URL_HASH` and `26`). +pub fn write( + m: &PackageManifest, + url_hash: u64, + registry_href_len: u64, + w: &mut impl Write, +) -> io::Result<()> { + let mut pos: u64 = 0; + + w.write_all(HEADER)?; + pos += HEADER.len() as u64; + + w.write_all(&url_hash.to_le_bytes())?; + w.write_all(®istry_href_len.to_le_bytes())?; + pos += 16; + + // pkg (aligned to align_of::()) + { + let bytes = as_bytes(std::slice::from_ref(&m.pkg)); + pos += write_alignment(w, std::mem::align_of::(), pos)? as u64; + w.write_all(bytes)?; + pos += bytes.len() as u64; + } + + write_array(w, &m.string_buf, &mut pos)?; + write_array(w, &m.versions, &mut pos)?; + write_array(w, &m.external_strings, &mut pos)?; + write_array(w, &m.external_strings_for_versions, &mut pos)?; + write_array(w, &m.package_versions, &mut pos)?; + write_array(w, &m.extern_strings_bin_entries, &mut pos)?; + write_array(w, &m.bundled_deps_buf, &mut pos)?; + + Ok(()) +} diff --git a/programs/bun2nix-core/src/wyhash.rs b/programs/bun2nix-core/src/wyhash.rs new file mode 100644 index 0000000..4ab9ca4 --- /dev/null +++ b/programs/bun2nix-core/src/wyhash.rs @@ -0,0 +1,416 @@ +// Vendored verbatim from bun `src/wyhash/lib.rs` @ commit 5621c5de90. +// Kept byte-identical to upstream; do not refactor or reformat. +// Note: `#![feature(hasher_prefixfree_extras)]` from the original crate root +// is omitted here — it is a crate-level attribute (inapplicable in a module +// file) and is not required by any item in lines 1–387 that we vendor. + +// +// `Wyhash11` is an older wyhash variant (32-byte rounds, 5 primes). +// +// `Wyhash` is the current wyhash (final4 variant, 48-byte rounds, 4 secrets) — +// used by RuntimeTranspilerCache, `bun.hash()`, router. +// +// THESE ARE DIFFERENT ALGORITHMS. They produce different outputs for the same input. +// + +// ════════════════════════════════════════════════════════════════════════════ +// Wyhash11 (legacy, 32-byte rounds, 5 primes) +// ════════════════════════════════════════════════════════════════════════════ + +const PRIMES: [u64; 5] = [ + 0xa0761d6478bd642f, + 0xe7037ed1a0b428db, + 0x8ebc6af09c88c6e3, + 0x589965cc75374cc3, + 0x1d8e4e27c47d124f, +]; + +// A single unaligned load. Mirrors the `Wyhash::read4`/`read8` treatment +// below (~lib.rs:600): the per-byte `[data[0], data[1], ...]` spelling left a +// cmp/je-to-panic ladder per byte in `WyhashStateless::final_`/`round`, and +// `final_` is the `StringHashMap` short-key hash — hit on every parser / +// resolver / module-registry HashMap probe during startup (identifier/path +// keys are <32B so `aligned_len == 0` and the whole key goes through `final_`). +// Every caller proves the length first (the `1..=31` arms of `final_` slice +// `rem_key = &b[0..rem_len]` with `rem_len` == the match scrutinee; `round` +// takes a `&[u8]` it `debug_assert!`s == 32), so a single unaligned load is sound. +#[inline(always)] +fn read_bytes(data: &[u8]) -> u64 { + debug_assert!(data.len() >= usize::from(BYTES)); + // Rust cannot mint an integer type from a const generic; dispatch on the only values used. + match BYTES { + 1 => u64::from(data[0]), + 2 => u64::from(u16::from_le_bytes([data[0], data[1]])), + // SAFETY: `data.len() >= BYTES == 4` (asserted above; every caller + // proves it). `read_unaligned` imposes no alignment requirement. + 4 => u64::from(u32::from_le(unsafe { + core::ptr::read_unaligned(data.as_ptr().cast::()) + })), + // SAFETY: `data.len() >= BYTES == 8` (asserted above; every caller + // proves it). `read_unaligned` imposes no alignment requirement. + 8 => u64::from_le(unsafe { core::ptr::read_unaligned(data.as_ptr().cast::()) }), + _ => unreachable!(), + } +} + +#[inline(always)] +fn read_8bytes_swapped(data: &[u8]) -> u64 { + (read_bytes::<4>(data) << 32) | read_bytes::<4>(&data[4..]) +} + +// `#[inline(always)]`, not just `#[inline]`: these 4 helpers (`mum` + the two +// `mix*` wrappers, alongside the already-`always` `read_8bytes_swapped`) are the +// entire body of the short-key (`0..=16` byte) hash that `StringHashMap` / +// hashbrown runs per identifier / keyword / module-path key while parsing every +// module. `#[inline]` (hint) was being declined across the bun_wyhash → +// bun_collections / bun_js_parser crate boundary, leaving an out-of-line `mum` +// (and a `call` per mix step) inside the hashbrown probe loop. Force the 4 mix +// steps to fold in with no call/ret — same reasoning the surrounding `final_*` +// helpers are `#[inline(always)]` / `#[cold]`-split for. +#[inline(always)] +fn mum(a: u64, b: u64) -> u64 { + let mut r = (a as u128) * (b as u128); + r = (r >> 64) ^ r; + r as u64 +} + +#[inline(always)] +fn mix0(a: u64, b: u64, seed: u64) -> u64 { + mum(a ^ seed ^ PRIMES[0], b ^ seed ^ PRIMES[1]) +} + +#[inline(always)] +fn mix1(a: u64, b: u64, seed: u64) -> u64 { + mum(a ^ seed ^ PRIMES[2], b ^ seed ^ PRIMES[3]) +} + +/// Cold tail of [`WyhashStateless::final_`] — the `17..=31`-byte remainder +/// arms. Split out (and marked `#[cold] #[inline(never)]`) so the common +/// short-key path (`0..=16`, which covers virtually every identifier/path key +/// the parser/resolver/module-registry hash through `StringHashMap`) stays +/// small enough to inline into the hashbrown probe loop. `key.len()` is in +/// `17..=31`; `seed` is the running `WyhashStateless::seed`. +#[cold] +#[inline(never)] +fn final_long(seed: u64, key: &[u8]) -> u64 { + debug_assert!((17..32).contains(&key.len())); + + let head = mix0( + read_8bytes_swapped(key), + read_8bytes_swapped(&key[8..]), + seed, + ); + let tail = match key.len() { + 17 => mix1(read_bytes::<1>(&key[16..]), PRIMES[4], seed), + 18 => mix1(read_bytes::<2>(&key[16..]), PRIMES[4], seed), + 19 => mix1( + (read_bytes::<2>(&key[16..]) << 8) | read_bytes::<1>(&key[18..]), + PRIMES[4], + seed, + ), + 20 => mix1(read_bytes::<4>(&key[16..]), PRIMES[4], seed), + 21 => mix1( + (read_bytes::<4>(&key[16..]) << 8) | read_bytes::<1>(&key[20..]), + PRIMES[4], + seed, + ), + 22 => mix1( + (read_bytes::<4>(&key[16..]) << 16) | read_bytes::<2>(&key[20..]), + PRIMES[4], + seed, + ), + 23 => mix1( + (read_bytes::<4>(&key[16..]) << 24) + | (read_bytes::<2>(&key[20..]) << 8) + | read_bytes::<1>(&key[22..]), + PRIMES[4], + seed, + ), + 24 => mix1(read_8bytes_swapped(&key[16..]), PRIMES[4], seed), + 25 => mix1( + read_8bytes_swapped(&key[16..]), + read_bytes::<1>(&key[24..]), + seed, + ), + 26 => mix1( + read_8bytes_swapped(&key[16..]), + read_bytes::<2>(&key[24..]), + seed, + ), + 27 => mix1( + read_8bytes_swapped(&key[16..]), + (read_bytes::<2>(&key[24..]) << 8) | read_bytes::<1>(&key[26..]), + seed, + ), + 28 => mix1( + read_8bytes_swapped(&key[16..]), + read_bytes::<4>(&key[24..]), + seed, + ), + 29 => mix1( + read_8bytes_swapped(&key[16..]), + (read_bytes::<4>(&key[24..]) << 8) | read_bytes::<1>(&key[28..]), + seed, + ), + 30 => mix1( + read_8bytes_swapped(&key[16..]), + (read_bytes::<4>(&key[24..]) << 16) | read_bytes::<2>(&key[28..]), + seed, + ), + 31 => mix1( + read_8bytes_swapped(&key[16..]), + (read_bytes::<4>(&key[24..]) << 24) + | (read_bytes::<2>(&key[28..]) << 8) + | read_bytes::<1>(&key[30..]), + seed, + ), + _ => unreachable!(), + }; + head ^ tail +} + +// Wyhash version which does not store internal state for handling partial buffers. +// This is needed so that we can maximize the speed for the short key case, which will +// use the non-iterative api which the public Wyhash exposes. +#[derive(Clone, Copy)] +struct WyhashStateless { + seed: u64, + msg_len: usize, +} + +impl WyhashStateless { + #[inline(always)] + pub(crate) fn init(seed: u64) -> WyhashStateless { + WyhashStateless { seed, msg_len: 0 } + } + + #[inline(always)] + fn round(&mut self, b: &[u8]) { + debug_assert!(b.len() == 32); + + self.seed = mix0( + read_bytes::<8>(&b[0..]), + read_bytes::<8>(&b[8..]), + self.seed, + ) ^ mix1( + read_bytes::<8>(&b[16..]), + read_bytes::<8>(&b[24..]), + self.seed, + ); + } + + #[inline(always)] + pub(crate) fn update(&mut self, b: &[u8]) { + debug_assert!(b.len().is_multiple_of(32)); + + let mut off: usize = 0; + while off < b.len() { + self.round(&b[off..off + 32]); + off += 32; + } + + self.msg_len += b.len(); + } + + // `final_` is the `StringHashMap` short-key hash (every parser / resolver / + // module-registry probe during startup). `#[inline(always)]` alone wasn't + // enough — the 31-arm length switch is large enough that LLVM still emitted + // an out-of-line `final_` symbol and `call`ed it. Identifier/path keys are + // almost always <17B, so split the cold `17..=31` tail into a separate + // `#[cold] #[inline(never)] final_long`, leaving `final_` with just the + // `0..=16` arms — small enough to inline cleanly into every hashbrown probe. + #[inline(always)] + pub(crate) fn final_(&mut self, b: &[u8]) -> u64 { + debug_assert!(b.len() < 32); + + let seed = self.seed; + let rem_len = b.len(); + let rem_key = &b[0..rem_len]; + + self.seed = match rem_len { + 0 => seed, + 1 => mix0(read_bytes::<1>(rem_key), PRIMES[4], seed), + 2 => mix0(read_bytes::<2>(rem_key), PRIMES[4], seed), + 3 => mix0( + (read_bytes::<2>(rem_key) << 8) | read_bytes::<1>(&rem_key[2..]), + PRIMES[4], + seed, + ), + 4 => mix0(read_bytes::<4>(rem_key), PRIMES[4], seed), + 5 => mix0( + (read_bytes::<4>(rem_key) << 8) | read_bytes::<1>(&rem_key[4..]), + PRIMES[4], + seed, + ), + 6 => mix0( + (read_bytes::<4>(rem_key) << 16) | read_bytes::<2>(&rem_key[4..]), + PRIMES[4], + seed, + ), + 7 => mix0( + (read_bytes::<4>(rem_key) << 24) + | (read_bytes::<2>(&rem_key[4..]) << 8) + | read_bytes::<1>(&rem_key[6..]), + PRIMES[4], + seed, + ), + 8 => mix0(read_8bytes_swapped(rem_key), PRIMES[4], seed), + 9 => mix0( + read_8bytes_swapped(rem_key), + read_bytes::<1>(&rem_key[8..]), + seed, + ), + 10 => mix0( + read_8bytes_swapped(rem_key), + read_bytes::<2>(&rem_key[8..]), + seed, + ), + 11 => mix0( + read_8bytes_swapped(rem_key), + (read_bytes::<2>(&rem_key[8..]) << 8) | read_bytes::<1>(&rem_key[10..]), + seed, + ), + 12 => mix0( + read_8bytes_swapped(rem_key), + read_bytes::<4>(&rem_key[8..]), + seed, + ), + 13 => mix0( + read_8bytes_swapped(rem_key), + (read_bytes::<4>(&rem_key[8..]) << 8) | read_bytes::<1>(&rem_key[12..]), + seed, + ), + 14 => mix0( + read_8bytes_swapped(rem_key), + (read_bytes::<4>(&rem_key[8..]) << 16) | read_bytes::<2>(&rem_key[12..]), + seed, + ), + 15 => mix0( + read_8bytes_swapped(rem_key), + (read_bytes::<4>(&rem_key[8..]) << 24) + | (read_bytes::<2>(&rem_key[12..]) << 8) + | read_bytes::<1>(&rem_key[14..]), + seed, + ), + 16 => mix0( + read_8bytes_swapped(rem_key), + read_8bytes_swapped(&rem_key[8..]), + seed, + ), + // Keys ≥17B are rare among identifier/path keys; keep this tail out + // of line so the `0..=16` arms above inline into every caller. + _ => final_long(seed, rem_key), + }; + + self.msg_len += b.len(); + mum(self.seed ^ (self.msg_len as u64), PRIMES[4]) + } + + // perf on build/create-next showed `WyhashStateless::hash` out-lined as a + // standalone symbol (91 self-samples) and `call`ed from + // `find_symbol_with_record_usage` and every `StringHashMap`/hashbrown probe + // — `#[inline]` (hint) was being declined across the bun_wyhash → + // bun_js_parser/bun_collections crate boundary; force it. + #[inline(always)] + pub(crate) fn hash(seed: u64, input: &[u8]) -> u64 { + let aligned_len = input.len() - (input.len() % 32); + + let mut c = WyhashStateless::init(seed); + c.update(&input[0..aligned_len]); + c.final_(&input[aligned_len..]) + } +} + +/// Fast non-cryptographic 64bit hash function. +/// See https://github.com/wangyi-fudan/wyhash +pub struct Wyhash11 { + state: WyhashStateless, + + buf: [u8; 32], + buf_len: usize, +} + +impl Wyhash11 { + #[inline] + pub fn init(seed: u64) -> Wyhash11 { + Wyhash11 { + state: WyhashStateless::init(seed), + buf: [0; 32], + buf_len: 0, + } + } + + #[inline] + pub fn update(&mut self, b: &[u8]) { + let mut off: usize = 0; + + if self.buf_len != 0 && self.buf_len + b.len() >= 32 { + off += 32 - self.buf_len; + self.buf[self.buf_len..self.buf_len + off].copy_from_slice(&b[0..off]); + self.state.update(&self.buf[0..]); + self.buf_len = 0; + } + + let remain_len = b.len() - off; + let aligned_len = remain_len - (remain_len % 32); + self.state.update(&b[off..off + aligned_len]); + + let tail = &b[off + aligned_len..]; + self.buf[self.buf_len..self.buf_len + tail.len()].copy_from_slice(tail); + self.buf_len += usize::from(u8::try_from(tail.len()).expect("int cast")); + } + + // Force-inline so no out-of-line copy of `WyhashStateless::final_`'s + // length-switch survives on the `Hasher`-driven streaming path. + #[inline(always)] + pub fn final_(&mut self) -> u64 { + let rem_key = &self.buf[0..self.buf_len]; + + self.state.final_(rem_key) + } + + #[inline(always)] + pub fn hash(seed: u64, input: &[u8]) -> u64 { + WyhashStateless::hash(seed, input) + } +} + +// Allow `Wyhash11` to be used with `core::hash::Hash::hash` (e.g., as the +// state for std/HashMap-style hashing). +impl core::hash::Hasher for Wyhash11 { + #[inline] + fn write(&mut self, bytes: &[u8]) { + self.update(bytes); + } + #[inline] + fn finish(&self) -> u64 { + // `final_` mutates `state`; clone so `Hasher::finish(&self)` stays + // semantically pure (matches std contract). + let mut s = self.state; + s.final_(&self.buf[0..self.buf_len]) + } +} + +/// bun's legacy wyhash (32-byte rounds, 5 primes) — the hash bun uses for +/// on-disk cache keys and `.npm` manifest filenames. Vendored verbatim from +/// bun `src/wyhash/lib.rs`; keep byte-identical to upstream. +pub fn wyhash11(seed: u64, input: &[u8]) -> u64 { + Wyhash11::hash(seed, input) +} + +#[cfg(test)] +mod tests { + use super::wyhash11; + + // These hex values are the lower-16 hex of wyhash11(0, name), taken from + // the Zig creator's golden cache-name tests (programs/cache-entry-creator + // src/main.zig "cachedNpmPackageFolderPrintBasename"): the build-metadata + // component of "react@1.2.3+build.123" hashes "build.123" -> F48F05ED5AABC3A0. + #[test] + fn matches_bun_known_vectors() { + assert_eq!( + format!("{:016X}", wyhash11(0, b"build.123")), + "F48F05ED5AABC3A0" + ); + } +} diff --git a/programs/bun2nix-core/tests/fixtures/ms.json b/programs/bun2nix-core/tests/fixtures/ms.json new file mode 100644 index 0000000..b88fa95 --- /dev/null +++ b/programs/bun2nix-core/tests/fixtures/ms.json @@ -0,0 +1,31 @@ +{ + "name": "ms", + "versions": [ + { + "version": "2.0.0", + "tarball_url": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dependencies": {}, + "peer_dependencies": {}, + "optional_dependencies": {}, + "optional_peers": [], + "bin": {}, + "os": [], + "cpu": [], + "has_install_script": false + }, + { + "version": "2.1.2", + "tarball_url": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dependencies": {}, + "peer_dependencies": {}, + "optional_dependencies": {}, + "optional_peers": [], + "bin": {}, + "os": [], + "cpu": [], + "has_install_script": false + } + ] +} diff --git a/programs/bun2nix-core/tests/fixtures/neoconfetti.json b/programs/bun2nix-core/tests/fixtures/neoconfetti.json new file mode 100644 index 0000000..02fe577 --- /dev/null +++ b/programs/bun2nix-core/tests/fixtures/neoconfetti.json @@ -0,0 +1,20 @@ +{ + "name": "@neoconfetti/svelte", + "versions": [ + { + "version": "2.2.2", + "tarball_url": "https://registry.npmjs.org/@neoconfetti/svelte/-/svelte-2.2.2.tgz", + "integrity": "sha512-E7xCFVEEm5Ctnj2udTJy1b9oaTvjz1zi1mYdEtE8rB5BVwq6kHisosDS+zdWN5PMfEMjtbsOV9Cl6tsNSAD1sA==", + "dependencies": {}, + "peer_dependencies": { + "svelte": "^3.0.0 || ^4.0.0 || ^5.0.0" + }, + "optional_dependencies": {}, + "optional_peers": [], + "bin": {}, + "os": [], + "cpu": [], + "has_install_script": false + } + ] +} diff --git a/programs/bun2nix-core/tests/golden.rs b/programs/bun2nix-core/tests/golden.rs new file mode 100644 index 0000000..6c92da6 --- /dev/null +++ b/programs/bun2nix-core/tests/golden.rs @@ -0,0 +1,294 @@ +use bun2nix_core::manifest::{ + self, Dep, SingleVersionInput, build, build_single_version, meta, read, resolve_str, serialize, +}; + +/// The default-registry url_hash bun stores in every `.npm` header must equal +/// `wyhash11(0, "https://registry.npmjs.org")`. This pins both the wyhash port +/// and the exact bytes hashed. +#[test] +fn default_url_hash_matches_header_constant() { + assert_eq!( + manifest::default_url_hash(), + manifest::DEFAULT_URL_HASH, + "wyhash11 of the registry URL must match the constant read from real .npm headers" + ); +} + +/// `@neoconfetti/svelte`'s manifest filename is `wyhash11(0, name)` hex. +#[test] +fn neoconfetti_filename() { + let f = manifest::manifest_file_name("@neoconfetti/svelte"); + eprintln!("@neoconfetti/svelte -> {f}"); + assert!(f.ends_with(".npm")); +} + +fn neoconfetti_input() -> SingleVersionInput<'static> { + SingleVersionInput { + name: "@neoconfetti/svelte", + version: (2, 2, 2), + sha512: None, + dependencies: vec![], + optional_dependencies: vec![], + peer_dependencies: vec![Dep { + name: "svelte".to_string(), + range: "^3.0.0 || ^4.0.0 || ^5.0.0".to_string(), + }], + } +} + +/// Round-trip: serialize a hand-built manifest, read it back, and assert every +/// field bun relies on for peer-dependency resolution survives intact. This is +/// the validation gate (byte-golden against real bun output is not reproducible +/// for full multi-version manifests — see the report). +#[test] +fn neoconfetti_round_trip() { + let m = build_single_version(&neoconfetti_input()); + let mut out = Vec::new(); + serialize::write( + &m, + manifest::DEFAULT_URL_HASH, + manifest::DEFAULT_REGISTRY_HREF_LEN, + &mut out, + ) + .unwrap(); + + // Header framing. + assert_eq!(&out[..serialize::HEADER.len()], serialize::HEADER); + + let rd = read(&out).expect("header must parse"); + assert_eq!(rd.url_hash, manifest::DEFAULT_URL_HASH); + assert_eq!(rd.href_len, manifest::DEFAULT_REGISTRY_HREF_LEN); + + // Package name resolves correctly. + assert_eq!( + resolve_str(&rd.pkg.name, &rd.string_buf), + b"@neoconfetti/svelte" + ); + + // One release version, 2.2.2. + assert_eq!(rd.versions.len(), 1); + let v = rd.versions[0]; + assert_eq!((v.major, v.minor, v.patch), (2, 2, 2)); + assert_eq!(rd.pkg.releases.keys.len, 1); + assert_eq!(rd.pkg.releases.values.len, 1); + + // The single PackageVersion has exactly one peer dependency: svelte. + assert_eq!(rd.package_versions.len(), 1); + let pv = rd.package_versions[0]; + let peer = pv.peer_dependencies; + assert_eq!(peer.name.len, 1); + assert_eq!(peer.value.len, 1); + + let dep_name = &rd.external_strings[peer.name.off as usize]; + let dep_range = &rd.external_strings_for_versions[peer.value.off as usize]; + assert_eq!(resolve_str(dep_name, &rd.string_buf), b"svelte"); + assert_eq!( + resolve_str(dep_range, &rd.string_buf), + b"^3.0.0 || ^4.0.0 || ^5.0.0" + ); +} + +// ────────────────────────────────────────────────────────────────────────── +// Task 4 builder tests (multi-version manifest) +// ────────────────────────────────────────────────────────────────────────── + +/// Construct a `PackageMeta` for @neoconfetti/svelte 2.2.2 from the fixture. +fn neoconfetti_meta() -> meta::PackageMeta { + let fixture = + std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/neoconfetti.json")) + .expect("neoconfetti.json fixture missing"); + serde_json::from_str(&fixture).expect("failed to parse neoconfetti.json") +} + +/// Single-version round-trip via the general builder: load the neoconfetti +/// fixture, call `build_manifest`, serialise, deserialise, assert fields. +/// +/// Verifies: name, tarball_url, peer_dependencies, public_max_age=u32::MAX. +#[test] +fn builder_round_trips() { + let pkg = neoconfetti_meta(); + let built = build::build_manifest(&pkg); + + let mut out = Vec::new(); + let url_hash = manifest::default_url_hash(); + let href_len = manifest::DEFAULT_REGISTRY_URL.trim_end_matches('/').len() as u64; + serialize::write(&built, url_hash, href_len, &mut out).unwrap(); + + let parsed = read(&out).expect("header must parse"); + + assert_eq!(parsed.name(), b"@neoconfetti/svelte"); + + let v = parsed + .find_version("2.2.2") + .expect("version 2.2.2 must be present"); + assert!( + v.peer_dependencies().contains_key("svelte"), + "svelte peer dep must survive round-trip" + ); + assert_eq!( + v.tarball_url(), + pkg.versions[0].tarball_url, + "tarball_url must survive round-trip" + ); + assert_eq!(parsed.public_max_age(), u32::MAX, "manifest must never expire"); +} + +/// Construct a `PackageMeta` for `ms` with two versions from the fixture. +fn ms_meta() -> meta::PackageMeta { + let fixture = + std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/ms.json")) + .expect("ms.json fixture missing"); + serde_json::from_str(&fixture).expect("failed to parse ms.json") +} + +/// Multi-version round-trip: load `ms` with versions 2.0.0 and 2.1.2, build a +/// manifest, serialise, deserialise, assert BOTH versions are present and their +/// tarball URLs survive the round-trip. +#[test] +fn builder_multi_version() { + let pkg = ms_meta(); + assert_eq!(pkg.versions.len(), 2, "fixture must have two versions"); + + let built = build::build_manifest(&pkg); + + let mut out = Vec::new(); + let url_hash = manifest::default_url_hash(); + let href_len = manifest::DEFAULT_REGISTRY_URL.trim_end_matches('/').len() as u64; + serialize::write(&built, url_hash, href_len, &mut out).unwrap(); + + let parsed = read(&out).expect("header must parse"); + + assert_eq!(parsed.name(), b"ms"); + assert_eq!(parsed.public_max_age(), u32::MAX, "manifest must never expire"); + + // Both versions must be findable and their tarball URLs must survive. + for vm in &pkg.versions { + let v = parsed + .find_version(&vm.version) + .unwrap_or_else(|| panic!("version {} must be present", vm.version)); + assert_eq!( + v.tarball_url(), + vm.tarball_url, + "tarball_url for version {} must survive round-trip", + vm.version + ); + } +} + +/// Round-trip test that validates the bun ABI ordering for peer dependencies: +/// optional peers must occupy indices `[0, non_optional_peer_dependencies_start)` +/// and non-optional peers must occupy `[start, len)`. The field itself must +/// equal the number of optional peers (not the number of non-optional peers). +/// +/// This test FAILS against the old (inverted) code that placed non-optional +/// peers first and stored the non-optional count in the field. +#[test] +fn peer_dep_optional_ordering_round_trip() { + use std::collections::BTreeMap; + + let pkg = meta::PackageMeta { + name: "fake-pkg".to_string(), + versions: vec![meta::VersionMeta { + version: "1.0.0".to_string(), + tarball_url: "https://registry.npmjs.org/fake-pkg/-/fake-pkg-1.0.0.tgz".to_string(), + integrity: String::new(), + dependencies: BTreeMap::new(), + optional_dependencies: BTreeMap::new(), + peer_dependencies: { + let mut m = BTreeMap::new(); + // BTreeMap iterates alphabetically, so "optional-peer" < "required-peer". + // After the fix, optional-peer must appear first regardless of + // alphabetical order. + m.insert("optional-peer".to_string(), "^1.0.0".to_string()); + m.insert("required-peer".to_string(), "^2.0.0".to_string()); + m + }, + optional_peers: vec!["optional-peer".to_string()], + bin: BTreeMap::new(), + os: vec![], + cpu: vec![], + has_install_script: false, + }], + }; + + let built = build::build_manifest(&pkg); + let mut out = Vec::new(); + serialize::write( + &built, + manifest::DEFAULT_URL_HASH, + manifest::DEFAULT_REGISTRY_HREF_LEN, + &mut out, + ) + .unwrap(); + + let parsed = read(&out).expect("manifest must parse"); + let v = parsed.find_version("1.0.0").expect("version 1.0.0 must be present"); + + let total = v.pv.peer_dependencies.name.len as usize; + let start = v.pv.non_optional_peer_dependencies_start as usize; + + assert_eq!(total, 2, "expected 2 peer deps total"); + // `non_optional_peer_dependencies_start` == count of optional peers == 1. + assert_eq!( + start, 1, + "non_optional_peer_dependencies_start must equal the number of optional peers (1), \ + not the number of non-optional peers" + ); + + // Indices [0, start) must be optional peers. + for i in 0..start { + let name = v.peer_dep_name_at(i); + assert_eq!( + name, "optional-peer", + "index {i} (< start={start}) must be an optional peer, got {name:?}" + ); + } + + // Indices [start, total) must be non-optional peers. + for i in start..total { + let name = v.peer_dep_name_at(i); + assert_eq!( + name, "required-peer", + "index {i} (>= start={start}) must be a non-optional (required) peer, got {name:?}" + ); + } +} + +/// Tripwire: assert that the manifest cache format version string is still +/// `bun-npm-manifest-cache-v0.0.7`. +/// +/// If bun bumps its manifest cache format, this test (and the functional +/// round-trip tests above) will fail. To re-port: +/// 1. Diff bun's `src/install/npm.rs` against the new version. +/// 2. Update the layout structs in `bun2nix-core/src/manifest/layout.rs` +/// and the serializer in `serialize.rs`. +/// 3. Bump the version string in `serialize::HEADER`. +/// 4. Update the string below to match. +#[test] +fn manifest_format_version_is_pinned() { + assert!( + serialize::HEADER.ends_with(b"bun-npm-manifest-cache-v0.0.7\n"), + "bun manifest cache format version changed — see comment above for re-port procedure" + ); +} + +/// Emit the generated `.npm` to a path given by the `EMIT_NPM` env var, so the +/// functional (offline-install) test can drop it into a bun cache. Skipped when +/// the env var is unset. +#[test] +fn emit_neoconfetti_npm() { + let Ok(path) = std::env::var("EMIT_NPM") else { + return; + }; + let m = build_single_version(&neoconfetti_input()); + let mut out = Vec::new(); + serialize::write( + &m, + manifest::DEFAULT_URL_HASH, + manifest::DEFAULT_REGISTRY_HREF_LEN, + &mut out, + ) + .unwrap(); + std::fs::write(&path, &out).unwrap(); + eprintln!("wrote {} bytes to {path}", out.len()); +} diff --git a/programs/bun2nix/Cargo.toml b/programs/bun2nix/Cargo.toml index c12e4b4..2cd3cd1 100644 --- a/programs/bun2nix/Cargo.toml +++ b/programs/bun2nix/Cargo.toml @@ -3,6 +3,7 @@ name = "bun2nix" path = "src/main.rs" [dependencies] +bun2nix-core = {path = "../bun2nix-core"} clap = {version = "4.5.31", features = ["derive", "string"]} jsonc-parser = {version = "0.26.2", features = ["serde"]} serde = {version = "1.0.218", features = ["derive"]} @@ -22,9 +23,5 @@ path = "src/lib.rs" [package] name = "bun2nix" -version = "2.1.2" -edition = "2024" - -[profile.release] -lto = true -codegen-units = 1 +edition.workspace = true +version.workspace = true diff --git a/programs/cache-entry-creator/Cargo.toml b/programs/cache-entry-creator/Cargo.toml new file mode 100644 index 0000000..be5c44e --- /dev/null +++ b/programs/cache-entry-creator/Cargo.toml @@ -0,0 +1,14 @@ +[[bin]] +name = "cache_entry_creator" +path = "src/main.rs" + +[dependencies] +bun2nix-core = {path = "../bun2nix-core"} +clap = {version = "4.5.31", features = ["derive"]} +serde_json.workspace = true + +[package] +name = "cache-entry-creator" +edition.workspace = true +license.workspace = true +version.workspace = true diff --git a/programs/cache-entry-creator/build.zig b/programs/cache-entry-creator/build.zig deleted file mode 100644 index 0f6de59..0000000 --- a/programs/cache-entry-creator/build.zig +++ /dev/null @@ -1,40 +0,0 @@ -const std = @import("std"); - -pub fn build(b: *std.Build) void { - const target = b.standardTargetOptions(.{}); - const optimize = b.standardOptimizeOption(.{}); - - const exe = b.addExecutable(.{ - .name = "cache_entry_creator", - .root_module = b.createModule(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }), - }); - - const clap = b.dependency("clap", .{}); - exe.root_module.addImport("clap", clap.module("clap")); - - b.installArtifact(exe); - - const run_step = b.step("run", "Run the app"); - - const run_cmd = b.addRunArtifact(exe); - run_step.dependOn(&run_cmd.step); - - run_cmd.step.dependOn(b.getInstallStep()); - - if (b.args) |args| { - run_cmd.addArgs(args); - } - - const exe_tests = b.addTest(.{ - .root_module = exe.root_module, - }); - - const run_exe_tests = b.addRunArtifact(exe_tests); - - const test_step = b.step("test", "Run tests"); - test_step.dependOn(&run_exe_tests.step); -} diff --git a/programs/cache-entry-creator/build.zig.zon b/programs/cache-entry-creator/build.zig.zon deleted file mode 100644 index 4c9e74f..0000000 --- a/programs/cache-entry-creator/build.zig.zon +++ /dev/null @@ -1,48 +0,0 @@ -.{ - // This is the default name used by packages depending on this one. For - // example, when a user runs `zig fetch --save `, this field is used - // as the key in the `dependencies` table. Although the user can choose a - // different name, most users will stick with this provided value. - // - // It is redundant to include "zig" in this name because it is already - // within the Zig package namespace. - .name = .cache_entry_creator, - // This is a [Semantic Version](https://semver.org/). - // In a future version of Zig it will be used for package deduplication. - .version = "2.1.2", - // Together with name, this represents a globally unique package - // identifier. This field is generated by the Zig toolchain when the - // package is first created, and then *never changes*. This allows - // unambiguous detection of one package being an updated version of - // another. - // - // When forking a Zig project, this id should be regenerated (delete the - // field and run `zig build`) if the upstream project is still maintained. - // Otherwise, the fork is *hostile*, attempting to take control over the - // original project's identity. Thus it is recommended to leave the comment - // on the following line intact, so that it shows up in code reviews that - // modify the field. - .fingerprint = 0xefd4327cd87716f4, // Changing this has security and trust implications. - // Tracks the earliest Zig version that the package considers to be a - // supported use case. - .minimum_zig_version = "0.15.1", - // This field is optional. - // Each dependency must either provide a `url` and `hash`, or a `path`. - // `zig build --fetch` can be used to fetch all dependencies of a package, recursively. - // Once all dependencies are fetched, `zig build` no longer requires - // internet connectivity. - .dependencies = .{ - .clap = .{ - .url = "git+https://github.com/Hejsil/zig-clap#b7e3348ed60f99ba32c75aa707ff7c87adc31b36", - .hash = "clap-0.11.0-oBajB-TnAQC7yPLnZRT5WzHZ_4Ly4dX2OILskli74b9H", - }, - }, - .paths = .{ - "build.zig", - "build.zig.zon", - "src", - // For example... - //"LICENSE", - //"README.md", - }, -} diff --git a/programs/cache-entry-creator/deps.nix b/programs/cache-entry-creator/deps.nix deleted file mode 100644 index d725779..0000000 --- a/programs/cache-entry-creator/deps.nix +++ /dev/null @@ -1,14 +0,0 @@ -# generated by zon2nix (https://github.com/nix-community/zon2nix) - -{ linkFarm, fetchgit }: - -linkFarm "zig-packages" [ - { - name = "clap-0.11.0-oBajB-TnAQC7yPLnZRT5WzHZ_4Ly4dX2OILskli74b9H"; - path = fetchgit { - url = "https://github.com/Hejsil/zig-clap"; - rev = "b7e3348ed60f99ba32c75aa707ff7c87adc31b36"; - hash = "sha256-3JWrCVr+M6WKxcz1xukdQJi+SQELy77ll6AM28BGkXA="; - }; - } -] diff --git a/programs/cache-entry-creator/src/main.rs b/programs/cache-entry-creator/src/main.rs new file mode 100644 index 0000000..13850ea --- /dev/null +++ b/programs/cache-entry-creator/src/main.rs @@ -0,0 +1,344 @@ +//! `cache_entry_creator` — bun cache-entry tool (Rust rewrite of the Zig binary). +//! +//! Two subcommands: +//! - `symlink` — compute the bun cache folder name and create a directory symlink +//! (preserves the Zig behavior; same CLI flags as the original tool). +//! - `manifest` — read a `Vec` JSON and write `.npm` manifest files. + +use std::{ + collections::BTreeMap, + fs, + io, + path::{Path, PathBuf}, +}; + +use bun2nix_core::{ + cache_name::cached_folder_print_basename, + manifest::{ + build::build_manifest, + default_url_hash, + manifest_file_name, + meta::{EntryMeta, PackageMeta, VersionMeta}, + serialize, + DEFAULT_REGISTRY_HREF_LEN, + }, +}; +use clap::{Parser, Subcommand}; + +// ─── CLI ───────────────────────────────────────────────────────────────────── + +#[derive(Parser)] +#[command( + name = "cache_entry_creator", + about = "Tool for producing correctly named and positioned bun cache entries." +)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Create a bun cache directory symlink for an npm/tarball/git package. + /// + /// Computes the bun cache folder name for `--name` and creates a directory + /// symlink from `/` to `--package`. This is a direct port + /// of the Zig tool's behavior; the Nix caller needs no flag-site changes. + Symlink { + /// The $out directory to write the symlink into (created if absent). + #[arg(long)] + out: PathBuf, + /// The package name+version as found in `bun.lock`, e.g. `react@18.3.1`. + #[arg(long)] + name: String, + /// Absolute path to the extracted package contents to symlink. + #[arg(long)] + package: PathBuf, + /// Optional registry hostname for non-default registries, + /// e.g. `npm.pkg.github.com`. + #[arg(long)] + registry: Option, + }, + + /// Write bun `.npm` manifest cache files for a set of package entries. + /// + /// Reads `--meta` as a `Vec` JSON, groups entries by package + /// name, builds a manifest per package, and writes `.npm` + /// files into `--out`. + Manifest { + /// The directory to write `.npm` files into (created if absent). + #[arg(long)] + out: PathBuf, + /// Path to the JSON file containing `Vec`. + #[arg(long)] + meta: PathBuf, + }, +} + +// ─── Main ──────────────────────────────────────────────────────────────────── + +fn main() -> Result<(), Box> { + let cli = Cli::parse(); + match cli.command { + Commands::Symlink { out, name, package, registry } => { + run_symlink(&out, &name, &package, registry.as_deref())?; + } + Commands::Manifest { out, meta } => { + run_manifest(&out, &meta)?; + } + } + Ok(()) +} + +// ─── symlink subcommand ─────────────────────────────────────────────────────── + +/// Core logic for `symlink` mode. +/// +/// Computes the bun cache folder name for `name`, creates parent dirs under +/// `out`, and creates a **directory symlink** from `/` to +/// `package`. +/// +/// Mirrors `PkgLinker::create_cache_entry` from the Zig implementation. +fn run_symlink( + out: &Path, + name: &str, + package: &Path, + registry: Option<&str>, +) -> io::Result<()> { + eprintln!("Creating entry for `{}`...", name); + + let basename = cached_folder_print_basename(name, registry); + let link_path = out.join(&basename); + + // Create parent directory (handles scoped packages like @types/foo which + // produce a basename of `@types/foo@ver@@@1`, needing a `@types/` subdir). + if let Some(parent) = link_path.parent() { + fs::create_dir_all(parent)?; + } + + eprintln!("Link out path: `{}`.", link_path.display()); + + // Create directory symlink from the computed location → the package store path. + #[cfg(unix)] + std::os::unix::fs::symlink(package, &link_path)?; + + #[cfg(not(unix))] + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "symlink mode requires a Unix system", + )); + + eprintln!("Successfully created cache entry symlink for `{}`.", name); + Ok(()) +} + +// ─── manifest subcommand ────────────────────────────────────────────────────── + +/// Core logic for `manifest` mode. +/// +/// Reads `meta_path` as `Vec`, sets each entry's +/// `manifest.integrity` from its `hash` field (the SRI is reused as npm +/// integrity; `build_manifest` decodes it), groups by package name, builds a +/// [`PackageManifest`], and serializes it to +/// `/`. +/// +/// v1 scope: entries with `registry == Some(_)` are skipped with a diagnostic. +fn run_manifest(out: &Path, meta_path: &Path) -> io::Result<()> { + let content = fs::read_to_string(meta_path)?; + let mut entries: Vec = + serde_json::from_str(&content).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + // Group entries by package name, populating integrity from the Nix hash. + let mut by_name: BTreeMap> = BTreeMap::new(); + + for entry in &mut entries { + // Re-use the SRI hash string as the npm integrity field. + entry.manifest.integrity = entry.hash.clone(); + + // Parse the package name from "name@version". + // `rsplit_once('@')` correctly handles scoped packages like + // "@scope/pkg@1.2.3" → ("@scope/pkg", "1.2.3"). + let pkg_name = match entry.name_version.rsplit_once('@') { + Some((name, _ver)) => name.to_string(), + None => entry.name_version.clone(), + }; + + // v1: skip non-default registry entries. Non-default-registry packages do not + // receive a `manifest` attr in `bun.nix` (only default-registry entries are + // enriched), so in v1 this branch is effectively unreachable; it exists as a + // defensive, documented limitation. Non-default-registry manifest support is a + // future extension. + if let Some(ref reg) = entry.registry { + eprintln!( + "cache_entry_creator: skipping {}: non-default registry ({}) manifests are not supported in v1", + entry.name_version, reg + ); + continue; + } + + by_name.entry(pkg_name).or_default().push(entry.manifest.clone()); + } + + fs::create_dir_all(out)?; + + for (name, versions) in by_name { + let pkg_meta = PackageMeta { name: name.clone(), versions }; + let manifest = build_manifest(&pkg_meta); + let filename = manifest_file_name(&name); + let out_path = out.join(&filename); + let mut file = fs::File::create(&out_path)?; + serialize::write(&manifest, default_url_hash(), DEFAULT_REGISTRY_HREF_LEN, &mut file)?; + eprintln!("Wrote manifest: {}", out_path.display()); + } + + Ok(()) +} + +// ─── Tests ─────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use bun2nix_core::manifest::{manifest_file_name, read}; + + /// Create a temporary directory for a test, unique per process + test name. + fn temp_test_dir(label: &str) -> PathBuf { + let dir = std::env::temp_dir() + .join(format!("cec-test-{}-{}", label, std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("create temp dir"); + dir + } + + // ── manifest mode integration test ────────────────────────────────────── + + #[test] + fn manifest_mode_neoconfetti_round_trip() { + // Build a small EntryMeta JSON for @neoconfetti/svelte@2.2.2 using + // data from programs/bun2nix/tests/fixtures/neoconfetti.json. + let entry_meta_json = r#"[ + { + "name_version": "@neoconfetti/svelte@2.2.2", + "hash": "sha512-E7xCFVEEm5Ctnj2udTJy1b9oaTvjz1zi1mYdEtE8rB5BVwq6kHisosDS+zdWN5PMfEMjtbsOV9Cl6tsNSAD1sA==", + "registry": null, + "manifest": { + "version": "2.2.2", + "tarball_url": "https://registry.npmjs.org/@neoconfetti/svelte/-/svelte-2.2.2.tgz", + "integrity": "", + "dependencies": {}, + "peer_dependencies": { + "svelte": "^3.0.0 || ^4.0.0 || ^5.0.0" + }, + "optional_dependencies": {}, + "optional_peers": [], + "bin": {}, + "os": [], + "cpu": [], + "has_install_script": false + } + } + ]"#; + + let out_dir = temp_test_dir("manifest-neoconfetti"); + let meta_path = out_dir.join("meta.json"); + fs::write(&meta_path, entry_meta_json).unwrap(); + + // Run manifest mode. + run_manifest(&out_dir, &meta_path).expect("manifest mode should succeed"); + + // (a) Assert the output file is named exactly .npm. + let expected_filename = manifest_file_name("@neoconfetti/svelte"); + let out_file = out_dir.join(&expected_filename); + assert!( + out_file.exists(), + "expected output file `{}` does not exist; dir contents: {:?}", + out_file.display(), + fs::read_dir(&out_dir) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect::>() + ); + + // (b) Round-trip: read the file back and verify key fields. + let bytes = fs::read(&out_file).unwrap(); + let rm = read(&bytes).expect("should parse .npm file"); + + // Package name. + let name_bytes = rm.name(); + assert_eq!( + std::str::from_utf8(&name_bytes).unwrap(), + "@neoconfetti/svelte", + "package name mismatch" + ); + + // public_max_age == u32::MAX (never expire). + assert_eq!( + rm.public_max_age(), + u32::MAX, + "public_max_age should be u32::MAX" + ); + + // Version 2.2.2 present with the expected peer dependency. + let pv = rm.find_version("2.2.2").expect("version 2.2.2 should be present"); + let peers = pv.peer_dependencies(); + assert_eq!( + peers.get("svelte").map(|s| s.as_str()), + Some("^3.0.0 || ^4.0.0 || ^5.0.0"), + "svelte peer dependency range mismatch" + ); + } + + // ── symlink mode test ──────────────────────────────────────────────────── + + #[test] + #[cfg(unix)] + fn symlink_mode_creates_link_at_expected_basename() { + let out_dir = temp_test_dir("symlink-react"); + + // Create a fake "package" directory to symlink to. + let pkg_dir = temp_test_dir("symlink-react-pkg"); + fs::write(pkg_dir.join("package.json"), r#"{"name":"react"}"#).unwrap(); + + // Run symlink mode. + run_symlink(&out_dir, "react@18.3.1", &pkg_dir, None) + .expect("symlink mode should succeed"); + + // Assert the symlink exists at the expected basename. + let expected_basename = + bun2nix_core::cache_name::cached_folder_print_basename("react@18.3.1", None); + let link_path = out_dir.join(&expected_basename); + + assert!( + link_path.exists() || link_path.is_symlink(), + "symlink not found at `{}`", + link_path.display() + ); + + // The symlink should point at pkg_dir. + let target = fs::read_link(&link_path).expect("read_link"); + assert_eq!(target, pkg_dir, "symlink target mismatch"); + } + + #[test] + #[cfg(unix)] + fn symlink_mode_scoped_package_creates_parent_dir() { + let out_dir = temp_test_dir("symlink-scoped"); + let pkg_dir = temp_test_dir("symlink-scoped-pkg"); + fs::write(pkg_dir.join("package.json"), r#"{"name":"@types/node"}"#).unwrap(); + + run_symlink(&out_dir, "@types/node@20.0.0", &pkg_dir, None) + .expect("scoped symlink mode should succeed"); + + let expected_basename = + bun2nix_core::cache_name::cached_folder_print_basename("@types/node@20.0.0", None); + // A parent directory (@types/) must be created because the cached basename + // contains a '/' from the scoped package name (@types/node). + let link_path = out_dir.join(&expected_basename); + assert!( + link_path.exists() || link_path.is_symlink(), + "symlink not found at `{}`; basename=`{}`", + link_path.display(), + expected_basename + ); + } +} diff --git a/programs/cache-entry-creator/src/main.zig b/programs/cache-entry-creator/src/main.zig deleted file mode 100644 index a4e5830..0000000 --- a/programs/cache-entry-creator/src/main.zig +++ /dev/null @@ -1,355 +0,0 @@ -const std = @import("std"); -const clap = @import("clap"); - -const wyhash = @import("./wyhash.zig").Wyhash11.hash; - -const mem = std.mem; -const path = std.path; -const fs = std.fs; - -const MakeError = std.fs.Dir.MakeError; - -const wyhash_seed: u64 = 0; - -const cli_error = error{MissingOutDirFlag}; - -pub const std_options = std.Options{ - .log_level = .debug, -}; - -/// CLI help message -const cli_help = - \\ Tool for producing correctly named and positioned bun cache entries. - \\ - \\ Does the following (roughly): - \\ - Creates $out dir - \\ - Calculates the correct output location for the package - \\ - Symlinks the package contents to the calculated output location - \\ - Creates any parent directories - \\ - \\ Args: - \\ -; - -/// CLI parameters -const params = clap.parseParamsComptime( - \\--help Display this help and exit. - \\--out The $out directory to create and write to - \\--name The package name (and version) as found in `bun.lock` - \\--package The contents of the package to copy - \\--registry Optional registry hostname for non-default registries - \\ -); - -/// Clap parser string matchers -const parsers = .{ - .path = clap.parsers.string, - .str = clap.parsers.string, -}; - -/// Main entry point -pub fn main() !void { - var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - defer arena.deinit(); - - const allocator = arena.allocator(); - - var diag = clap.Diagnostic{}; - var res = clap.parse(clap.Help, ¶ms, parsers, .{ - .diagnostic = &diag, - .allocator = allocator, - .assignment_separators = "=:", - }) catch |err| { - try diag.reportToFile(.stderr(), err); - return err; - }; - defer res.deinit(); - - if (res.args.help != 0) { - std.debug.print(cli_help, .{}); - return clap.usageToFile(.stdout(), clap.Help, ¶ms); - } - - const linker = PkgLinker.init(res.args.out, res.args.name, res.args.package, res.args.registry) orelse { - std.debug.print(cli_help, .{}); - return clap.usageToFile(.stdout(), clap.Help, ¶ms); - }; - - const cache_entry_location = try cachedFolderPrintBasename( - allocator, - linker.name, - linker.registry, - ); - defer allocator.free(cache_entry_location); - - try linker.create_cache_entry(allocator, cache_entry_location); - - std.log.info("Successfully created cache entry symlink for `{s}`.\n", .{linker.name}); -} - -/// # Package Linker -/// -/// Responsible for sym-linking the packages to their resulting directory -/// in the out path -pub const PkgLinker = struct { - out: []const u8, - name: []const u8, - package: []const u8, - registry: ?[]const u8, - - /// Create a new package linker - pub fn init(out: ?[]const u8, name: ?[]const u8, package: ?[]const u8, registry: ?[]const u8) ?PkgLinker { - return PkgLinker{ - .out = out orelse return null, - .name = name orelse return null, - .package = package orelse return null, - .registry = registry, - }; - } - - /// # Create cache entry - /// - /// Creates a new cache entry at the output location passed. - /// - /// Only the leaf nodes may be symlinks hence yhis creates one of two cases: - /// - /// typescript@4.0.0 - /// - Create a symlink at $out/typescript@4.0.0 - /// - /// @types/bun - /// - Create parent directory $out/@types - /// - Create a symlink at $out/@types/bun - pub fn create_cache_entry( - linker: PkgLinker, - allocator: mem.Allocator, - cache_entry_location: []u8, - ) !void { - std.log.info("Creating entry for `{s}`...\n", .{linker.name}); - - const link_out_absolute = try std.fmt.allocPrint( - allocator, - "{s}/{s}", - .{ linker.out, cache_entry_location }, - ); - defer allocator.free(link_out_absolute); - - std.log.debug("Link out path: `{s}`.\n", .{link_out_absolute}); - - const link_parent_dir = try fs.path.resolve( - allocator, - &[_][]const u8{ link_out_absolute, ".." }, - ); - defer allocator.free(link_parent_dir); - - std.log.debug("Link parent dir: `{s}`.\n", .{link_parent_dir}); - - try fs.cwd().makePath(link_parent_dir); - std.log.debug("Created parent directory.\n", .{}); - - try fs.symLinkAbsolute( - linker.package, - link_out_absolute, - .{ .is_directory = true }, - ); - } -}; - -pub fn cachedFolderPrintBasename( - allocator: mem.Allocator, - input: []const u8, - registry: ?[]const u8, -) ![]u8 { - return if (mem.startsWith(u8, input, "tarball:")) - cachedTarballFolderPrintBasename(allocator, input) - else if (mem.startsWith(u8, input, "github:")) - cachedGithubFolderPrintBasename(allocator, input) - else if (mem.startsWith(u8, input, "git:")) - cachedGitFolderPrintBasename(allocator, input) - else - cachedNpmPackageFolderPrintBasename(allocator, input, registry); -} - -/// Produce a correct bun cache folder name for a given npm identifier -/// -/// Adapted from [here](https://github.com/oven-sh/bun/blob/134341d2b48168cbb86f74879bf6c1c8e24b799c/src/install/PackageManager/PackageManagerDirectories.zig#L288) -/// -/// When a non-default registry is used, the format includes the registry hostname: -/// e.g., `@scope/pkg@1.0.0@@npm.pkg.github.com@@@1` -pub fn cachedNpmPackageFolderPrintBasename( - allocator: mem.Allocator, - pkg: []const u8, - registry: ?[]const u8, -) ![]u8 { - // Suffix is "@@{registry}@@@1" for non-default registries, or "@@@1" for default - const suffix = if (registry) |reg| - try std.fmt.allocPrint(allocator, "@@{s}@@@1", .{reg}) - else - try allocator.dupe(u8, "@@@1"); - defer allocator.free(suffix); - - const version_start = mem.lastIndexOfScalar(u8, pkg, '@') orelse { - return std.fmt.allocPrint(allocator, "{s}{s}", .{ pkg, suffix }); - }; - const name = pkg[0..version_start]; - const ver = pkg[version_start..]; - - if (mem.indexOfScalar(u8, ver, '-')) |preIndex| { - const version = ver[0..preIndex]; - const pre_and_build = ver[preIndex + 1 ..]; - - if (mem.indexOfScalar(u8, pre_and_build, '+')) |buildIndex| { - const pre = pre_and_build[0..buildIndex]; - const build = pre_and_build[buildIndex + 1 ..]; - - return std.fmt.allocPrint(allocator, "{s}{s}-{x:0>16}+{X:0>16}{s}", .{ - name, - version, - wyhash(wyhash_seed, pre), - wyhash(wyhash_seed, build), - suffix, - }); - } - - return std.fmt.allocPrint(allocator, "{s}{s}-{x:0>16}{s}", .{ - name, - version, - wyhash(wyhash_seed, pre_and_build), - suffix, - }); - } - - if (mem.indexOfScalar(u8, ver, '+')) |buildIndex| { - const version = ver[0..buildIndex]; - const build = ver[buildIndex + 1 ..]; - - return std.fmt.allocPrint(allocator, "{s}{s}+{X:0>16}{s}", .{ - name, - version, - wyhash(wyhash_seed, build), - suffix, - }); - } - - return std.fmt.allocPrint(allocator, "{s}{s}", .{ pkg, suffix }); -} - -/// Produce a correct bun cache folder name for a given tarball dependency -/// -/// Adapted from [here](https://github.com/oven-sh/bun/blob/550522e99b303d8172b7b16c5750d458cb056434/src/install/PackageManager/PackageManagerDirectories.zig#L353) -pub fn cachedTarballFolderPrintBasename( - allocator: mem.Allocator, - url: []const u8, -) ![]u8 { - const pre = "tarball:"; - const without_pre = url[pre.len..]; - - return std.fmt.allocPrint(allocator, "@T@{x:0>16}@@@1", .{ - wyhash(wyhash_seed, without_pre), - }); -} - -/// Produce a correct bun cache folder name for a given github dependency -/// -/// Adapted from [here](https://github.com/oven-sh/bun/blob/550522e99b303d8172b7b16c5750d458cb056434/src/install/PackageManager/PackageManagerDirectories.zig#L353) -pub fn cachedGithubFolderPrintBasename( - allocator: mem.Allocator, - url: []const u8, -) ![]u8 { - const pre = "github:"; - const without_pre = url[pre.len..]; - - return std.fmt.allocPrint(allocator, "@GH@{s}@@@1", .{ - without_pre, - }); -} - -/// Produce a correct bun cache folder name for a given git dependency -/// -/// Adapted from [here](https://github.com/oven-sh/bun/blob/550522e99b303d8172b7b16c5750d458cb056434/src/install/PackageManager/PackageManagerDirectories.zig#L353) -pub fn cachedGitFolderPrintBasename( - allocator: mem.Allocator, - url: []const u8, -) ![]u8 { - const pre = "git:"; - const without_pre = url[pre.len..]; - - return std.fmt.allocPrint(allocator, "@G@{s}", .{ - without_pre, - }); -} - -const expectEqualSlices = std.testing.expectEqualSlices; -const testing_allocator = std.testing.allocator; - -fn testBaseNameFn( - tests: []const struct { []const u8, []const u8 }, - func: anytype, -) !void { - for (tests) |case| { - const input, const output = case; - - const res = try func(testing_allocator, input); - defer testing_allocator.free(res); - - try expectEqualSlices(u8, output, res); - } -} - -fn testNpmBaseNameFn( - tests: []const struct { []const u8, ?[]const u8, []const u8 }, -) !void { - for (tests) |case| { - const input, const registry, const output = case; - - const res = try cachedNpmPackageFolderPrintBasename(testing_allocator, input, registry); - defer testing_allocator.free(res); - - try expectEqualSlices(u8, output, res); - } -} - -test "cachedNpmPackageFolderPrintBasename function" { - const tests = &[_]struct { []const u8, ?[]const u8, []const u8 }{ - // Without registry (default npm registry) - .{ "react@1.2.3-beta.1+build.123", null, "react@1.2.3-c0734e9369ab610d+F48F05ED5AABC3A0@@@1" }, - .{ "tailwindcss@4.0.0-beta.9", null, "tailwindcss@4.0.0-73c5c46324e78b9b@@@1" }, - .{ "react@1.2.3+build.123", null, "react@1.2.3+F48F05ED5AABC3A0@@@1" }, - .{ "react@1.2.3", null, "react@1.2.3@@@1" }, - .{ "undici-types@6.20.0", null, "undici-types@6.20.0@@@1" }, - .{ "@types/react-dom@19.0.4", null, "@types/react-dom@19.0.4@@@1" }, - .{ "react-compiler-runtime@19.0.0-beta-e552027-20250112", null, "react-compiler-runtime@19.0.0-0f3fc645a5103715@@@1" }, - // With registry (non-default registry like GitHub Packages) - .{ "@scope/package@1.0.0", "npm.pkg.github.com", "@scope/package@1.0.0@@npm.pkg.github.com@@@1" }, - .{ "private-pkg@2.0.0", "my.registry.com", "private-pkg@2.0.0@@my.registry.com@@@1" }, - // With registry and pre-release version - .{ "@scope/pkg@1.0.0-beta.1", "npm.pkg.github.com", "@scope/pkg@1.0.0-c0734e9369ab610d@@npm.pkg.github.com@@@1" }, - // With registry and build metadata - .{ "@scope/pkg@1.0.0+build.123", "npm.pkg.github.com", "@scope/pkg@1.0.0+F48F05ED5AABC3A0@@npm.pkg.github.com@@@1" }, - }; - - try testNpmBaseNameFn(tests); -} - -test "cachedTarballFolderPrintBasename function" { - const tests = &[_]struct { []const u8, []const u8 }{ - .{ "tarball:https://registry.npmjs.org/zod/-/zod-3.21.4.tgz", "@T@3be02e19198e30ee@@@1" }, - }; - - try testBaseNameFn(tests, cachedTarballFolderPrintBasename); -} - -test "cachedGithubFolderPrintBasename function" { - const tests = &[_]struct { []const u8, []const u8 }{ - .{ "github:colinhacks-zod-f9bbb50", "@GH@colinhacks-zod-f9bbb50@@@1" }, - }; - - try testBaseNameFn(tests, cachedGithubFolderPrintBasename); -} - -test "cachedGitFolderPrintBasename function" { - const tests = &[_]struct { []const u8, []const u8 }{ - .{ "git:ee100d81f12ae315a81c2a664979a6cc1bce99a2", "@G@ee100d81f12ae315a81c2a664979a6cc1bce99a2" }, - }; - - try testBaseNameFn(tests, cachedGitFolderPrintBasename); -} diff --git a/programs/cache-entry-creator/src/wyhash.zig b/programs/cache-entry-creator/src/wyhash.zig deleted file mode 100644 index 4aae4c3..0000000 --- a/programs/cache-entry-creator/src/wyhash.zig +++ /dev/null @@ -1,181 +0,0 @@ -// -// this file is a copy of Wyhash from the zig standard library, version v0.11.0-dev.2609+5e19250a1 -// taken from https://github.com/oven-sh/bun/blob/a7816cfb23fc77db3cfe95d1f90215bbc54a58b5/src/wyhash.zig#L6 -// - -const primes = [_]u64{ - 0xa0761d6478bd642f, - 0xe7037ed1a0b428db, - 0x8ebc6af09c88c6e3, - 0x589965cc75374cc3, - 0x1d8e4e27c47d124f, -}; - -fn read_bytes(comptime bytes: u8, data: []const u8) u64 { - const T = std.meta.Int(.unsigned, 8 * bytes); - return mem.readInt(T, data[0..bytes], .little); -} - -fn read_8bytes_swapped(data: []const u8) u64 { - return (read_bytes(4, data) << 32 | read_bytes(4, data[4..])); -} - -fn mum(a: u64, b: u64) u64 { - var r = std.math.mulWide(u64, a, b); - r = (r >> 64) ^ r; - return @as(u64, @truncate(r)); -} - -fn mix0(a: u64, b: u64, seed: u64) u64 { - return mum(a ^ seed ^ primes[0], b ^ seed ^ primes[1]); -} - -fn mix1(a: u64, b: u64, seed: u64) u64 { - return mum(a ^ seed ^ primes[2], b ^ seed ^ primes[3]); -} - -// Wyhash version which does not store internal state for handling partial buffers. -// This is needed so that we can maximize the speed for the short key case, which will -// use the non-iterative api which the public Wyhash exposes. -const WyhashStateless = struct { - seed: u64, - msg_len: usize, - - pub fn init(seed: u64) WyhashStateless { - return WyhashStateless{ - .seed = seed, - .msg_len = 0, - }; - } - - inline fn round(self: *WyhashStateless, b: []const u8) void { - assert(b.len == 32); - - self.seed = mix0( - read_bytes(8, b[0..]), - read_bytes(8, b[8..]), - self.seed, - ) ^ mix1( - read_bytes(8, b[16..]), - read_bytes(8, b[24..]), - self.seed, - ); - } - - pub inline fn update(self: *WyhashStateless, b: []const u8) void { - assert(b.len % 32 == 0); - - var off: usize = 0; - while (off < b.len) : (off += 32) { - self.round(b[off .. off + 32]); - // @call(bun.callmod_inline, self.round, .{b[off .. off + 32]}); - } - - self.msg_len += b.len; - } - - pub inline fn final(self: *WyhashStateless, b: []const u8) u64 { - assert(b.len < 32); - - const seed = self.seed; - const rem_len = @as(u5, @intCast(b.len)); - const rem_key = b[0..rem_len]; - - self.seed = switch (rem_len) { - 0 => seed, - 1 => mix0(read_bytes(1, rem_key), primes[4], seed), - 2 => mix0(read_bytes(2, rem_key), primes[4], seed), - 3 => mix0((read_bytes(2, rem_key) << 8) | read_bytes(1, rem_key[2..]), primes[4], seed), - 4 => mix0(read_bytes(4, rem_key), primes[4], seed), - 5 => mix0((read_bytes(4, rem_key) << 8) | read_bytes(1, rem_key[4..]), primes[4], seed), - 6 => mix0((read_bytes(4, rem_key) << 16) | read_bytes(2, rem_key[4..]), primes[4], seed), - 7 => mix0((read_bytes(4, rem_key) << 24) | (read_bytes(2, rem_key[4..]) << 8) | read_bytes(1, rem_key[6..]), primes[4], seed), - 8 => mix0(read_8bytes_swapped(rem_key), primes[4], seed), - 9 => mix0(read_8bytes_swapped(rem_key), read_bytes(1, rem_key[8..]), seed), - 10 => mix0(read_8bytes_swapped(rem_key), read_bytes(2, rem_key[8..]), seed), - 11 => mix0(read_8bytes_swapped(rem_key), (read_bytes(2, rem_key[8..]) << 8) | read_bytes(1, rem_key[10..]), seed), - 12 => mix0(read_8bytes_swapped(rem_key), read_bytes(4, rem_key[8..]), seed), - 13 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 8) | read_bytes(1, rem_key[12..]), seed), - 14 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 16) | read_bytes(2, rem_key[12..]), seed), - 15 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 24) | (read_bytes(2, rem_key[12..]) << 8) | read_bytes(1, rem_key[14..]), seed), - 16 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed), - 17 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(1, rem_key[16..]), primes[4], seed), - 18 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(2, rem_key[16..]), primes[4], seed), - 19 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(2, rem_key[16..]) << 8) | read_bytes(1, rem_key[18..]), primes[4], seed), - 20 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(4, rem_key[16..]), primes[4], seed), - 21 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 8) | read_bytes(1, rem_key[20..]), primes[4], seed), - 22 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 16) | read_bytes(2, rem_key[20..]), primes[4], seed), - 23 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 24) | (read_bytes(2, rem_key[20..]) << 8) | read_bytes(1, rem_key[22..]), primes[4], seed), - 24 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), primes[4], seed), - 25 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(1, rem_key[24..]), seed), - 26 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(2, rem_key[24..]), seed), - 27 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(2, rem_key[24..]) << 8) | read_bytes(1, rem_key[26..]), seed), - 28 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(4, rem_key[24..]), seed), - 29 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 8) | read_bytes(1, rem_key[28..]), seed), - 30 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 16) | read_bytes(2, rem_key[28..]), seed), - 31 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 24) | (read_bytes(2, rem_key[28..]) << 8) | read_bytes(1, rem_key[30..]), seed), - }; - - self.msg_len += b.len; - return mum(self.seed ^ self.msg_len, primes[4]); - } - - pub fn hash(seed: u64, input: []const u8) u64 { - const aligned_len = input.len - (input.len % 32); - - var c = WyhashStateless.init(seed); - c.update(input[0..aligned_len]); - // @call(bun.callmod_inline, c.update, .{input[0..aligned_len]}); - return c.final(input[aligned_len..]); - // return @call(bun.callmod_inline, c.final, .{input[aligned_len..]}); - } -}; - -/// Fast non-cryptographic 64bit hash function. -/// See https://github.com/wangyi-fudan/wyhash -pub const Wyhash11 = struct { - state: WyhashStateless, - - buf: [32]u8, - buf_len: usize, - - pub fn init(seed: u64) Wyhash11 { - return Wyhash11{ - .state = WyhashStateless.init(seed), - .buf = undefined, - .buf_len = 0, - }; - } - - pub fn update(self: *Wyhash11, b: []const u8) void { - var off: usize = 0; - - if (self.buf_len != 0 and self.buf_len + b.len >= 32) { - off += 32 - self.buf_len; - mem.copyForwards(u8, self.buf[self.buf_len..], b[0..off]); - self.state.update(self.buf[0..]); - self.buf_len = 0; - } - - const remain_len = b.len - off; - const aligned_len = remain_len - (remain_len % 32); - self.state.update(b[off .. off + aligned_len]); - - mem.copyForwards(u8, self.buf[self.buf_len..], b[off + aligned_len ..]); - self.buf_len += @as(u8, @intCast(b[off + aligned_len ..].len)); - } - - pub fn final(self: *Wyhash11) u64 { - const rem_key = self.buf[0..self.buf_len]; - - return self.state.final(rem_key); - } - - pub fn hash(seed: u64, input: []const u8) u64 { - return WyhashStateless.hash(seed, input); - } -}; - -const std = @import("std"); -const mem = std.mem; -const assert = std.debug.assert; From 19b52c18dd70a5dcb80777feedbaa41d90268642 Mon Sep 17 00:00:00 2001 From: Alexis Williams Date: Thu, 23 Jul 2026 18:13:00 -0700 Subject: [PATCH 06/13] feat(bun2nix): reconstruct offline npm manifests from bun.lock The dependency-graph metadata bun stores inline in each npm lockfile entry mirrors the abbreviated registry manifest, so the deserializer rebuilds a per-version VersionMeta from it with no network access. Default-registry entries render a manifest attr in bun.nix (tarball URL, dependency groups, bin/os/cpu, install-script flag) for the offline manifest cache to consume downstream. --- programs/bun2nix/src/lib.rs | 14 +- .../src/lockfile/package_deserializer.rs | 229 +++++++++++++++++- programs/bun2nix/src/main.rs | 9 +- programs/bun2nix/src/nix_expression.rs | 149 ++++++++++++ programs/bun2nix/src/package.rs | 60 ++++- programs/bun2nix/src/package/fetcher.rs | 8 +- programs/bun2nix/src/package/manifest_nix.rs | 149 ++++++++++++ .../bun2nix/templates/output.nix_template | 2 +- .../bun2nix/tests/fixtures/neoconfetti.json | 44 ++++ 9 files changed, 646 insertions(+), 18 deletions(-) create mode 100644 programs/bun2nix/src/package/manifest_nix.rs create mode 100644 programs/bun2nix/tests/fixtures/neoconfetti.json diff --git a/programs/bun2nix/src/lib.rs b/programs/bun2nix/src/lib.rs index ef23d44..9f62564 100644 --- a/programs/bun2nix/src/lib.rs +++ b/programs/bun2nix/src/lib.rs @@ -25,14 +25,14 @@ use wasm_bindgen::prelude::*; #[cfg_attr(target_arch = "wasm32", no_mangle)] pub fn convert_lockfile_to_nix_expression(contents: String, options: Options) -> Result { let packages = build_packages(&contents)?; - - NixExpression::new(packages)?.render_with_options(options) + render_packages(packages, options) } /// # Build Packages from a Lockfile /// /// Parses a bun lockfile and produces the sorted, de-duplicated list of -/// [`Package`]s it describes. +/// [`Package`]s it describes. Every npm-registry package carries a manifest +/// reconstructed from the lockfile's inline metadata. pub fn build_packages(contents: &str) -> Result> { let lockfile = contents.parse::()?; @@ -78,6 +78,14 @@ pub fn build_packages(contents: &str) -> Result> { Ok(packages) } +/// # Render Packages to a Nix Expression +/// +/// Renders a (possibly manifest-enriched) package list into the final `bun.nix` +/// text using the supplied [`Options`]. +pub fn render_packages(packages: Vec, options: Options) -> Result { + NixExpression::new(packages)?.render_with_options(options) +} + /// Collapse `.` and `..` segments lexically (`a/b/../c` → `a/c`). fn normalize_path(path: &str) -> String { let mut parts: Vec<&str> = Vec::new(); diff --git a/programs/bun2nix/src/lockfile/package_deserializer.rs b/programs/bun2nix/src/lockfile/package_deserializer.rs index 28b8ec5..39f1bd3 100644 --- a/programs/bun2nix/src/lockfile/package_deserializer.rs +++ b/programs/bun2nix/src/lockfile/package_deserializer.rs @@ -1,3 +1,7 @@ +use std::collections::BTreeMap; + +use bun2nix_core::manifest::meta::VersionMeta; + use crate::{ Package, error::{Error, Result}, @@ -7,6 +11,101 @@ use crate::{ mod prefetch; pub use prefetch::Prefetch; +/// The dependency-graph metadata bun stores inline as element 2 of an npm +/// lockfile entry (`[id, url, meta, hash]`). Mirrors the per-version fields of +/// the abbreviated npm registry manifest, which is why it reconstructs the same +/// `VersionMeta` the network path used to fetch. +#[derive(serde::Deserialize)] +struct RawLockMeta { + #[serde(default)] + dependencies: BTreeMap, + #[serde(rename = "optionalDependencies", default)] + optional_dependencies: BTreeMap, + #[serde(rename = "peerDependencies", default)] + peer_dependencies: BTreeMap, + /// Already split out by bun (no `peerDependenciesMeta` reconstruction needed). + #[serde(rename = "optionalPeers", default)] + optional_peers: Vec, + #[serde(default)] + bin: RawBin, + #[serde(default)] + os: RawStringList, + #[serde(default)] + cpu: RawStringList, +} + +/// npm `bin` can be a bare string or a `{ name: path }` object. +#[derive(serde::Deserialize)] +#[serde(untagged)] +enum RawBin { + Str(String), + Map(BTreeMap), +} + +impl Default for RawBin { + fn default() -> Self { + RawBin::Map(BTreeMap::new()) + } +} + +/// `os`/`cpu` in bun.lock metadata can be a bare string (bun collapses +/// single-element lists, e.g. `"cpu": "x64"` or `"os": "none"`) or an array. +#[derive(serde::Deserialize)] +#[serde(untagged)] +enum RawStringList { + Str(String), + List(Vec), +} + +impl Default for RawStringList { + fn default() -> Self { + RawStringList::List(Vec::new()) + } +} + +impl From for Vec { + fn from(v: RawStringList) -> Self { + match v { + RawStringList::Str(s) => vec![s], + RawStringList::List(l) => l, + } + } +} + +impl RawLockMeta { + /// Build the registry-shaped `VersionMeta` for `@` using the + /// already-derived `tarball_url`. `integrity` is left empty (the Nix layer + /// fills it from the entry hash) and `has_install_script` is `false` (the + /// lockfile carries no install-script signal). + fn into_version_meta(self, ident: &str, tarball_url: &str) -> Result { + let (name, version) = ident + .rsplit_once('@') + .ok_or(Error::NoAtInPackageIdentifier)?; + + // For a bare-string bin, npm/bun key it by the package-name basename + // (the last path segment, dropping any `@scope/`). + let basename = name.rsplit('/').next().unwrap_or(name); + let bin = match self.bin { + RawBin::Str(path) => [(basename.to_string(), path)].into_iter().collect(), + RawBin::Map(map) => map, + }; + + Ok(VersionMeta { + version: version.to_string(), + tarball_url: tarball_url.to_string(), + integrity: String::new(), + dependencies: self.dependencies, + peer_dependencies: self.peer_dependencies, + optional_dependencies: self.optional_dependencies, + optional_peers: self.optional_peers, + bin, + os: self.os.into(), + cpu: self.cpu.into(), + has_install_script: false, + }) + } +} + type Values = Vec; /// # Package Deserializer @@ -78,7 +177,7 @@ impl PackageDeserializer { // [identifier, tarball_url, metadata, hash] // - identifier: "name@version" // - tarball_url: "" for default registry, or exact URL to tarball - // - metadata: object with dependencies, bin, etc. + // - metadata: object with dependencies, peerDependencies, bin, etc. // - hash: integrity hash (sha512-...) let npm_identifier_raw = swap_remove_value(&mut self.values, 0); @@ -87,8 +186,6 @@ impl PackageDeserializer { let hash = swap_remove_value(&mut self.values, 0); // After swap_remove(0): [meta, tarball_url] - // Get the tarball URL from what's now at index 1 - // (originally at index 1, but the metadata swapped in at index 0) let tarball_url = self .values .get(1) @@ -102,7 +199,22 @@ impl PackageDeserializer { let fetcher = Fetcher::new_npm_package(&npm_identifier_raw, hash, tarball_url)?; - Ok(Package::new(npm_identifier_raw, fetcher)) + // Default-registry npm entries (`name: None`) carry an offline manifest + // reconstructed from the inline metadata object; non-default-registry + // entries are out of scope (v1) and get none. The metadata object is now + // at index 0 (`[meta, tarball_url]`). + let manifest = if let Fetcher::FetchUrl { name: None, url, .. } = &fetcher { + let raw: RawLockMeta = serde_json::from_value(self.values.swap_remove(0))?; + Some(raw.into_version_meta(&npm_identifier_raw, url)?) + } else { + None + }; + + let package = Package::new(npm_identifier_raw, fetcher); + Ok(match manifest { + Some(m) => package.with_manifest(m), + None => package, + }) } /// # Deserialize a Github Package @@ -410,4 +522,113 @@ mod tests { ); } } + + #[test] + fn npm_entry_reconstructs_manifest_from_lockfile_meta() { + let values = vec![ + json!("react-dom@19.2.7"), + json!(""), + json!({ + "dependencies": { "scheduler": "^0.27.0" }, + "peerDependencies": { "react": "^19.2.7" } + }), + json!(SHA), + ]; + + let pkg = PackageDeserializer::deserialize_package("react-dom".into(), values).unwrap(); + let m = pkg + .manifest + .expect("a default-registry npm entry must carry a manifest"); + + assert_eq!(m.version, "19.2.7"); + assert_eq!( + m.tarball_url, + "https://registry.npmjs.org/react-dom/-/react-dom-19.2.7.tgz" + ); + assert!(m.integrity.is_empty(), "integrity is reused from the entry hash"); + assert_eq!(m.dependencies.get("scheduler"), Some(&"^0.27.0".to_string())); + assert_eq!(m.peer_dependencies.get("react"), Some(&"^19.2.7".to_string())); + assert!(m.optional_dependencies.is_empty()); + assert!(m.optional_peers.is_empty()); + assert!(m.bin.is_empty()); + assert!(m.os.is_empty()); + assert!(m.cpu.is_empty()); + assert!(!m.has_install_script); + } + + #[test] + fn optional_peers_and_object_bin_pass_through() { + let values = vec![ + json!("next@16.0.3"), + json!(""), + json!({ + "peerDependencies": { "react": "^19.0.0", "sass": "^1.3.0" }, + "optionalPeers": ["sass"], + "bin": { "next": "dist/bin/next" } + }), + json!(SHA), + ]; + + let pkg = PackageDeserializer::deserialize_package("next".into(), values).unwrap(); + let m = pkg.manifest.unwrap(); + + assert_eq!(m.optional_peers, vec!["sass".to_string()]); + assert_eq!(m.bin.get("next"), Some(&"dist/bin/next".to_string())); + } + + // bun collapses single-element os/cpu lists to bare strings in bun.lock + // (e.g. platform packages like @cloudflare/workerd-darwin-64). + #[test] + fn string_form_os_cpu_parse_as_single_element_lists() { + let values = vec![ + json!("@cloudflare/workerd-darwin-64@1.20251118.0"), + json!(""), + json!({ "os": "darwin", "cpu": "x64" }), + json!(SHA), + ]; + + let pkg = PackageDeserializer::deserialize_package( + "@cloudflare/workerd-darwin-64".into(), + values, + ) + .unwrap(); + let m = pkg.manifest.unwrap(); + + assert_eq!(m.os, vec!["darwin".to_string()]); + assert_eq!(m.cpu, vec!["x64".to_string()]); + } + + #[test] + fn bare_string_bin_normalizes_to_scoped_basename() { + let values = vec![ + json!("@neoconfetti/svelte@2.2.2"), + json!(""), + json!({ "bin": "./cli.js" }), + json!(SHA), + ]; + + let pkg = + PackageDeserializer::deserialize_package("@neoconfetti/svelte".into(), values).unwrap(); + let m = pkg.manifest.unwrap(); + + // Bare-string bin maps to { : }. + assert_eq!(m.bin.get("svelte"), Some(&"./cli.js".to_string())); + assert_eq!( + m.tarball_url, + "https://registry.npmjs.org/@neoconfetti/svelte/-/svelte-2.2.2.tgz" + ); + } + + #[test] + fn non_default_registry_entry_has_no_manifest() { + let values = vec![ + json!("foo@1.0.0"), + json!("https://npm.example.com/foo/-/foo-1.0.0.tgz"), + json!({ "dependencies": { "bar": "^1.0.0" } }), + json!(SHA), + ]; + + let pkg = PackageDeserializer::deserialize_package("foo".into(), values).unwrap(); + assert!(pkg.manifest.is_none(), "non-default registries are out of scope in v1"); + } } diff --git a/programs/bun2nix/src/main.rs b/programs/bun2nix/src/main.rs index 7af4906..8fe89cb 100644 --- a/programs/bun2nix/src/main.rs +++ b/programs/bun2nix/src/main.rs @@ -3,7 +3,7 @@ #![warn(missing_docs)] -use bun2nix::{Options, Result, convert_lockfile_to_nix_expression}; +use bun2nix::{Options, Result, build_packages, render_packages}; use log::error; use std::{ @@ -52,8 +52,10 @@ fn run() -> Result<()> { let lockfile = fs::read_to_string(&cli.lock_file)?; - let nix = convert_lockfile_to_nix_expression( - lockfile, + let packages = build_packages(&lockfile)?; + + let nix = render_packages( + packages, Options { copy_prefix: cli.copy_prefix, }, @@ -68,3 +70,4 @@ fn run() -> Result<()> { Ok(()) } + diff --git a/programs/bun2nix/src/nix_expression.rs b/programs/bun2nix/src/nix_expression.rs index 6a6cee8..657504c 100644 --- a/programs/bun2nix/src/nix_expression.rs +++ b/programs/bun2nix/src/nix_expression.rs @@ -37,3 +37,152 @@ impl NixExpression { Ok(self.render_with_values(&values)?) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::package::Fetcher; + use bun2nix_core::manifest::meta::VersionMeta; + use std::collections::BTreeMap; + + fn opts() -> Options { + Options { + copy_prefix: "./".to_string(), + } + } + + fn render(packages: Vec) -> String { + NixExpression::new(packages) + .unwrap() + .render_with_options(opts()) + .unwrap() + } + + /// A package with an attached manifest must: + /// (a) emit the manifest's `tarball_url` as the entry `url` (not inference), + /// (b) render a `manifest = { ... }` block with `tarballUrl`, `dependencies`, + /// and `optionalPeers`, and + /// (c) NOT emit an `integrity` key (it is reused downstream from `hash`). + #[test] + fn renders_manifest_block_and_prefers_tarball_url() { + let inferred = "https://registry.npmjs.org/foo/-/foo-1.0.0.tgz"; + let manifest_url = "https://registry.npmjs.org/foo/-/foo-1.0.0-DIFFERENT.tgz"; + + let fetcher = Fetcher::FetchUrl { + url: inferred.to_string(), + hash: "sha512-AAAA".to_string(), + name: None, + }; + + let vm = VersionMeta { + version: "1.0.0".to_string(), + tarball_url: manifest_url.to_string(), + integrity: "sha512-SHOULD_NOT_APPEAR".to_string(), + dependencies: BTreeMap::from([("bar".to_string(), "^1.0.0".to_string())]), + peer_dependencies: BTreeMap::new(), + optional_dependencies: BTreeMap::new(), + optional_peers: vec!["baz".to_string()], + bin: BTreeMap::new(), + os: vec![], + cpu: vec![], + has_install_script: false, + }; + + let pkg = Package::new("foo@1.0.0".to_string(), fetcher).with_manifest(vm); + let out = render(vec![pkg]); + + // (a) manifest tarball_url wins over inference + assert!( + out.contains(manifest_url), + "expected manifest tarball_url in output:\n{out}" + ); + assert!( + !out.contains(inferred), + "inferred url should have been replaced:\n{out}" + ); + + // (b) manifest block present with required keys + assert!(out.contains("manifest = {"), "missing manifest block:\n{out}"); + assert!(out.contains("tarballUrl ="), "missing tarballUrl:\n{out}"); + assert!(out.contains("dependencies ="), "missing dependencies:\n{out}"); + assert!(out.contains("optionalPeers ="), "missing optionalPeers:\n{out}"); + assert!(out.contains("\"bar\" = \"^1.0.0\""), "missing dep entry:\n{out}"); + + // (c) no integrity key (reused from entry hash downstream) + assert!( + !out.contains("integrity"), + "integrity must not be emitted:\n{out}" + ); + assert!(!out.contains("SHOULD_NOT_APPEAR"), "integrity value leaked:\n{out}"); + } + + /// A package with no manifest must render byte-identically to the + /// pre-change output (backward compatibility / offline fallback). + #[test] + fn manifest_none_renders_identically_to_before() { + let fetcher = Fetcher::FetchUrl { + url: "https://registry.npmjs.org/foo/-/foo-1.0.0.tgz".to_string(), + hash: "sha512-AAAA".to_string(), + name: None, + }; + let pkg = Package::new("foo@1.0.0".to_string(), fetcher); + let out = render(vec![pkg]); + + let expected = "# Autogenerated by `bun2nix`, editing manually is not recommended\n\ +#\n\ +# Set of Bun packages to install\n\ +#\n\ +# Consume this with `fetchBunDeps` (recommended)\n\ +# or `pkgs.callPackage` if you wish to handle\n\ +# it manually.\n\ +{\n \ +copyPathToStore,\n \ +fetchFromGitHub,\n \ +fetchgit,\n \ +fetchurl,\n \ +...\n\ +}:\n\ +{\n \ +\"foo@1.0.0\" = fetchurl {\n \ +url = \"https://registry.npmjs.org/foo/-/foo-1.0.0.tgz\";\n \ +hash = \"sha512-AAAA\";\n \ +};\n\ +}"; + + assert_eq!(out, expected); + } + + /// Nix string values must be escaped so the output is always valid Nix. + #[test] + fn escapes_nix_special_characters_in_values() { + let fetcher = Fetcher::FetchUrl { + url: "https://registry.npmjs.org/foo/-/foo-1.0.0.tgz".to_string(), + hash: "sha512-AAAA".to_string(), + name: None, + }; + let vm = VersionMeta { + version: "1.0.0".to_string(), + tarball_url: "https://registry.npmjs.org/foo/-/foo-1.0.0.tgz".to_string(), + integrity: String::new(), + // A dependency range containing characters that are special in Nix. + dependencies: BTreeMap::from([( + "weird".to_string(), + "back\\slash \"quote\" ${interp}".to_string(), + )]), + peer_dependencies: BTreeMap::new(), + optional_dependencies: BTreeMap::new(), + optional_peers: vec![], + bin: BTreeMap::new(), + os: vec![], + cpu: vec![], + has_install_script: false, + }; + let pkg = Package::new("foo@1.0.0".to_string(), fetcher).with_manifest(vm); + let out = render(vec![pkg]); + + assert!( + out.contains("back\\\\slash \\\"quote\\\" \\${interp}"), + "special characters not escaped:\n{out}" + ); + } +} diff --git a/programs/bun2nix/src/package.rs b/programs/bun2nix/src/package.rs index 421967a..541700e 100644 --- a/programs/bun2nix/src/package.rs +++ b/programs/bun2nix/src/package.rs @@ -1,15 +1,17 @@ //! This module holds the core implementation for the package type and related methods use std::{ - fmt::Debug, + fmt::{Debug, Write as _}, hash::{Hash, Hasher}, }; +use bun2nix_core::manifest::meta::VersionMeta; use serde::Serialize; mod fetcher; +mod manifest_nix; -pub use fetcher::Fetcher; +pub use fetcher::{DEFAULT_REGISTRY, Fetcher}; #[derive(Debug, Serialize, Clone)] #[serde(rename_all = "camelCase", default)] @@ -23,6 +25,15 @@ pub struct Package { /// The fetch method to use for the package pub fetcher: Fetcher, + + /// Optional registry manifest metadata for this package. + /// + /// Only default-registry npm packages ever carry `Some`; git, GitHub, + /// tarball, workspace and non-default-registry packages always stay `None`. + /// When present, the rendered `bun.nix` entry gains a `manifest = { ... }` + /// attribute and its `url` is taken from the manifest's `tarball_url`. + #[serde(skip)] + pub manifest: Option, } impl Package { @@ -31,7 +42,50 @@ impl Package { /// Creates a given package using it's name /// and fetcher information pub fn new(name: String, fetcher: Fetcher) -> Self { - Self { name, fetcher } + Self { + name, + fetcher, + manifest: None, + } + } + + /// # Attach Manifest + /// + /// Attaches registry manifest metadata to this package. When the fetcher is + /// an npm `FetchUrl`, its `url` is overwritten with the manifest's + /// authoritative `tarball_url` (the `hash` is left untouched). For any other + /// fetcher the url is left as-is. + pub fn with_manifest(mut self, manifest: VersionMeta) -> Self { + if let Fetcher::FetchUrl { url, .. } = &mut self.fetcher { + url.clone_from(&manifest.tarball_url); + } + + self.manifest = Some(manifest); + self + } + + /// # Manifest Nix Suffix + /// + /// Renders the trailing `// { manifest = { ... }; }` attrset-union suffix + /// for this package's `bun.nix` entry, or an empty string when there is no + /// manifest. Invoked directly from the output template. + /// + /// The output is deterministic (all maps are `BTreeMap`s) and every string + /// value/key is escaped for Nix. No `integrity` key is emitted — it is + /// reconstructed downstream from the entry `hash`. + pub fn manifest_nix(&self) -> String { + match &self.manifest { + None => String::new(), + Some(meta) => { + let mut out = String::new(); + // Indent so the block aligns under the entry (entry is at 2 + // spaces, the fetcher's closing brace at 2 spaces). + let _ = write!(out, " // {{\n manifest = "); + manifest_nix::render_version_meta(&mut out, meta, 4); + out.push_str(";\n }"); + out + } + } } } diff --git a/programs/bun2nix/src/package/fetcher.rs b/programs/bun2nix/src/package/fetcher.rs index 06bfba5..1b773c1 100644 --- a/programs/bun2nix/src/package/fetcher.rs +++ b/programs/bun2nix/src/package/fetcher.rs @@ -138,10 +138,10 @@ impl Fetcher { /// ``` pub fn to_npm_url(ident: &str, tarball_url: Option<&str>) -> Result { // If an explicit tarball URL is provided, use it directly - if let Some(url) = tarball_url { - if !url.is_empty() { - return Ok(url.to_string()); - } + if let Some(url) = tarball_url + && !url.is_empty() + { + return Ok(url.to_string()); } // Otherwise, construct the URL from the default registry diff --git a/programs/bun2nix/src/package/manifest_nix.rs b/programs/bun2nix/src/package/manifest_nix.rs new file mode 100644 index 0000000..f251c01 --- /dev/null +++ b/programs/bun2nix/src/package/manifest_nix.rs @@ -0,0 +1,149 @@ +//! Deterministic Nix-attrset rendering for registry [`VersionMeta`]. +//! +//! Rendering the nested maps/lists of a manifest entry as a Nix attribute set +//! is awkward in a template, so it is done here in Rust where quoting, string +//! escaping and ordering can be controlled precisely. The result is spliced +//! into the output template verbatim (the template escaper is a no-op). + +use std::collections::BTreeMap; +use std::fmt::Write as _; + +use bun2nix_core::manifest::meta::VersionMeta; + +/// Escape a string so it is a valid Nix double-quoted string body. +/// +/// * `\` → `\\` +/// * `"` → `\"` +/// * `${` → `\${` (otherwise Nix would treat it as antiquotation) +fn escape_nix_string(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '\\' => out.push_str("\\\\"), + '"' => out.push_str("\\\""), + '$' if chars.peek() == Some(&'{') => { + // Escape the antiquotation opener so `${` is literal. + out.push_str("\\${"); + chars.next(); + } + other => out.push(other), + } + } + out +} + +/// Render a quoted, escaped Nix string literal (`"..."`). +fn nix_string(s: &str) -> String { + format!("\"{}\"", escape_nix_string(s)) +} + +/// Render a `BTreeMap` as a Nix attribute set at the given +/// indentation. Empty maps render as `{ }`; keys and values are both escaped. +fn render_str_map(out: &mut String, map: &BTreeMap, indent: usize) { + if map.is_empty() { + out.push_str("{ }"); + return; + } + + let pad = " ".repeat(indent); + let inner = " ".repeat(indent + 2); + out.push_str("{\n"); + for (k, v) in map { + let _ = writeln!(out, "{inner}{} = {};", nix_string(k), nix_string(v)); + } + let _ = write!(out, "{pad}}}"); +} + +/// Render a `&[String]` as a Nix list. Empty lists render as `[ ]`; otherwise +/// the elements are space-separated escaped strings (`[ "a" "b" ]`). +fn render_str_list(out: &mut String, list: &[String]) { + if list.is_empty() { + out.push_str("[ ]"); + return; + } + + out.push_str("[ "); + for s in list { + out.push_str(&nix_string(s)); + out.push(' '); + } + out.push(']'); +} + +/// Render a [`VersionMeta`] as a Nix attribute set at the given indentation +/// (the column at which the opening `{` sits, used to align nested entries). +/// +/// Note: the `integrity` field is deliberately **not** emitted — it is rebuilt +/// downstream from the entry `hash`. +pub fn render_version_meta(out: &mut String, meta: &VersionMeta, indent: usize) { + let pad = " ".repeat(indent); + let inner = " ".repeat(indent + 2); + + out.push_str("{\n"); + + let _ = writeln!(out, "{inner}tarballUrl = {};", nix_string(&meta.tarball_url)); + + let _ = write!(out, "{inner}dependencies = "); + render_str_map(out, &meta.dependencies, indent + 2); + out.push_str(";\n"); + + let _ = write!(out, "{inner}peerDependencies = "); + render_str_map(out, &meta.peer_dependencies, indent + 2); + out.push_str(";\n"); + + let _ = write!(out, "{inner}optionalDependencies = "); + render_str_map(out, &meta.optional_dependencies, indent + 2); + out.push_str(";\n"); + + let _ = write!(out, "{inner}optionalPeers = "); + render_str_list(out, &meta.optional_peers); + out.push_str(";\n"); + + let _ = write!(out, "{inner}bin = "); + render_str_map(out, &meta.bin, indent + 2); + out.push_str(";\n"); + + let _ = write!(out, "{inner}os = "); + render_str_list(out, &meta.os); + out.push_str(";\n"); + + let _ = write!(out, "{inner}cpu = "); + render_str_list(out, &meta.cpu); + out.push_str(";\n"); + + let _ = writeln!(out, "{inner}hasInstallScript = {};", meta.has_install_script); + + let _ = write!(out, "{pad}}}"); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn escapes_backslash_quote_and_antiquotation() { + assert_eq!( + escape_nix_string("back\\slash \"q\" ${x}"), + "back\\\\slash \\\"q\\\" \\${x}" + ); + } + + #[test] + fn empty_collections_render_compactly() { + let mut out = String::new(); + render_str_map(&mut out, &BTreeMap::new(), 0); + assert_eq!(out, "{ }"); + + let mut out = String::new(); + render_str_list(&mut out, &[]); + assert_eq!(out, "[ ]"); + } + + #[test] + fn list_is_space_separated() { + let mut out = String::new(); + render_str_list(&mut out, &["linux".to_string(), "darwin".to_string()]); + assert_eq!(out, "[ \"linux\" \"darwin\" ]"); + } +} diff --git a/programs/bun2nix/templates/output.nix_template b/programs/bun2nix/templates/output.nix_template index 39fbda4..e5aac63 100644 --- a/programs/bun2nix/templates/output.nix_template +++ b/programs/bun2nix/templates/output.nix_template @@ -14,6 +14,6 @@ }: { {%- for pkg in packages %} - "{{ pkg.name }}" = {{ pkg.fetcher }}; + "{{ pkg.name }}" = {{ pkg.fetcher }}{{ pkg.manifest_nix() }}; {%- endfor %} } diff --git a/programs/bun2nix/tests/fixtures/neoconfetti.json b/programs/bun2nix/tests/fixtures/neoconfetti.json new file mode 100644 index 0000000..c180887 --- /dev/null +++ b/programs/bun2nix/tests/fixtures/neoconfetti.json @@ -0,0 +1,44 @@ +{ + "name": "@neoconfetti/svelte", + "dist-tags": { + "latest": "2.2.2" + }, + "versions": { + "2.2.1": { + "name": "@neoconfetti/svelte", + "version": "2.2.1", + "dependencies": {}, + "peerDependencies": { + "svelte": "^3.0.0 || ^4.0.0 || ^5.0.0" + }, + "peerDependenciesMeta": { + "svelte": { + "optional": false + } + }, + "dist": { + "integrity": "sha512-oldversionhash==", + "tarball": "https://registry.npmjs.org/@neoconfetti/svelte/-/svelte-2.2.1.tgz" + }, + "hasInstallScript": false + }, + "2.2.2": { + "name": "@neoconfetti/svelte", + "version": "2.2.2", + "dependencies": {}, + "peerDependencies": { + "svelte": "^3.0.0 || ^4.0.0 || ^5.0.0" + }, + "peerDependenciesMeta": { + "svelte": { + "optional": false + } + }, + "dist": { + "integrity": "sha512-E7xCFVEEm5Ctnj2udTJy1b9oaTvjz1zi1mYdEtE8rB5BVwq6kHisosDS+zdWN5PMfEMjtbsOV9Cl6tsNSAD1sA==", + "tarball": "https://registry.npmjs.org/@neoconfetti/svelte/-/svelte-2.2.2.tgz" + }, + "hasInstallScript": false + } + } +} From fa9747ab3ad7721f54f96c1c6bce615d4dfdbb69 Mon Sep 17 00:00:00 2001 From: Alexis Williams Date: Thu, 23 Jul 2026 18:13:10 -0700 Subject: [PATCH 07/13] feat(nix): synthesize offline .npm manifest cache in fetchBunDeps Collect EntryMeta records from bun.nix manifest attrs and run cache_entry_creator manifest to write the .npm files bun consults at resolve time, merged into the dependency cache. The install hook exports BUN_MANIFEST_CACHE=2 so bun reads the on-disk manifest cache instead of hitting the network. Manifest-less bun.nix files yield an empty cache and build as before. --- nix/fetch-bun-deps.nix | 61 +++++++++++++++++++++++++++++++++++++-- nix/mk-derivation/hook.sh | 5 ++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/nix/fetch-bun-deps.nix b/nix/fetch-bun-deps.nix index 414d92b..2701f7c 100644 --- a/nix/fetch-bun-deps.nix +++ b/nix/fetch-bun-deps.nix @@ -158,7 +158,12 @@ in }; config.perSystem = - { pkgs, config, ... }: + { + pkgs, + config, + self', + ... + }: { fetchBunDeps.function = { @@ -186,6 +191,57 @@ in buildPackage = config.fetchBunDeps.buildPackage args; overridePackage = config.fetchBunDeps.overridePackage args; + + # Collect `EntryMeta` records for every default-registry package that + # carries a `manifest` attr (added by newer `bun2nix`). The + # `cache_entry_creator manifest` subcommand turns these into the + # synthesized `.npm` files bun consults at resolve time. + # + # The `bun.nix` `manifest` attr is camelCase and omits `version`/ + # `integrity`; `VersionMeta` deserialises snake_case and requires both. + # `version` is parsed from the `@` key; `integrity` is + # left empty here and refilled by the tool from `hash` (the SRI string + # the `fetchurl` FOD exposes as `outputHash`). + manifestEntries = builtins.filter (e: e != null) ( + lib.mapAttrsToList ( + name: pkg: + if pkg ? manifest then + let + m = pkg.manifest; + in + { + name_version = name; + hash = pkg.outputHash or ""; + registry = null; + manifest = { + version = lib.last (lib.splitString "@" name); + tarball_url = m.tarballUrl; + integrity = ""; + dependencies = m.dependencies; + peer_dependencies = m.peerDependencies; + optional_dependencies = m.optionalDependencies; + optional_peers = m.optionalPeers; + bin = m.bin; + os = m.os; + cpu = m.cpu; + has_install_script = m.hasInstallScript; + }; + } + else + null + ) packages + ); + + manifestMeta = pkgs.writeText "bun-manifest-meta.json" (builtins.toJSON manifestEntries); + + # Always produced; for a manifest-less `bun.nix` the entry list is + # empty and this yields an empty `share/bun-cache`, which merges + # harmlessly into the symlinkJoin below (old files still build). + manifestCache = pkgs.runCommandLocal "bun-manifest-cache" { } '' + mkdir -p "$out/share/bun-cache" + "${lib.getExe self'.packages.cacheEntryCreator}" manifest \ + --out "$out/share/bun-cache" --meta ${manifestMeta} + ''; in assert lib.asserts.assertEachOneOf "overrides" (builtins.attrNames overrides) ( @@ -201,7 +257,8 @@ in (builtins.mapAttrs overridePackage) (builtins.mapAttrs buildPackage) builtins.attrValues - ]; + ] + ++ [ manifestCache ]; }; }; } diff --git a/nix/mk-derivation/hook.sh b/nix/mk-derivation/hook.sh index 01f41c6..7a1a714 100644 --- a/nix/mk-derivation/hook.sh +++ b/nix/mk-derivation/hook.sh @@ -35,6 +35,11 @@ EOF BUN_INSTALL_CACHE_DIR=$(mktemp -d) export BUN_INSTALL_CACHE_DIR + # Force bun to honor the on-disk `.npm` manifest cache at resolve time. + # Without this bun ignores the synthesized manifests entirely and tries to + # hit the network (mode 1 is insufficient; mode 2 is required — Task 3). + export BUN_MANIFEST_CACHE=2 + # Use -RL to dereference symlinks so bun finds actual directories cp -r "$bunDeps"/share/bun-cache/. "$BUN_INSTALL_CACHE_DIR" From 4419c8efe9396ad59bbce3b0a30f4b7a934d8023 Mon Sep 17 00:00:00 2001 From: Alexis Williams Date: Thu, 23 Jul 2026 18:13:10 -0700 Subject: [PATCH 08/13] test(nix): peer-dep offline install regression check (issue #71) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit react-dom@19 peer-depends on react; bun.lock is excluded from the fixture source, so bun must resolve peer deps from the synthesized manifest cache — the code path that previously fell back to the network and failed in the sandbox. flake.nix now skips auto-importing fixture bun.nix files (they are package expressions, not flake-parts modules). Also pins the manifest format version as a tripwire. --- flake.nix | 3 + nix/checks/peer-dep-offline-install.nix | 58 +++++++++++++++ .../peer-dep-offline-install/fixture/bun.lock | 20 ++++++ .../peer-dep-offline-install/fixture/bun.nix | 71 +++++++++++++++++++ .../fixture/package.json | 9 +++ 5 files changed, 161 insertions(+) create mode 100644 nix/checks/peer-dep-offline-install.nix create mode 100644 nix/checks/peer-dep-offline-install/fixture/bun.lock create mode 100644 nix/checks/peer-dep-offline-install/fixture/bun.nix create mode 100644 nix/checks/peer-dep-offline-install/fixture/package.json diff --git a/flake.nix b/flake.nix index 421d1a8..a2fbde8 100644 --- a/flake.nix +++ b/flake.nix @@ -31,6 +31,9 @@ imports = lib.pipe ./nix [ lib.filesystem.listFilesRecursive (lib.filter (lib.hasSuffix ".nix")) + # Exclude fixture subdirectories — their bun.nix files are package + # expressions (called via pkgs.callPackage), not flake-parts modules. + (lib.filter (path: !lib.hasInfix "/fixture/" (toString path))) ]; } ); diff --git a/nix/checks/peer-dep-offline-install.nix b/nix/checks/peer-dep-offline-install.nix new file mode 100644 index 0000000..7a56bbc --- /dev/null +++ b/nix/checks/peer-dep-offline-install.nix @@ -0,0 +1,58 @@ +# Regression guard for issue #71: peer-dependency resolution offline. +# +# react-dom@19 peer-depends on react. Before the manifest-cache fix, bun had +# no offline source for peer-dep manifests and fell back to the network even +# when all tarballs were cached, producing: +# error: ConnectionRefused downloading package manifest +# +# The guard is non-vacuous because bun.lock is EXCLUDED from the source (see +# filter below). Without a lockfile bun must resolve peer deps from scratch; +# it uses the synthesised .npm manifest files (from `manifest = {}` attrs in +# bun.nix, written to BUN_INSTALL_CACHE_DIR by fetchBunDeps). If those files +# are absent bun falls back to the network, which is blocked in the Nix +# sandbox → build fails. With them it succeeds in ~1 ms. +# +# Evidence: run `nix build` with bun.nix manifest attrs stripped (or with +# BUN_MANIFEST_CACHE unset) and observe: +# error: ... ConnectionRefused downloading package manifest +_: { + perSystem = + { config, ... }: + { + checks.peerDepOfflineInstall = config.mkDerivation.function { + packageJson = ./peer-dep-offline-install/fixture/package.json; + + src = builtins.path { + path = ./peer-dep-offline-install/fixture; + name = "peer-dep-offline-install-fixture-src"; + # Exclude node_modules (working-tree artefact) and bun.lock. + # Omitting bun.lock forces bun to resolve peer deps at install time + # using the synthesised manifest cache — that is the code path that + # failed in issue #71 and is guarded here. + filter = + path: _type: + let + base = builtins.baseNameOf path; + in + base != "node_modules" && base != "bun.lock"; + }; + + bunDeps = config.fetchBunDeps.function { + bunNix = ./peer-dep-offline-install/fixture/bun.nix; + }; + + # Skip lifecycle scripts and bun build — we only care that + # `bun install` resolves peer deps offline via the manifest cache. + dontRunLifecycleScripts = true; + + buildPhase = '' + echo "bun install resolved peer deps offline — manifest cache working" + ''; + + installPhase = '' + mkdir -p "$out" + echo "peerDepOfflineInstall: PASS" > "$out/result" + ''; + }; + }; +} diff --git a/nix/checks/peer-dep-offline-install/fixture/bun.lock b/nix/checks/peer-dep-offline-install/fixture/bun.lock new file mode 100644 index 0000000..d07e155 --- /dev/null +++ b/nix/checks/peer-dep-offline-install/fixture/bun.lock @@ -0,0 +1,20 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "peer-dep-offline-install-fixture", + "dependencies": { + "react": "^19.0.0", + "react-dom": "^19.0.0", + }, + }, + }, + "packages": { + "react": ["react@19.2.7", "", {}, "sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ=="], + + "react-dom": ["react-dom@19.2.7", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.7" } }, "sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ=="], + + "scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], + } +} diff --git a/nix/checks/peer-dep-offline-install/fixture/bun.nix b/nix/checks/peer-dep-offline-install/fixture/bun.nix new file mode 100644 index 0000000..f443d9c --- /dev/null +++ b/nix/checks/peer-dep-offline-install/fixture/bun.nix @@ -0,0 +1,71 @@ +# Autogenerated by `bun2nix`, editing manually is not recommended +# +# Set of Bun packages to install +# +# Consume this with `fetchBunDeps` (recommended) +# or `pkgs.callPackage` if you wish to handle +# it manually. +{ + fetchurl, + ... +}: +{ + "react-dom@19.2.7" = + fetchurl { + url = "https://registry.npmjs.org/react-dom/-/react-dom-19.2.7.tgz"; + hash = "sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ=="; + } + // { + manifest = { + tarballUrl = "https://registry.npmjs.org/react-dom/-/react-dom-19.2.7.tgz"; + dependencies = { + "scheduler" = "^0.27.0"; + }; + peerDependencies = { + "react" = "^19.2.7"; + }; + optionalDependencies = { }; + optionalPeers = [ ]; + bin = { }; + os = [ ]; + cpu = [ ]; + hasInstallScript = false; + }; + }; + "react@19.2.7" = + fetchurl { + url = "https://registry.npmjs.org/react/-/react-19.2.7.tgz"; + hash = "sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ=="; + } + // { + manifest = { + tarballUrl = "https://registry.npmjs.org/react/-/react-19.2.7.tgz"; + dependencies = { }; + peerDependencies = { }; + optionalDependencies = { }; + optionalPeers = [ ]; + bin = { }; + os = [ ]; + cpu = [ ]; + hasInstallScript = false; + }; + }; + "scheduler@0.27.0" = + fetchurl { + url = "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz"; + hash = "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="; + } + // { + manifest = { + tarballUrl = "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz"; + dependencies = { }; + peerDependencies = { }; + optionalDependencies = { }; + optionalPeers = [ ]; + bin = { }; + os = [ ]; + cpu = [ ]; + hasInstallScript = false; + }; + }; +} diff --git a/nix/checks/peer-dep-offline-install/fixture/package.json b/nix/checks/peer-dep-offline-install/fixture/package.json new file mode 100644 index 0000000..f93d7ac --- /dev/null +++ b/nix/checks/peer-dep-offline-install/fixture/package.json @@ -0,0 +1,9 @@ +{ + "name": "peer-dep-offline-install-fixture", + "version": "0.0.1", + "private": true, + "dependencies": { + "react": "^19.0.0", + "react-dom": "^19.0.0" + } +} From e1bbada9b0f78c0ca4b23adab80c6999f9a97415 Mon Sep 17 00:00:00 2001 From: Alexis Williams Date: Thu, 23 Jul 2026 18:13:23 -0700 Subject: [PATCH 09/13] feat: non-default-registry offline manifest support (issue #71) Key each package's .npm manifest by the registry bun will compute at offline-install time. A new bun2nix-core config module parses project-local bunfig.toml/.npmrc contents (.npmrc overrides per key) and ports bun's scope_for_package_name; global/env/CLI layers are deliberately ignored because the Nix sandbox's bun cannot see them either. Non-default entries carry a registry attr inside their bun.nix manifest block, cache-entry-creator groups by (name, registry) and writes -.npm with the registry's url_hash in the header, and the wasm CLI passes the two config files' contents across the boundary (no filesystem access in core). --- nix/fetch-bun-deps.nix | 31 +- programs/Cargo.lock | 356 +++++++++++++++++- programs/Cargo.toml | 18 +- programs/bun2nix-core/Cargo.toml | 12 +- programs/bun2nix-core/src/config.rs | 257 +++++++++++++ programs/bun2nix-core/src/lib.rs | 1 + programs/bun2nix-core/src/manifest/mod.rs | 37 +- programs/bun2nix-core/tests/golden.rs | 32 +- programs/bun2nix/index.ts | 26 +- programs/bun2nix/src/error.rs | 2 + programs/bun2nix/src/lib.rs | 63 +++- .../src/lockfile/package_deserializer.rs | 35 +- programs/bun2nix/src/main.rs | 32 +- programs/bun2nix/src/nix_expression.rs | 87 ++++- programs/bun2nix/src/package.rs | 18 +- programs/bun2nix/src/package/manifest_nix.rs | 25 +- programs/cache-entry-creator/src/main.rs | 140 ++++--- 17 files changed, 1053 insertions(+), 119 deletions(-) create mode 100644 programs/bun2nix-core/src/config.rs diff --git a/nix/fetch-bun-deps.nix b/nix/fetch-bun-deps.nix index 2701f7c..d1373c5 100644 --- a/nix/fetch-bun-deps.nix +++ b/nix/fetch-bun-deps.nix @@ -192,10 +192,12 @@ in buildPackage = config.fetchBunDeps.buildPackage args; overridePackage = config.fetchBunDeps.overridePackage args; - # Collect `EntryMeta` records for every default-registry package that - # carries a `manifest` attr (added by newer `bun2nix`). The + # Collect `EntryMeta` records for every package that carries a + # `manifest` attr (added by newer `bun2nix`). The # `cache_entry_creator manifest` subcommand turns these into the - # synthesized `.npm` files bun consults at resolve time. + # synthesized `.npm` files bun consults at resolve time; a + # `manifest.registry` attr (non-default registries) selects the + # registry-keyed `-.npm` filename. # # The `bun.nix` `manifest` attr is camelCase and omits `version`/ # `integrity`; `VersionMeta` deserialises snake_case and requires both. @@ -212,18 +214,18 @@ in { name_version = name; hash = pkg.outputHash or ""; - registry = null; + registry = m.registry or null; manifest = { version = lib.last (lib.splitString "@" name); tarball_url = m.tarballUrl; integrity = ""; - dependencies = m.dependencies; + inherit (m) dependencies; peer_dependencies = m.peerDependencies; optional_dependencies = m.optionalDependencies; optional_peers = m.optionalPeers; - bin = m.bin; - os = m.os; - cpu = m.cpu; + inherit (m) bin; + inherit (m) os; + inherit (m) cpu; has_install_script = m.hasInstallScript; }; } @@ -253,12 +255,13 @@ in pkgs.symlinkJoin { name = "bun-cache"; - paths = lib.pipe packages [ - (builtins.mapAttrs overridePackage) - (builtins.mapAttrs buildPackage) - builtins.attrValues - ] - ++ [ manifestCache ]; + paths = + lib.pipe packages [ + (builtins.mapAttrs overridePackage) + (builtins.mapAttrs buildPackage) + builtins.attrValues + ] + ++ [ manifestCache ]; }; }; } diff --git a/programs/Cargo.lock b/programs/Cargo.lock index be7c1b6..9142745 100644 --- a/programs/Cargo.lock +++ b/programs/Cargo.lock @@ -100,7 +100,7 @@ dependencies = [ "memchr", "serde", "serde_derive", - "winnow", + "winnow 0.7.15", ] [[package]] @@ -144,15 +144,17 @@ dependencies = [ [[package]] name = "bun2nix-core" -version = "2.1.0" +version = "2.1.2" dependencies = [ "serde", "serde_json", + "toml", + "url", ] [[package]] name = "cache-entry-creator" -version = "2.1.0" +version = "2.1.2" dependencies = [ "bun2nix-core", "clap", @@ -243,6 +245,17 @@ dependencies = [ "thiserror", ] +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "either" version = "1.16.0" @@ -272,12 +285,146 @@ dependencies = [ "log", ] +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -333,6 +480,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + [[package]] name = "log" version = "0.4.33" @@ -378,6 +531,15 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "proc-macro-error-attr2" version = "2.0.0" @@ -502,6 +664,27 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "strsim" version = "0.11.1" @@ -519,6 +702,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "thiserror" version = "2.0.18" @@ -539,12 +733,79 @@ dependencies = [ "syn", ] +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow 1.0.4", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -634,6 +895,95 @@ dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/programs/Cargo.toml b/programs/Cargo.toml index 79bd198..27cebaf 100644 --- a/programs/Cargo.toml +++ b/programs/Cargo.toml @@ -1,16 +1,18 @@ +[profile.release] +lto = true +codegen-units = 1 + [workspace] resolver = "2" members = ["bun2nix", "bun2nix-core", "cache-entry-creator"] +[workspace.dependencies] +serde = {version = "1", features = ["derive"]} +serde_json = "1" +toml = "0.9" +url = "2" + [workspace.package] version = "2.1.2" edition = "2024" license = "MIT" - -[workspace.dependencies] -serde = { version = "1", features = ["derive"] } -serde_json = "1" - -[profile.release] -lto = true -codegen-units = 1 diff --git a/programs/bun2nix-core/Cargo.toml b/programs/bun2nix-core/Cargo.toml index 40ad5c1..12b790c 100644 --- a/programs/bun2nix-core/Cargo.toml +++ b/programs/bun2nix-core/Cargo.toml @@ -1,9 +1,11 @@ +[dependencies] +serde = {workspace = true} +serde_json = {workspace = true} +toml = {workspace = true} +url = {workspace = true} + [package] name = "bun2nix-core" -version.workspace = true edition.workspace = true license.workspace = true - -[dependencies] -serde = { workspace = true } -serde_json = { workspace = true } +version.workspace = true diff --git a/programs/bun2nix-core/src/config.rs b/programs/bun2nix-core/src/config.rs new file mode 100644 index 0000000..2c8f81f --- /dev/null +++ b/programs/bun2nix-core/src/config.rs @@ -0,0 +1,257 @@ +//! Project-local bun registry configuration: parse `bunfig.toml` and `.npmrc` +//! *contents* into a [`RegistryConfig`] and resolve each package's registry the +//! way bun's `scope_for_package_name` does at install time. No filesystem +//! access — callers read the files and pass their contents. + +use std::collections::BTreeMap; +use std::fmt; + +use url::Url; + +/// The default registry href, normalized (no trailing slash). +pub const DEFAULT_REGISTRY_HREF: &str = "https://registry.npmjs.org"; + +/// Registry configuration merged from `bunfig.toml` and `.npmrc` +/// (`.npmrc` overrides per key). All hrefs are normalized via +/// [`normalize_href`]. +#[derive(Debug, Default, Clone)] +pub struct RegistryConfig { + default_href: Option, + /// Scope name (without the leading `@`) → registry href. + /// + /// bun keys its scope map by `wyhash11(scope)` with a stored-name equality + /// guard on lookup; a string-keyed map has identical observable semantics. + scopes: BTreeMap, +} + +/// Errors from parsing registry configuration. +#[derive(Debug)] +pub enum ConfigError { + /// `bunfig.toml` content was not valid TOML. + BunfigParse(String), + /// A configured registry URL failed WHATWG parsing. + InvalidRegistryUrl(String), +} + +impl fmt::Display for ConfigError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ConfigError::BunfigParse(e) => write!(f, "invalid bunfig.toml: {e}"), + ConfigError::InvalidRegistryUrl(u) => write!(f, "invalid registry URL: {u}"), + } + } +} + +impl std::error::Error for ConfigError {} + +/// WHATWG-parse a registry URL, strip credentials, and strip trailing +/// slashes/backslashes (the exact bytes bun hashes for the `.npm` key). +pub fn normalize_href(raw: &str) -> Result { + let mut url = Url::parse(raw).map_err(|_| ConfigError::InvalidRegistryUrl(raw.to_string()))?; + let _ = url.set_username(""); + let _ = url.set_password(None); + Ok(crate::manifest::without_trailing_slash(url.as_str()).to_string()) +} + +/// A bunfig registry value is either a bare URL string or a table with a +/// `url` key (plus auth fields we ignore). +fn registry_value_href(value: &toml::Value) -> Option<&str> { + match value { + toml::Value::String(s) => Some(s), + toml::Value::Table(t) => t.get("url").and_then(toml::Value::as_str), + _ => None, + } +} + +fn parse_bunfig(content: &str, cfg: &mut RegistryConfig) -> Result<(), ConfigError> { + let value: toml::Value = + toml::from_str(content).map_err(|e| ConfigError::BunfigParse(e.to_string()))?; + let install = value.get("install"); + + if let Some(href) = install + .and_then(|i| i.get("registry")) + .and_then(registry_value_href) + { + cfg.default_href = Some(normalize_href(href)?); + } + + if let Some(scopes) = install + .and_then(|i| i.get("scopes")) + .and_then(toml::Value::as_table) + { + for (key, value) in scopes { + if let Some(href) = registry_value_href(value) { + cfg.scopes.insert( + key.trim_start_matches('@').to_string(), + normalize_href(href)?, + ); + } + } + } + Ok(()) +} + +fn parse_npmrc(content: &str, cfg: &mut RegistryConfig) -> Result<(), ConfigError> { + for line in content.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') || line.starts_with(';') { + continue; + } + let Some((key, value)) = line.split_once('=') else { + continue; + }; + let key = key.trim(); + let value = value.trim().trim_matches('"').trim_matches('\''); + if value.is_empty() { + continue; + } + if key == "registry" { + cfg.default_href = Some(normalize_href(value)?); + } else if let Some(scope) = key + .strip_prefix('@') + .and_then(|k| k.strip_suffix(":registry")) + { + cfg.scopes + .insert(scope.trim().to_string(), normalize_href(value)?); + } + } + Ok(()) +} + +impl RegistryConfig { + /// Merge `bunfig.toml` then `.npmrc` contents (each optional). `.npmrc` + /// is applied second, so it overrides bunfig per key — default registry + /// and each scope independently. + pub fn parse(bunfig: Option<&str>, npmrc: Option<&str>) -> Result { + let mut cfg = RegistryConfig::default(); + if let Some(content) = bunfig { + parse_bunfig(content, &mut cfg)?; + } + if let Some(content) = npmrc { + parse_npmrc(content, &mut cfg)?; + } + Ok(cfg) + } + + /// The registry a package resolves to under this config, mirroring bun's + /// `scope_for_package_name`: scoped packages consult the per-scope map + /// first, everything else (including scope misses) uses the default + /// registry. Returns `None` when the result is the npmjs default. + pub fn scope_for_package_name(&self, package_name: &str) -> Option<&str> { + let href = if let Some(rest) = package_name.strip_prefix('@') { + let scope = rest.split('/').next().unwrap_or(rest); + self.scopes + .get(scope) + .map(String::as_str) + .or(self.default_href.as_deref()) + } else { + self.default_href.as_deref() + }; + href.filter(|h| *h != DEFAULT_REGISTRY_HREF) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn no_config_means_default_registry() { + let cfg = RegistryConfig::parse(None, None).unwrap(); + assert_eq!(cfg.scope_for_package_name("react"), None); + assert_eq!(cfg.scope_for_package_name("@types/node"), None); + } + + #[test] + fn bunfig_default_registry_string_form() { + let bunfig = "[install]\nregistry = \"https://registry.npmmirror.com/\"\n"; + let cfg = RegistryConfig::parse(Some(bunfig), None).unwrap(); + assert_eq!( + cfg.scope_for_package_name("react"), + Some("https://registry.npmmirror.com") + ); + } + + #[test] + fn bunfig_object_form_and_scopes() { + let bunfig = r#" +[install] +registry = { url = "https://registry.npmmirror.com", token = "secret" } + +[install.scopes] +"@myorg" = "https://npm.example.com/" +other = { url = "https://other.example.com" } +"#; + let cfg = RegistryConfig::parse(Some(bunfig), None).unwrap(); + assert_eq!( + cfg.scope_for_package_name("@myorg/pkg"), + Some("https://npm.example.com") + ); + // Scope keys work with or without the leading '@'. + assert_eq!( + cfg.scope_for_package_name("@other/pkg"), + Some("https://other.example.com") + ); + // Unknown scope falls back to the default registry. + assert_eq!( + cfg.scope_for_package_name("@unknown/pkg"), + Some("https://registry.npmmirror.com") + ); + } + + #[test] + fn npmrc_overrides_bunfig_per_key() { + let bunfig = "[install]\nregistry = \"https://a.example.com\"\n\n[install.scopes]\n\"@s\" = \"https://b.example.com\"\n"; + let npmrc = "registry=https://c.example.com/\n# comment\n; also a comment\n@s:registry=https://d.example.com/\n"; + let cfg = RegistryConfig::parse(Some(bunfig), Some(npmrc)).unwrap(); + assert_eq!( + cfg.scope_for_package_name("x"), + Some("https://c.example.com") + ); + assert_eq!( + cfg.scope_for_package_name("@s/x"), + Some("https://d.example.com") + ); + } + + #[test] + fn npmjs_href_resolves_to_default() { + let npmrc = "registry=https://registry.npmjs.org/\n"; + let cfg = RegistryConfig::parse(None, Some(npmrc)).unwrap(); + assert_eq!(cfg.scope_for_package_name("react"), None); + } + + #[test] + fn credentials_are_stripped_from_href() { + let npmrc = "registry=https://user:pass@npm.example.com/\n"; + let cfg = RegistryConfig::parse(None, Some(npmrc)).unwrap(); + assert_eq!( + cfg.scope_for_package_name("react"), + Some("https://npm.example.com") + ); + } + + #[test] + fn path_hrefs_keep_their_path() { + let npmrc = "registry=https://example.com/npm/\n"; + let cfg = RegistryConfig::parse(None, Some(npmrc)).unwrap(); + assert_eq!( + cfg.scope_for_package_name("react"), + Some("https://example.com/npm") + ); + } + + #[test] + fn invalid_registry_url_is_an_error() { + assert!(RegistryConfig::parse(None, Some("registry=not a url\n")).is_err()); + assert!(RegistryConfig::parse(Some("[install]\nregistry = \"::nope::\"\n"), None).is_err()); + } + + #[test] + fn default_href_matches_manifest_registry_url() { + assert_eq!( + normalize_href(crate::manifest::DEFAULT_REGISTRY_URL).unwrap(), + DEFAULT_REGISTRY_HREF + ); + } +} diff --git a/programs/bun2nix-core/src/lib.rs b/programs/bun2nix-core/src/lib.rs index 336bfce..77cc697 100644 --- a/programs/bun2nix-core/src/lib.rs +++ b/programs/bun2nix-core/src/lib.rs @@ -1,3 +1,4 @@ pub mod cache_name; +pub mod config; pub mod manifest; pub mod wyhash; diff --git a/programs/bun2nix-core/src/manifest/mod.rs b/programs/bun2nix-core/src/manifest/mod.rs index 6ecbe8b..38de42d 100644 --- a/programs/bun2nix-core/src/manifest/mod.rs +++ b/programs/bun2nix-core/src/manifest/mod.rs @@ -32,15 +32,44 @@ pub const DEFAULT_URL_HASH: u64 = 0x9c1e_4d1f_1eff_5fcd; /// the `.npm` header right after `url_hash`. pub const DEFAULT_REGISTRY_HREF_LEN: u64 = 26; +/// Strip trailing `/` and `\` from a registry href while more than one byte +/// remains — the bytes bun hashes and measures for the `.npm` header. +pub fn without_trailing_slash(href: &str) -> &str { + let bytes = href.as_bytes(); + let mut end = href.len(); + while end > 1 && matches!(bytes[end - 1], b'/' | b'\\') { + end -= 1; + } + &href[..end] +} + +/// wyhash11 of a registry href without its trailing slash — the `url_hash` +/// stored in the `.npm` header and in non-default manifest filenames. +pub fn url_hash(href: &str) -> u64 { + wyhash11(0, without_trailing_slash(href).as_bytes()) +} + +/// Registry href length (without trailing slash) stored in the `.npm` header +/// right after `url_hash`. +pub fn registry_href_len(href: &str) -> u64 { + without_trailing_slash(href).len() as u64 +} + /// wyhash11 of the default registry URL **without** the trailing slash — matches /// bun's `DEFAULT_URL_HASH`. pub fn default_url_hash() -> u64 { - wyhash11(0, DEFAULT_REGISTRY_URL.trim_end_matches('/').as_bytes()) + url_hash(DEFAULT_REGISTRY_URL) } -/// `.npm` filename for a package on the default registry (`.npm`). -pub fn manifest_file_name(name: &str) -> String { - format!("{:016x}.npm", wyhash11(0, name.as_bytes())) +/// `.npm` filename for a package: `.npm` on the +/// default registry, `-.npm` for a +/// non-default registry. +pub fn manifest_file_name(name: &str, registry: Option<&str>) -> String { + let file_id = wyhash11(0, name.as_bytes()); + match registry { + None => format!("{file_id:016x}.npm"), + Some(href) => format!("{file_id:016x}-{:016x}.npm", url_hash(href)), + } } // ────────────────────────────────────────────────────────────────────────── diff --git a/programs/bun2nix-core/tests/golden.rs b/programs/bun2nix-core/tests/golden.rs index 6c92da6..d90e5e3 100644 --- a/programs/bun2nix-core/tests/golden.rs +++ b/programs/bun2nix-core/tests/golden.rs @@ -17,7 +17,7 @@ fn default_url_hash_matches_header_constant() { /// `@neoconfetti/svelte`'s manifest filename is `wyhash11(0, name)` hex. #[test] fn neoconfetti_filename() { - let f = manifest::manifest_file_name("@neoconfetti/svelte"); + let f = manifest::manifest_file_name("@neoconfetti/svelte", None); eprintln!("@neoconfetti/svelte -> {f}"); assert!(f.ends_with(".npm")); } @@ -292,3 +292,33 @@ fn emit_neoconfetti_npm() { std::fs::write(&path, &out).unwrap(); eprintln!("wrote {} bytes to {path}", out.len()); } + +/// Golden values for a non-default registry (registry.npmmirror.com), +/// pinned against the vendored Wyhash11. +#[test] +fn npmmirror_url_hash_and_filename() { + let href = "https://registry.npmmirror.com"; + assert_eq!(manifest::url_hash(href), 0x02200d3777602379); + // Trailing slash is stripped before hashing. + assert_eq!(manifest::url_hash("https://registry.npmmirror.com/"), 0x02200d3777602379); + assert_eq!(manifest::registry_href_len(href), 30); + assert_eq!( + manifest::manifest_file_name("react", Some(href)), + "94c49019ded8e790-02200d3777602379.npm" + ); + assert_eq!(manifest::manifest_file_name("react", None), "94c49019ded8e790.npm"); +} + +/// The generalized hash must reproduce bun's DEFAULT_URL_HASH for the +/// default registry URL (which carries a trailing slash). +#[test] +fn generalized_url_hash_matches_default_constant() { + assert_eq!( + manifest::url_hash(manifest::DEFAULT_REGISTRY_URL), + manifest::DEFAULT_URL_HASH + ); + assert_eq!( + manifest::registry_href_len(manifest::DEFAULT_REGISTRY_URL), + manifest::DEFAULT_REGISTRY_HREF_LEN + ); +} diff --git a/programs/bun2nix/index.ts b/programs/bun2nix/index.ts index 1bd8917..1fd6cd0 100644 --- a/programs/bun2nix/index.ts +++ b/programs/bun2nix/index.ts @@ -2,6 +2,7 @@ import { convert_lockfile_to_nix_expression, Options } from "./bun2nix-wasm.js"; +import { dirname, join } from "node:path"; import sade from "sade"; import pkgJson from "./package.json" with { type: "json" }; @@ -17,6 +18,12 @@ type CliOpts = { "copy-prefix": string; }; +/** Read a file's text, or undefined when it does not exist. */ +async function readOptionalFile(path: string): Promise { + const file = Bun.file(path); + return (await file.exists()) ? await file.text() : undefined; +} + /** * Generate a nix expression for a given bun lockfile * Writes to stdout if `output-file` is not specified. @@ -27,9 +34,20 @@ export async function generateNixExpression(opts: CliOpts): Promise { const lock_file = Bun.file(opts["lock-file"]); const contents = await lock_file.text(); + // Project-local config only: bunfig.toml then .npmrc next to the lockfile, + // matching what bun sees at offline-install time in the Nix sandbox. + const dir = dirname(opts["lock-file"]); + const bunfig = await readOptionalFile(join(dir, "bunfig.toml")); + const npmrc = await readOptionalFile(join(dir, ".npmrc")); + const options = new Options(opts["copy-prefix"]); - const nix_expression = convertLockfileToNixExpression(contents, options); + const nix_expression = convertLockfileToNixExpression( + contents, + options, + bunfig, + npmrc, + ); const output_file = opts["output-file"] || Bun.stdout; await Bun.write(output_file, nix_expression + "\n"); @@ -61,11 +79,15 @@ prog.parse(process.argv); * * @param {string} contents - The contents of a bun lockfile * @param {Options} options - Lockfile conversion options + * @param {string} [bunfig] - Project-local bunfig.toml contents + * @param {string} [npmrc] - Project-local .npmrc contents * @return {string} The generated nix expression */ export function convertLockfileToNixExpression( contents: string, options: Options, + bunfig?: string, + npmrc?: string, ): string { - return convert_lockfile_to_nix_expression(contents, options); + return convert_lockfile_to_nix_expression(contents, options, bunfig, npmrc); } diff --git a/programs/bun2nix/src/error.rs b/programs/bun2nix/src/error.rs index b3ad3cc..9046412 100644 --- a/programs/bun2nix/src/error.rs +++ b/programs/bun2nix/src/error.rs @@ -77,6 +77,8 @@ See https://bun.sh/docs/install/lockfile to find out more information about the Try `bun2nix -h` for help. ")] ReadLockfileError(#[from] io::Error), + #[error("Failed to parse project-local registry config (bunfig.toml / .npmrc):\n{0}")] + RegistryConfig(#[from] bun2nix_core::config::ConfigError), } #[cfg(target_arch = "wasm32")] diff --git a/programs/bun2nix/src/lib.rs b/programs/bun2nix/src/lib.rs index 9f62564..02e5252 100644 --- a/programs/bun2nix/src/lib.rs +++ b/programs/bun2nix/src/lib.rs @@ -9,6 +9,7 @@ pub mod nix_expression; pub mod options; pub mod package; +pub use bun2nix_core::config::RegistryConfig; pub use error::{Error, Result}; pub use lockfile::Lockfile; use nix_expression::NixExpression; @@ -20,11 +21,20 @@ use wasm_bindgen::prelude::*; /// # Convert Bun Lockfile to a Nix expression /// -/// Takes a string input of the contents of a bun lockfile and converts it into a ready to use Nix expression which fetches the packages +/// Takes the contents of a bun lockfile — plus optional project-local +/// `bunfig.toml` / `.npmrc` contents for non-default-registry resolution — +/// and converts it into a ready to use Nix expression which fetches the +/// packages. #[cfg_attr(target_arch = "wasm32", wasm_bindgen)] #[cfg_attr(target_arch = "wasm32", no_mangle)] -pub fn convert_lockfile_to_nix_expression(contents: String, options: Options) -> Result { - let packages = build_packages(&contents)?; +pub fn convert_lockfile_to_nix_expression( + contents: String, + options: Options, + bunfig: Option, + npmrc: Option, +) -> Result { + let registry_config = RegistryConfig::parse(bunfig.as_deref(), npmrc.as_deref())?; + let packages = build_packages(&contents, ®istry_config)?; render_packages(packages, options) } @@ -32,8 +42,9 @@ pub fn convert_lockfile_to_nix_expression(contents: String, options: Options) -> /// /// Parses a bun lockfile and produces the sorted, de-duplicated list of /// [`Package`]s it describes. Every npm-registry package carries a manifest -/// reconstructed from the lockfile's inline metadata. -pub fn build_packages(contents: &str) -> Result> { +/// reconstructed from the lockfile's inline metadata, and (for scopes/packages +/// mapped to a non-default registry by `registry_config`) a `registry` href. +pub fn build_packages(contents: &str, registry_config: &RegistryConfig) -> Result> { let lockfile = contents.parse::()?; if lockfile.lockfile_version != 1 { @@ -73,6 +84,17 @@ pub fn build_packages(contents: &str) -> Result> { *path = normalize_path(&format!("{dir}/{path}")); } } + + if package.manifest.is_none() { + continue; + } + let name = package + .name + .rsplit_once('@') + .map_or(package.name.as_str(), |(name, _version)| name); + package.registry = registry_config + .scope_for_package_name(name) + .map(str::to_string); } Ok(packages) @@ -109,6 +131,35 @@ fn normalize_path(path: &str) -> String { mod tests { use super::*; + const LOCK: &str = r#"{ + "lockfileVersion": 1, + "workspaces": { "": { "name": "t" } }, + "packages": { + "react": ["react@19.2.7", "https://registry.npmmirror.com/react/-/react-19.2.7.tgz", {}, "sha512-AAAA"], + } +}"#; + + #[test] + fn build_packages_sets_registry_from_config() { + let cfg = RegistryConfig::parse( + Some("[install]\nregistry = \"https://registry.npmmirror.com\"\n"), + None, + ) + .unwrap(); + let pkgs = build_packages(LOCK, &cfg).unwrap(); + assert_eq!( + pkgs[0].registry.as_deref(), + Some("https://registry.npmmirror.com") + ); + assert!(pkgs[0].manifest.is_some()); + } + + #[test] + fn build_packages_default_config_sets_no_registry() { + let pkgs = build_packages(LOCK, &RegistryConfig::default()).unwrap(); + assert_eq!(pkgs[0].registry, None); + } + // A vendored tarball nested under a workspace ("/") is // recorded relative to the workspace dir; bun.nix needs it root-relative. #[test] @@ -127,7 +178,7 @@ mod tests { "@oc/ui/@oc/client": ["@oc/client@../app/vendor/client-1.0.0.tgz", {}, "sha512-AAAA"], } }"#; - let pkgs = build_packages(lock).unwrap(); + let pkgs = build_packages(lock, &RegistryConfig::default()).unwrap(); let path_of = |name: &str| { let p = pkgs.iter().find(|p| p.name == name).unwrap(); match &p.fetcher { diff --git a/programs/bun2nix/src/lockfile/package_deserializer.rs b/programs/bun2nix/src/lockfile/package_deserializer.rs index 39f1bd3..5c7b70a 100644 --- a/programs/bun2nix/src/lockfile/package_deserializer.rs +++ b/programs/bun2nix/src/lockfile/package_deserializer.rs @@ -199,11 +199,11 @@ impl PackageDeserializer { let fetcher = Fetcher::new_npm_package(&npm_identifier_raw, hash, tarball_url)?; - // Default-registry npm entries (`name: None`) carry an offline manifest - // reconstructed from the inline metadata object; non-default-registry - // entries are out of scope (v1) and get none. The metadata object is now - // at index 0 (`[meta, tarball_url]`). - let manifest = if let Fetcher::FetchUrl { name: None, url, .. } = &fetcher { + // Every npm entry carries an offline manifest reconstructed from the + // inline metadata object (index 0 after the swaps: `[meta, tarball_url]`). + // The tarball URL is the fetcher's: inferred for the default registry, + // verbatim from the lockfile otherwise. + let manifest = if let Fetcher::FetchUrl { url, .. } = &fetcher { let raw: RawLockMeta = serde_json::from_value(self.values.swap_remove(0))?; Some(raw.into_version_meta(&npm_identifier_raw, url)?) } else { @@ -545,9 +545,18 @@ mod tests { m.tarball_url, "https://registry.npmjs.org/react-dom/-/react-dom-19.2.7.tgz" ); - assert!(m.integrity.is_empty(), "integrity is reused from the entry hash"); - assert_eq!(m.dependencies.get("scheduler"), Some(&"^0.27.0".to_string())); - assert_eq!(m.peer_dependencies.get("react"), Some(&"^19.2.7".to_string())); + assert!( + m.integrity.is_empty(), + "integrity is reused from the entry hash" + ); + assert_eq!( + m.dependencies.get("scheduler"), + Some(&"^0.27.0".to_string()) + ); + assert_eq!( + m.peer_dependencies.get("react"), + Some(&"^19.2.7".to_string()) + ); assert!(m.optional_dependencies.is_empty()); assert!(m.optional_peers.is_empty()); assert!(m.bin.is_empty()); @@ -620,7 +629,7 @@ mod tests { } #[test] - fn non_default_registry_entry_has_no_manifest() { + fn non_default_registry_entry_reconstructs_manifest_with_lockfile_url() { let values = vec![ json!("foo@1.0.0"), json!("https://npm.example.com/foo/-/foo-1.0.0.tgz"), @@ -629,6 +638,12 @@ mod tests { ]; let pkg = PackageDeserializer::deserialize_package("foo".into(), values).unwrap(); - assert!(pkg.manifest.is_none(), "non-default registries are out of scope in v1"); + let m = pkg + .manifest + .expect("non-default-registry npm entries carry a manifest too"); + + // The explicit lockfile URL is used verbatim as the tarball URL. + assert_eq!(m.tarball_url, "https://npm.example.com/foo/-/foo-1.0.0.tgz"); + assert_eq!(m.dependencies.get("bar"), Some(&"^1.0.0".to_string())); } } diff --git a/programs/bun2nix/src/main.rs b/programs/bun2nix/src/main.rs index 8fe89cb..807cc57 100644 --- a/programs/bun2nix/src/main.rs +++ b/programs/bun2nix/src/main.rs @@ -3,13 +3,13 @@ #![warn(missing_docs)] -use bun2nix::{Options, Result, build_packages, render_packages}; -use log::error; +use bun2nix::{Options, RegistryConfig, Result, build_packages, render_packages}; +use log::{error, warn}; use std::{ fs::{self, File}, io::Write, - path::PathBuf, + path::{Path, PathBuf}, }; use clap::Parser; @@ -33,6 +33,20 @@ pub struct Cli { copy_prefix: String, } +/// Read an optional project-local config file. Absence is normal; any other +/// IO failure is surfaced as a warning because silently ignoring the file +/// would generate manifest cache keys bun won't find at install time. +fn read_optional_config(path: &Path) -> Option { + match fs::read_to_string(path) { + Ok(content) => Some(content), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, + Err(e) => { + warn!("Failed to read {}: {e}", path.display()); + None + } + } +} + fn main() { let log_env = Env::default().default_filter_or("warn"); env_logger::Builder::from_env(log_env).init(); @@ -52,7 +66,16 @@ fn run() -> Result<()> { let lockfile = fs::read_to_string(&cli.lock_file)?; - let packages = build_packages(&lockfile)?; + // Project-local config only: ./bunfig.toml then ./.npmrc next to the + // lockfile (.npmrc overrides per key). Global/env/CLI layers are + // deliberately not read — the Nix sandbox's bun can't see them either, + // and the .npm cache key must match what bun computes there. + let dir = cli.lock_file.parent().unwrap_or(Path::new(".")); + let bunfig = read_optional_config(&dir.join("bunfig.toml")); + let npmrc = read_optional_config(&dir.join(".npmrc")); + let registry_config = RegistryConfig::parse(bunfig.as_deref(), npmrc.as_deref())?; + + let packages = build_packages(&lockfile, ®istry_config)?; let nix = render_packages( packages, @@ -70,4 +93,3 @@ fn run() -> Result<()> { Ok(()) } - diff --git a/programs/bun2nix/src/nix_expression.rs b/programs/bun2nix/src/nix_expression.rs index 657504c..7474f8b 100644 --- a/programs/bun2nix/src/nix_expression.rs +++ b/programs/bun2nix/src/nix_expression.rs @@ -102,18 +102,33 @@ mod tests { ); // (b) manifest block present with required keys - assert!(out.contains("manifest = {"), "missing manifest block:\n{out}"); + assert!( + out.contains("manifest = {"), + "missing manifest block:\n{out}" + ); assert!(out.contains("tarballUrl ="), "missing tarballUrl:\n{out}"); - assert!(out.contains("dependencies ="), "missing dependencies:\n{out}"); - assert!(out.contains("optionalPeers ="), "missing optionalPeers:\n{out}"); - assert!(out.contains("\"bar\" = \"^1.0.0\""), "missing dep entry:\n{out}"); + assert!( + out.contains("dependencies ="), + "missing dependencies:\n{out}" + ); + assert!( + out.contains("optionalPeers ="), + "missing optionalPeers:\n{out}" + ); + assert!( + out.contains("\"bar\" = \"^1.0.0\""), + "missing dep entry:\n{out}" + ); // (c) no integrity key (reused from entry hash downstream) assert!( !out.contains("integrity"), "integrity must not be emitted:\n{out}" ); - assert!(!out.contains("SHOULD_NOT_APPEAR"), "integrity value leaked:\n{out}"); + assert!( + !out.contains("SHOULD_NOT_APPEAR"), + "integrity value leaked:\n{out}" + ); } /// A package with no manifest must render byte-identically to the @@ -185,4 +200,66 @@ hash = \"sha512-AAAA\";\n \ "special characters not escaped:\n{out}" ); } + + /// A package with a non-default registry renders `registry = "...";` + /// inside its manifest block; a default-registry package must not. + #[test] + fn renders_registry_attr_for_non_default_entries() { + let url = "https://registry.npmmirror.com/react/-/react-19.2.7.tgz"; + let fetcher = Fetcher::FetchUrl { + url: url.to_string(), + hash: "sha512-AAAA".to_string(), + name: Some("react-19.2.7.tgz".to_string()), + }; + let vm = VersionMeta { + version: "19.2.7".to_string(), + tarball_url: url.to_string(), + integrity: String::new(), + dependencies: BTreeMap::new(), + peer_dependencies: BTreeMap::new(), + optional_dependencies: BTreeMap::new(), + optional_peers: vec![], + bin: BTreeMap::new(), + os: vec![], + cpu: vec![], + has_install_script: false, + }; + let mut pkg = Package::new("react@19.2.7".to_string(), fetcher).with_manifest(vm); + pkg.registry = Some("https://registry.npmmirror.com".to_string()); + let out = render(vec![pkg]); + + assert!( + out.contains("registry = \"https://registry.npmmirror.com\";"), + "missing registry attr:\n{out}" + ); + } + + #[test] + fn no_registry_attr_for_default_entries() { + let fetcher = Fetcher::FetchUrl { + url: "https://registry.npmjs.org/foo/-/foo-1.0.0.tgz".to_string(), + hash: "sha512-AAAA".to_string(), + name: None, + }; + let vm = VersionMeta { + version: "1.0.0".to_string(), + tarball_url: "https://registry.npmjs.org/foo/-/foo-1.0.0.tgz".to_string(), + integrity: String::new(), + dependencies: BTreeMap::new(), + peer_dependencies: BTreeMap::new(), + optional_dependencies: BTreeMap::new(), + optional_peers: vec![], + bin: BTreeMap::new(), + os: vec![], + cpu: vec![], + has_install_script: false, + }; + let pkg = Package::new("foo@1.0.0".to_string(), fetcher).with_manifest(vm); + let out = render(vec![pkg]); + + assert!( + !out.contains("registry ="), + "unexpected registry attr:\n{out}" + ); + } } diff --git a/programs/bun2nix/src/package.rs b/programs/bun2nix/src/package.rs index 541700e..438e5a4 100644 --- a/programs/bun2nix/src/package.rs +++ b/programs/bun2nix/src/package.rs @@ -28,12 +28,19 @@ pub struct Package { /// Optional registry manifest metadata for this package. /// - /// Only default-registry npm packages ever carry `Some`; git, GitHub, - /// tarball, workspace and non-default-registry packages always stay `None`. - /// When present, the rendered `bun.nix` entry gains a `manifest = { ... }` - /// attribute and its `url` is taken from the manifest's `tarball_url`. + /// Every npm-registry package carries `Some`; git, GitHub, tarball, + /// workspace and file packages stay `None`. When present, the rendered + /// `bun.nix` entry gains a `manifest = { ... }` attribute and its `url` + /// is taken from the manifest's `tarball_url`. #[serde(skip)] pub manifest: Option, + + /// Registry href for this package's manifest cache key, normalized and + /// without trailing slash. `None` ⇒ the default npmjs registry. Resolved + /// from project-local bun config, never from the tarball URL (a registry + /// may CDN-host tarballs on a different host). + #[serde(skip)] + pub registry: Option, } impl Package { @@ -46,6 +53,7 @@ impl Package { name, fetcher, manifest: None, + registry: None, } } @@ -81,7 +89,7 @@ impl Package { // Indent so the block aligns under the entry (entry is at 2 // spaces, the fetcher's closing brace at 2 spaces). let _ = write!(out, " // {{\n manifest = "); - manifest_nix::render_version_meta(&mut out, meta, 4); + manifest_nix::render_version_meta(&mut out, meta, self.registry.as_deref(), 4); out.push_str(";\n }"); out } diff --git a/programs/bun2nix/src/package/manifest_nix.rs b/programs/bun2nix/src/package/manifest_nix.rs index f251c01..ce5b681 100644 --- a/programs/bun2nix/src/package/manifest_nix.rs +++ b/programs/bun2nix/src/package/manifest_nix.rs @@ -73,16 +73,31 @@ fn render_str_list(out: &mut String, list: &[String]) { /// Render a [`VersionMeta`] as a Nix attribute set at the given indentation /// (the column at which the opening `{` sits, used to align nested entries). +/// `registry` (when `Some`) is emitted first as a `registry` attr — the +/// non-default registry href keying this entry's `.npm` manifest. /// /// Note: the `integrity` field is deliberately **not** emitted — it is rebuilt /// downstream from the entry `hash`. -pub fn render_version_meta(out: &mut String, meta: &VersionMeta, indent: usize) { +pub fn render_version_meta( + out: &mut String, + meta: &VersionMeta, + registry: Option<&str>, + indent: usize, +) { let pad = " ".repeat(indent); let inner = " ".repeat(indent + 2); out.push_str("{\n"); - let _ = writeln!(out, "{inner}tarballUrl = {};", nix_string(&meta.tarball_url)); + if let Some(href) = registry { + let _ = writeln!(out, "{inner}registry = {};", nix_string(href)); + } + + let _ = writeln!( + out, + "{inner}tarballUrl = {};", + nix_string(&meta.tarball_url) + ); let _ = write!(out, "{inner}dependencies = "); render_str_map(out, &meta.dependencies, indent + 2); @@ -112,7 +127,11 @@ pub fn render_version_meta(out: &mut String, meta: &VersionMeta, indent: usize) render_str_list(out, &meta.cpu); out.push_str(";\n"); - let _ = writeln!(out, "{inner}hasInstallScript = {};", meta.has_install_script); + let _ = writeln!( + out, + "{inner}hasInstallScript = {};", + meta.has_install_script + ); let _ = write!(out, "{pad}}}"); } diff --git a/programs/cache-entry-creator/src/main.rs b/programs/cache-entry-creator/src/main.rs index 13850ea..08bb29e 100644 --- a/programs/cache-entry-creator/src/main.rs +++ b/programs/cache-entry-creator/src/main.rs @@ -7,20 +7,18 @@ use std::{ collections::BTreeMap, - fs, - io, + fs, io, path::{Path, PathBuf}, }; use bun2nix_core::{ cache_name::cached_folder_print_basename, manifest::{ + DEFAULT_REGISTRY_HREF_LEN, build::build_manifest, - default_url_hash, - manifest_file_name, + default_url_hash, manifest_file_name, meta::{EntryMeta, PackageMeta, VersionMeta}, - serialize, - DEFAULT_REGISTRY_HREF_LEN, + registry_href_len, serialize, url_hash, }, }; use clap::{Parser, Subcommand}; @@ -63,8 +61,9 @@ enum Commands { /// Write bun `.npm` manifest cache files for a set of package entries. /// /// Reads `--meta` as a `Vec` JSON, groups entries by package - /// name, builds a manifest per package, and writes `.npm` - /// files into `--out`. + /// name and registry, builds a manifest per group, and writes + /// `.npm` (default registry) or + /// `-.npm` files into `--out`. Manifest { /// The directory to write `.npm` files into (created if absent). #[arg(long)] @@ -80,7 +79,12 @@ enum Commands { fn main() -> Result<(), Box> { let cli = Cli::parse(); match cli.command { - Commands::Symlink { out, name, package, registry } => { + Commands::Symlink { + out, + name, + package, + registry, + } => { run_symlink(&out, &name, &package, registry.as_deref())?; } Commands::Manifest { out, meta } => { @@ -99,12 +103,7 @@ fn main() -> Result<(), Box> { /// `package`. /// /// Mirrors `PkgLinker::create_cache_entry` from the Zig implementation. -fn run_symlink( - out: &Path, - name: &str, - package: &Path, - registry: Option<&str>, -) -> io::Result<()> { +fn run_symlink(out: &Path, name: &str, package: &Path, registry: Option<&str>) -> io::Result<()> { eprintln!("Creating entry for `{}`...", name); let basename = cached_folder_print_basename(name, registry); @@ -138,18 +137,18 @@ fn run_symlink( /// /// Reads `meta_path` as `Vec`, sets each entry's /// `manifest.integrity` from its `hash` field (the SRI is reused as npm -/// integrity; `build_manifest` decodes it), groups by package name, builds a -/// [`PackageManifest`], and serializes it to -/// `/`. -/// -/// v1 scope: entries with `registry == Some(_)` are skipped with a diagnostic. +/// integrity; `build_manifest` decodes it), groups by `(package name, +/// registry)`, builds a [`PackageManifest`] per group, and serializes it to +/// `/` with the registry's +/// `url_hash`/`href_len` in the header. fn run_manifest(out: &Path, meta_path: &Path) -> io::Result<()> { let content = fs::read_to_string(meta_path)?; - let mut entries: Vec = - serde_json::from_str(&content).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + let mut entries: Vec = serde_json::from_str(&content) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; - // Group entries by package name, populating integrity from the Nix hash. - let mut by_name: BTreeMap> = BTreeMap::new(); + // Group entries by (package name, registry href); each group becomes one + // `.npm` file keyed by that registry's url_hash. + let mut groups: BTreeMap<(String, Option), Vec> = BTreeMap::new(); for entry in &mut entries { // Re-use the SRI hash string as the npm integrity field. @@ -163,31 +162,28 @@ fn run_manifest(out: &Path, meta_path: &Path) -> io::Result<()> { None => entry.name_version.clone(), }; - // v1: skip non-default registry entries. Non-default-registry packages do not - // receive a `manifest` attr in `bun.nix` (only default-registry entries are - // enriched), so in v1 this branch is effectively unreachable; it exists as a - // defensive, documented limitation. Non-default-registry manifest support is a - // future extension. - if let Some(ref reg) = entry.registry { - eprintln!( - "cache_entry_creator: skipping {}: non-default registry ({}) manifests are not supported in v1", - entry.name_version, reg - ); - continue; - } - - by_name.entry(pkg_name).or_default().push(entry.manifest.clone()); + groups + .entry((pkg_name, entry.registry.clone())) + .or_default() + .push(entry.manifest.clone()); } fs::create_dir_all(out)?; - for (name, versions) in by_name { - let pkg_meta = PackageMeta { name: name.clone(), versions }; + for ((name, registry), versions) in groups { + let pkg_meta = PackageMeta { + name: name.clone(), + versions, + }; let manifest = build_manifest(&pkg_meta); - let filename = manifest_file_name(&name); + let (hash, href_len) = match registry.as_deref() { + None => (default_url_hash(), DEFAULT_REGISTRY_HREF_LEN), + Some(href) => (url_hash(href), registry_href_len(href)), + }; + let filename = manifest_file_name(&name, registry.as_deref()); let out_path = out.join(&filename); let mut file = fs::File::create(&out_path)?; - serialize::write(&manifest, default_url_hash(), DEFAULT_REGISTRY_HREF_LEN, &mut file)?; + serialize::write(&manifest, hash, href_len, &mut file)?; eprintln!("Wrote manifest: {}", out_path.display()); } @@ -203,8 +199,7 @@ mod tests { /// Create a temporary directory for a test, unique per process + test name. fn temp_test_dir(label: &str) -> PathBuf { - let dir = std::env::temp_dir() - .join(format!("cec-test-{}-{}", label, std::process::id())); + let dir = std::env::temp_dir().join(format!("cec-test-{}-{}", label, std::process::id())); let _ = fs::remove_dir_all(&dir); fs::create_dir_all(&dir).expect("create temp dir"); dir @@ -247,7 +242,7 @@ mod tests { run_manifest(&out_dir, &meta_path).expect("manifest mode should succeed"); // (a) Assert the output file is named exactly .npm. - let expected_filename = manifest_file_name("@neoconfetti/svelte"); + let expected_filename = manifest_file_name("@neoconfetti/svelte", None); let out_file = out_dir.join(&expected_filename); assert!( out_file.exists(), @@ -279,7 +274,9 @@ mod tests { ); // Version 2.2.2 present with the expected peer dependency. - let pv = rm.find_version("2.2.2").expect("version 2.2.2 should be present"); + let pv = rm + .find_version("2.2.2") + .expect("version 2.2.2 should be present"); let peers = pv.peer_dependencies(); assert_eq!( peers.get("svelte").map(|s| s.as_str()), @@ -300,8 +297,7 @@ mod tests { fs::write(pkg_dir.join("package.json"), r#"{"name":"react"}"#).unwrap(); // Run symlink mode. - run_symlink(&out_dir, "react@18.3.1", &pkg_dir, None) - .expect("symlink mode should succeed"); + run_symlink(&out_dir, "react@18.3.1", &pkg_dir, None).expect("symlink mode should succeed"); // Assert the symlink exists at the expected basename. let expected_basename = @@ -341,4 +337,52 @@ mod tests { expected_basename ); } + + #[test] + fn manifest_mode_non_default_registry_writes_keyed_file() { + let entry_meta_json = r#"[ + { + "name_version": "react@19.2.7", + "hash": "sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ==", + "registry": "https://registry.npmmirror.com", + "manifest": { + "version": "19.2.7", + "tarball_url": "https://registry.npmmirror.com/react/-/react-19.2.7.tgz", + "integrity": "", + "dependencies": {}, + "peer_dependencies": {}, + "optional_dependencies": {}, + "optional_peers": [], + "bin": {}, + "os": [], + "cpu": [], + "has_install_script": false + } + } + ]"#; + + let out_dir = temp_test_dir("manifest-npmmirror"); + let meta_path = out_dir.join("meta.json"); + fs::write(&meta_path, entry_meta_json).unwrap(); + + run_manifest(&out_dir, &meta_path).expect("manifest mode should succeed"); + + // -.npm + let out_file = out_dir.join("94c49019ded8e790-02200d3777602379.npm"); + assert!( + out_file.exists(), + "expected `{}`; dir contents: {:?}", + out_file.display(), + fs::read_dir(&out_dir) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect::>() + ); + + let rm = read(&fs::read(&out_file).unwrap()).expect("should parse .npm file"); + assert_eq!(rm.url_hash, 0x02200d3777602379, "header url_hash mismatch"); + assert_eq!(rm.href_len, 30, "header href_len mismatch"); + assert_eq!(rm.name(), b"react"); + assert!(rm.find_version("19.2.7").is_some()); + } } From 2841b6a38c23c4b969364fb435cfeeee0c2a1143 Mon Sep 17 00:00:00 2001 From: Alexis Williams Date: Thu, 23 Jul 2026 18:13:30 -0700 Subject: [PATCH 10/13] test(nix): non-default-registry offline install check (npmmirror fixture) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit End-to-end guard: the fixture pins react/react-dom to https://registry.npmmirror.com via a committed project-local bunfig.toml; bun.lock is excluded from the source, so sandbox bun must find the registry-keyed -.npm manifests under the url_hash it derives from that same config — proving the generation-time key matches byte-for-byte. --- .../non-default-registry-offline-install.nix | 53 +++++++++++++ .../fixture/bun.lock | 20 +++++ .../fixture/bun.nix | 77 +++++++++++++++++++ .../fixture/bunfig.toml | 2 + .../fixture/package.json | 9 +++ 5 files changed, 161 insertions(+) create mode 100644 nix/checks/non-default-registry-offline-install.nix create mode 100644 nix/checks/non-default-registry-offline-install/fixture/bun.lock create mode 100644 nix/checks/non-default-registry-offline-install/fixture/bun.nix create mode 100644 nix/checks/non-default-registry-offline-install/fixture/bunfig.toml create mode 100644 nix/checks/non-default-registry-offline-install/fixture/package.json diff --git a/nix/checks/non-default-registry-offline-install.nix b/nix/checks/non-default-registry-offline-install.nix new file mode 100644 index 0000000..4e176e2 --- /dev/null +++ b/nix/checks/non-default-registry-offline-install.nix @@ -0,0 +1,53 @@ +# Regression guard for issue #71 (non-default registries): offline +# peer-dependency resolution for packages resolved from a non-default npm +# registry (https://registry.npmmirror.com, pinned via the fixture's committed +# bunfig.toml). +# +# bun.lock is EXCLUDED from the source, so bun must resolve peer deps at +# install time from the synthesized manifest cache. For a non-default +# registry bun looks up `-.npm`, deriving +# the registry href from ./bunfig.toml inside the sandbox — so a green build +# proves the generation-time key (computed from the same committed config) +# matches byte-for-byte. If the registry-keyed manifests are absent or +# mis-keyed, bun falls back to the network, which the sandbox blocks → the +# build fails. +_: { + perSystem = + { config, ... }: + { + checks.nonDefaultRegistryOfflineInstall = config.mkDerivation.function { + packageJson = ./non-default-registry-offline-install/fixture/package.json; + + src = builtins.path { + path = ./non-default-registry-offline-install/fixture; + name = "non-default-registry-offline-install-fixture-src"; + # Exclude node_modules (working-tree artefact) and bun.lock (forces + # manifest-cache resolution). bunfig.toml MUST stay included — bun + # derives the manifest-cache registry key from it in the sandbox. + filter = + path: _type: + let + base = builtins.baseNameOf path; + in + base != "node_modules" && base != "bun.lock"; + }; + + bunDeps = config.fetchBunDeps.function { + bunNix = ./non-default-registry-offline-install/fixture/bun.nix; + }; + + # Skip lifecycle scripts and bun build — we only care that + # `bun install` resolves deps offline via the registry-keyed cache. + dontRunLifecycleScripts = true; + + buildPhase = '' + echo "bun install resolved non-default-registry deps offline — manifest cache working" + ''; + + installPhase = '' + mkdir -p "$out" + echo "nonDefaultRegistryOfflineInstall: PASS" > "$out/result" + ''; + }; + }; +} diff --git a/nix/checks/non-default-registry-offline-install/fixture/bun.lock b/nix/checks/non-default-registry-offline-install/fixture/bun.lock new file mode 100644 index 0000000..35d80ab --- /dev/null +++ b/nix/checks/non-default-registry-offline-install/fixture/bun.lock @@ -0,0 +1,20 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "non-default-registry-offline-install-fixture", + "dependencies": { + "react": "^19.0.0", + "react-dom": "^19.0.0", + }, + }, + }, + "packages": { + "react": ["react@19.2.8", "https://registry.npmmirror.com/react/-/react-19.2.8.tgz", {}, "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw=="], + + "react-dom": ["react-dom@19.2.8", "https://registry.npmmirror.com/react-dom/-/react-dom-19.2.8.tgz", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.8" } }, "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ=="], + + "scheduler": ["scheduler@0.27.0", "https://registry.npmmirror.com/scheduler/-/scheduler-0.27.0.tgz", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], + } +} diff --git a/nix/checks/non-default-registry-offline-install/fixture/bun.nix b/nix/checks/non-default-registry-offline-install/fixture/bun.nix new file mode 100644 index 0000000..f0178b5 --- /dev/null +++ b/nix/checks/non-default-registry-offline-install/fixture/bun.nix @@ -0,0 +1,77 @@ +# Autogenerated by `bun2nix`, editing manually is not recommended +# +# Set of Bun packages to install +# +# Consume this with `fetchBunDeps` (recommended) +# or `pkgs.callPackage` if you wish to handle +# it manually. +{ + fetchurl, + ... +}: +{ + "react-dom@19.2.8" = + fetchurl { + url = "https://registry.npmmirror.com/react-dom/-/react-dom-19.2.8.tgz"; + hash = "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ=="; + name = "react-dom-19.2.8.tgz"; + } + // { + manifest = { + registry = "https://registry.npmmirror.com"; + tarballUrl = "https://registry.npmmirror.com/react-dom/-/react-dom-19.2.8.tgz"; + dependencies = { + "scheduler" = "^0.27.0"; + }; + peerDependencies = { + "react" = "^19.2.8"; + }; + optionalDependencies = { }; + optionalPeers = [ ]; + bin = { }; + os = [ ]; + cpu = [ ]; + hasInstallScript = false; + }; + }; + "react@19.2.8" = + fetchurl { + url = "https://registry.npmmirror.com/react/-/react-19.2.8.tgz"; + hash = "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw=="; + name = "react-19.2.8.tgz"; + } + // { + manifest = { + registry = "https://registry.npmmirror.com"; + tarballUrl = "https://registry.npmmirror.com/react/-/react-19.2.8.tgz"; + dependencies = { }; + peerDependencies = { }; + optionalDependencies = { }; + optionalPeers = [ ]; + bin = { }; + os = [ ]; + cpu = [ ]; + hasInstallScript = false; + }; + }; + "scheduler@0.27.0" = + fetchurl { + url = "https://registry.npmmirror.com/scheduler/-/scheduler-0.27.0.tgz"; + hash = "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="; + name = "scheduler-0.27.0.tgz"; + } + // { + manifest = { + registry = "https://registry.npmmirror.com"; + tarballUrl = "https://registry.npmmirror.com/scheduler/-/scheduler-0.27.0.tgz"; + dependencies = { }; + peerDependencies = { }; + optionalDependencies = { }; + optionalPeers = [ ]; + bin = { }; + os = [ ]; + cpu = [ ]; + hasInstallScript = false; + }; + }; +} diff --git a/nix/checks/non-default-registry-offline-install/fixture/bunfig.toml b/nix/checks/non-default-registry-offline-install/fixture/bunfig.toml new file mode 100644 index 0000000..107f2f0 --- /dev/null +++ b/nix/checks/non-default-registry-offline-install/fixture/bunfig.toml @@ -0,0 +1,2 @@ +[install] +registry = "https://registry.npmmirror.com" diff --git a/nix/checks/non-default-registry-offline-install/fixture/package.json b/nix/checks/non-default-registry-offline-install/fixture/package.json new file mode 100644 index 0000000..8626b62 --- /dev/null +++ b/nix/checks/non-default-registry-offline-install/fixture/package.json @@ -0,0 +1,9 @@ +{ + "name": "non-default-registry-offline-install-fixture", + "version": "0.0.1", + "private": true, + "dependencies": { + "react": "^19.0.0", + "react-dom": "^19.0.0" + } +} From dabbbe59d3d62607bece1d3d368eb3810de436e0 Mon Sep 17 00:00:00 2001 From: Alexis Williams Date: Thu, 23 Jul 2026 18:13:30 -0700 Subject: [PATCH 11/13] docs: document offline peer-dependency resolution and registry support --- docs/src/SUMMARY.md | 1 + docs/src/peer-dependency-resolution.md | 55 ++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 docs/src/peer-dependency-resolution.md diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 0e6a9a9..52b9914 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -12,4 +12,5 @@ - [`fetchBunDeps`](./building-packages/fetchBunDeps.md) - [`writeBunApplication`](./building-packages/writeBunApplication.md) - [`writeBunScriptBin`](./building-packages/writeBunScriptBin.md) +- [Offline Peer Dependency Resolution](./peer-dependency-resolution.md) - [V2 Update Guide](./v2-update-guide.md) diff --git a/docs/src/peer-dependency-resolution.md b/docs/src/peer-dependency-resolution.md new file mode 100644 index 0000000..e9a3715 --- /dev/null +++ b/docs/src/peer-dependency-resolution.md @@ -0,0 +1,55 @@ +# Offline Peer Dependency Resolution + +Packages with peer dependencies — the Svelte ecosystem, React rendering libraries, and many others — require npm package _manifests_ at install time so bun can resolve those dependencies. Without manifests, `bun install` tries to contact the npm registry even when all tarballs are already cached, which fails inside the Nix sandbox and produces an error like: + +``` +error: ConnectionRefused downloading package manifest +``` + +Since v2.2.0, `bun2nix` synthesizes the manifest files bun needs, making peer-dependency-heavy packages build fully offline. This fixes [issue #71](https://github.com/nix-community/bun2nix/issues/71). + +## At Generation Time + +When you run `bun2nix` — for example via the `postinstall` hook right after `bun install` — it reconstructs each package's manifest metadata directly from `bun.lock` (no network access needed) and embeds it as a `manifest` attribute on each registry entry in the generated `bun.nix`: + +```nix +"react-dom@19.2.7" = fetchurl { + url = "https://registry.npmjs.org/react-dom/-/react-dom-19.2.7.tgz"; + hash = "sha512-..."; +} // { + manifest = { + tarballUrl = "https://registry.npmjs.org/react-dom/-/react-dom-19.2.7.tgz"; + dependencies = { "scheduler" = "^0.27.0"; }; + peerDependencies = { "react" = "^19.2.7"; }; + optionalDependencies = { }; + optionalPeers = [ ]; + bin = { }; + os = [ ]; + cpu = [ ]; + hasInstallScript = false; + }; +}; +``` + +Packages resolved from a **non-default registry** — configured in a project-local `bunfig.toml` (`[install].registry`, `[install.scopes]`) or `.npmrc` (`registry=`, `@scope:registry=`) committed next to the lockfile — additionally carry a `registry` attribute inside the `manifest` block. `bun2nix` reads only those two project-local files (`.npmrc` overriding `bunfig.toml` per key): the offline `bun install` runs in the Nix sandbox where global config, environment variables, and CLI flags do not exist, so the manifest cache key must be derived from exactly the config bun will see there. A registry configured only in `~/.npmrc` or `BUN_CONFIG_REGISTRY` cannot work offline and is deliberately ignored. + +## At Build Time + +The Nix build is fully offline — no extra setup required. + +[`fetchBunDeps`](./building-packages/fetchBunDeps.md) reads the `manifest` attributes from `bun.nix` and runs `cache-entry-creator manifest` to synthesize the binary `.npm` cache files bun looks for at resolve time. These files are merged into the dependency cache alongside the regular package tarballs. + +The [install hook](./building-packages/hook.md) automatically exports `BUN_MANIFEST_CACHE=2`, which tells bun to use the on-disk manifest cache instead of hitting the network. + +## Graceful Degradation + +Manifests are reconstructed from `bun.lock`, not fetched, so npm-registry entries always carry a `manifest` attribute — there is no network step that can fail or get skipped. Only entries that aren't npm-registry packages (git, GitHub, tarball, workspace, and file dependencies) have no `manifest`. + +Older `bun.nix` files generated before this feature, which have no `manifest` attributes at all, still build: `fetchBunDeps` synthesizes an empty manifest cache for them, matching the behavior of older releases. + +Projects that do not have peer-dependency-heavy packages are unaffected. + +## Limitations + +- **Registry authentication is not covered.** The manifest cache key excludes credentials, so private registries key correctly, but fetching auth-gated tarballs still relies on `fetchBunDeps`'s `bunfigPath`/`npmrcPath` credential support. +- **Non-default registries must be configured in committed project-local config** (`bunfig.toml` or `.npmrc` next to `bun.lock`) — see above. Since `.npmrc` commonly holds tokens and is gitignored, prefer `bunfig.toml` for the registry URL itself. From 94c859ddfbf709849ac78874a0b961641b73fb1a Mon Sep 17 00:00:00 2001 From: Alexis Williams Date: Thu, 23 Jul 2026 19:02:00 -0700 Subject: [PATCH 12/13] feat(bun2nix-core): match bun semantics for pre-release versions and os/cpu Pre-release versions were stored tag-stripped in the releases map, but bun's find_by_version only searches prereleases when the queried version has a tag, so any pre-release pin (e.g. nitro@3.0.1-alpha.1) missed the synthesized manifest and fell back to the network. Parse the full major.minor.patch[-pre][+build] form with a port of bun's Tag.parse state machine, intern pre/build tags (Tag.eql compares wyhash11 of the pre span), and split versions into the releases/prereleases maps, each sorted ascending with bun's numeric-aware order_pre (find_best_version scans from the end). os/cpu parsing now ports bun's Negatable accumulator: "any", "none", "!name" negation, and unrecognized-value handling, all of which bun writes back into bun.lock metadata. --- programs/bun2nix-core/src/manifest/build.rs | 471 +++++++++++++++++--- programs/bun2nix-core/src/manifest/mod.rs | 51 ++- programs/bun2nix-core/tests/golden.rs | 118 +++++ 3 files changed, 548 insertions(+), 92 deletions(-) diff --git a/programs/bun2nix-core/src/manifest/build.rs b/programs/bun2nix-core/src/manifest/build.rs index 5b93017..74f8b59 100644 --- a/programs/bun2nix-core/src/manifest/build.rs +++ b/programs/bun2nix-core/src/manifest/build.rs @@ -8,12 +8,14 @@ //! [`super::build_dep_group`] — the same interning primitives used by //! [`super::build_single_version`] — so string-buffer logic lives in one place. +use std::cmp::Ordering; + use super::{ StringArena, build_dep_group, layout::{ Architecture, Bin, BinValue, DistTagMap, ExternVersionMap, ExternalString, ExternalStringList, Integrity, IntegrityTag, NpmPackage, OperatingSystem, PackageVersion, - PackageVersionList, SemverString, SemverVersion, VersionSlice, + PackageVersionList, SemverString, SemverVersion, Tag, VersionSlice, }, meta::{PackageMeta, VersionMeta}, serialize::PackageManifest, @@ -46,27 +48,62 @@ pub fn build_manifest(pkg: &PackageMeta) -> PackageManifest { // Package name (inlined if ≤ 8 bytes, stored in the arena otherwise). let name_ext = arena.intern(&pkg.name); - // Sort versions ascending (deterministic output regardless of JSON order). - let mut sorted: Vec<&VersionMeta> = pkg.versions.iter().collect(); - sorted.sort_by_key(|v| parse_semver(&v.version)); - - let mut semver_versions: Vec = Vec::with_capacity(sorted.len()); - let mut package_versions: Vec = Vec::with_capacity(sorted.len()); + // Partition into releases and pre-releases: bun's `find_by_version` picks + // the map by `version.tag.has_pre()`, so a pre-release stored under + // `releases` is unreachable. Both maps index contiguous slices of the one + // shared versions buffer — releases first, then prereleases. + let mut releases: Vec<(&VersionMeta, ParsedVersion<'_>)> = Vec::new(); + let mut prereleases: Vec<(&VersionMeta, ParsedVersion<'_>)> = Vec::new(); + for vm in &pkg.versions { + let pv = parse_version(&vm.version); + if pv.pre.is_empty() { + releases.push((vm, pv)); + } else { + prereleases.push((vm, pv)); + } + } - for vm in &sorted { - let (major, minor, patch) = parse_semver(&vm.version); + // Sort ascending: bun's `find_best_version` scans each list from the end + // and takes the first satisfying version, assuming ascending order. + // Pre-release tags order by bun's dot-segment rules (numeric-aware); the + // version-string tiebreak keeps output deterministic when two tags compare + // equal (e.g. "1" vs "01"). + releases.sort_by(|a, b| { + triple(&a.1).cmp(&triple(&b.1)).then_with(|| a.0.version.cmp(&b.0.version)) + }); + prereleases.sort_by(|a, b| { + triple(&a.1) + .cmp(&triple(&b.1)) + .then_with(|| order_pre(a.1.pre, b.1.pre)) + .then_with(|| a.0.version.cmp(&b.0.version)) + }); + + let n_rel = releases.len() as u32; + let n_pre = prereleases.len() as u32; + + let mut semver_versions: Vec = Vec::with_capacity(pkg.versions.len()); + let mut package_versions: Vec = Vec::with_capacity(pkg.versions.len()); + + for (vm, parsed) in releases.iter().chain(prereleases.iter()) { + // Empty tag components stay `ExternalString::default()` (hash 0): + // `Tag::eql` compares pre hashes, and bun parses a tagless version + // query to hash 0 — interning "" would store wyhash11(0, "") instead. + let tag = Tag { + pre: intern_tag(&mut arena, parsed.pre), + build: intern_tag(&mut arena, parsed.build), + }; semver_versions.push(SemverVersion { - major, - minor, - patch, - ..SemverVersion::default() + major: parsed.major, + minor: parsed.minor, + patch: parsed.patch, + tag, }); let pv = build_one_version(vm, &mut arena, &mut names, &mut values, &mut bin_entries); package_versions.push(pv); } - let n = sorted.len() as u32; + let n = n_rel + n_pre; let n_names = names.len() as u32; let external_strings = names.into_boxed_slice(); let external_strings_for_versions = values.into_boxed_slice(); @@ -80,12 +117,16 @@ pub fn build_manifest(pkg: &PackageMeta) -> PackageManifest { ..NpmPackage::default() }; - // releases: keys = entire versions array, values = entire package_versions array. + // releases occupy [0, n_rel) of the shared buffers, prereleases + // [n_rel, n_rel + n_pre). pkg_struct.releases = ExternVersionMap { - keys: VersionSlice::new(0, n), - values: PackageVersionList::new(0, n), + keys: VersionSlice::new(0, n_rel), + values: PackageVersionList::new(0, n_rel), + }; + pkg_struct.prereleases = ExternVersionMap { + keys: VersionSlice::new(n_rel, n_pre), + values: PackageVersionList::new(n_rel, n_pre), }; - pkg_struct.prereleases = ExternVersionMap::default(); // dist_tags: left empty (v1 scope — see brief "Leave dist_tags empty"). pkg_struct.dist_tags = DistTagMap::default(); pkg_struct.versions_buf = VersionSlice::new(0, n); @@ -268,53 +309,112 @@ fn b64_val(b: u8) -> Option { // OS / CPU bit-flags // ────────────────────────────────────────────────────────────────────────── -/// Parse an npm `"os"` array into [`OperatingSystem`] bitflags. -/// An empty list means "all OSes" (`OperatingSystem::ALL`). -fn parse_os(os: &[String]) -> OperatingSystem { - if os.is_empty() { - return OperatingSystem::ALL; - } - let mut flags: u16 = 0; - for s in os { - flags |= match s.as_str() { - "aix" => OperatingSystem::AIX, - "darwin" | "macos" => OperatingSystem::DARWIN, - "freebsd" => OperatingSystem::FREEBSD, - "linux" => OperatingSystem::LINUX, - "openbsd" => OperatingSystem::OPENBSD, - "sunos" => OperatingSystem::SUNOS, - "win32" => OperatingSystem::WIN32, - "android" => OperatingSystem::ANDROID, - _ => 0, +/// Fold an npm `"os"`/`"cpu"` token list into a bitset, porting bun's +/// `Negatable` accumulator + `combine`: +/// +/// - `"any"` is a wildcard (→ all bits) and `"none"` an unrecognized value +/// (→ no bits), unless a later recognized token resets either flag; +/// - `"!name"` adds to a removed set; +/// - empty / only-unrecognized → NONE, only-removed → ALL minus removed, +/// only-added → added, mixed → added minus removed; +/// - an empty list means "all". +/// +/// bun writes these token shapes back into `bun.lock` (`"none"`, one name, +/// one negated name, or an array), so round-tripping them exactly matters. +fn combine_negatable(list: &[String], all: u16, lookup: fn(&str) -> Option) -> u16 { + let mut added: u16 = 0; + let mut removed: u16 = 0; + let mut had_wildcard = false; + let mut had_unrecognized = false; + + for s in list { + if s.is_empty() { + continue; + } + if s == "any" { + had_wildcard = true; + continue; + } + if s == "none" { + had_unrecognized = true; + continue; + } + let (is_not, name) = match s.strip_prefix('!') { + Some(rest) => (true, rest), + None => (false, s.as_str()), + }; + let Some(bit) = lookup(name) else { + if !is_not { + had_unrecognized = true; + } + continue; }; + // A recognized token resets the wildcard/unrecognized flags, so + // ["any", "linux"] collapses to LINUX. + had_wildcard = false; + had_unrecognized = false; + if is_not { + removed |= bit; + } else { + added |= bit; + } } - OperatingSystem(flags) + + let added = if had_wildcard { all } else { added }; + if added == 0 && removed == 0 { + if had_unrecognized { + return 0; + } + return all; + } + if added == 0 { + return all & !removed; + } + if removed == 0 { + return added; + } + added & !removed +} + +fn os_bit(name: &str) -> Option { + Some(match name { + "aix" => OperatingSystem::AIX, + "darwin" => OperatingSystem::DARWIN, + "freebsd" => OperatingSystem::FREEBSD, + "linux" => OperatingSystem::LINUX, + "openbsd" => OperatingSystem::OPENBSD, + "sunos" => OperatingSystem::SUNOS, + "win32" => OperatingSystem::WIN32, + "android" => OperatingSystem::ANDROID, + _ => return None, + }) } -/// Parse an npm `"cpu"` array into [`Architecture`] bitflags. -/// An empty list means "all architectures" (`Architecture::ALL`). +fn cpu_bit(name: &str) -> Option { + Some(match name { + "arm" => Architecture::ARM, + "arm64" => Architecture::ARM64, + "ia32" => Architecture::IA32, + "mips" => Architecture::MIPS, + "mipsel" => Architecture::MIPSEL, + "ppc" => Architecture::PPC, + "ppc64" => Architecture::PPC64, + "s390" => Architecture::S390, + "s390x" => Architecture::S390X, + "x32" => Architecture::X32, + "x64" => Architecture::X64, + _ => return None, + }) +} + +/// Parse an npm `"os"` list into [`OperatingSystem`] bitflags. +fn parse_os(os: &[String]) -> OperatingSystem { + OperatingSystem(combine_negatable(os, OperatingSystem::ALL_VALUE, os_bit)) +} + +/// Parse an npm `"cpu"` list into [`Architecture`] bitflags. fn parse_cpu(cpu: &[String]) -> Architecture { - if cpu.is_empty() { - return Architecture::ALL; - } - let mut flags: u16 = 0; - for s in cpu { - flags |= match s.as_str() { - "arm" => Architecture::ARM, - "arm64" => Architecture::ARM64, - "ia32" => Architecture::IA32, - "mips" => Architecture::MIPS, - "mipsel" => Architecture::MIPSEL, - "ppc" => Architecture::PPC, - "ppc64" => Architecture::PPC64, - "s390" => Architecture::S390, - "s390x" => Architecture::S390X, - "x32" => Architecture::X32, - "x64" => Architecture::X64, - _ => 0, - }; - } - Architecture(flags) + Architecture(combine_negatable(cpu, Architecture::ALL_VALUE, cpu_bit)) } // ────────────────────────────────────────────────────────────────────────── @@ -386,15 +486,250 @@ fn ss_hi(ss: &SemverString) -> u32 { // Semver helpers // ────────────────────────────────────────────────────────────────────────── -/// Parse `"major.minor.patch"` (optionally with a pre-release suffix) into a -/// sortable `(major, minor, patch)` triple. Unknown components default to 0. -pub(crate) fn parse_semver(s: &str) -> (u64, u64, u64) { +/// A version string decomposed into its numeric triple and tag components. +/// `pre`/`build` are empty when the version has no such component. +pub(crate) struct ParsedVersion<'a> { + pub(crate) major: u64, + pub(crate) minor: u64, + pub(crate) patch: u64, + pub(crate) pre: &'a str, + pub(crate) build: &'a str, +} + +/// Parse `"major.minor.patch[-pre][+build]"`. Unknown numeric components +/// default to 0; the tag portion follows bun's `Tag.parse` state machine so +/// the pre span (and therefore its hash) matches what bun computes when it +/// parses the same version at install time. +pub(crate) fn parse_version(s: &str) -> ParsedVersion<'_> { let mut parts = s.splitn(3, '.'); let major: u64 = parts.next().and_then(|p| p.parse().ok()).unwrap_or(0); let minor: u64 = parts.next().and_then(|p| p.parse().ok()).unwrap_or(0); - let patch: u64 = parts - .next() - .and_then(|p| p.split('-').next()?.parse().ok()) - .unwrap_or(0); - (major, minor, patch) + let rest = parts.next().unwrap_or(""); + let digits_end = rest.bytes().position(|b| !b.is_ascii_digit()).unwrap_or(rest.len()); + let patch: u64 = rest[..digits_end].parse().unwrap_or(0); + let (pre, build) = parse_tag(&rest[digits_end..]); + ParsedVersion { + major, + minor, + patch, + pre, + build, + } +} + +/// Extract `(pre, build)` spans from the tag portion of a version string +/// (everything after the patch digits, including the leading `-`/`+`). +/// +/// Port of bun's `Tag.parse` state machine: pre starts after the FIRST `-` +/// (later hyphens are kept inside the span, so `--canary.0` yields +/// `-canary.0`), build starts after the first `+`, and scanning stops at the +/// first character outside `[A-Za-z0-9.+-]`. +fn parse_tag(tag: &str) -> (&str, &str) { + #[derive(PartialEq, Clone, Copy)] + enum State { + None, + Pre, + Build, + } + let mut pre = ""; + let mut build = ""; + let mut state = State::None; + let mut start = 0usize; + + for (i, c) in tag.bytes().enumerate() { + match c { + b'+' => { + if state == State::Pre { + pre = &tag[start..i]; + } + if state != State::Build { + state = State::Build; + start = i + 1; + } + } + b'-' => { + if state != State::Pre { + state = State::Pre; + start = i + 1; + } + } + b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'.' => {} + _ => { + match state { + State::None => {} + State::Pre => pre = &tag[start..i], + State::Build => build = &tag[start..i], + } + return (pre, build); + } + } + } + match state { + State::None => {} + State::Pre => pre = &tag[start..], + State::Build => build = &tag[start..], + } + (pre, build) +} + +/// Intern a tag component, or return the all-zero `ExternalString` (hash 0) +/// when the component is absent — the value bun's `Tag.eql` expects for a +/// tagless version. +fn intern_tag(arena: &mut StringArena, s: &str) -> ExternalString { + if s.is_empty() { + ExternalString::default() + } else { + arena.intern(s) + } +} + +#[inline] +fn triple(p: &ParsedVersion<'_>) -> (u64, u64, u64) { + (p.major, p.minor, p.patch) +} + +/// Order two pre-release tags by bun's `Tag.order_pre` rules: split on `.`, +/// compare segments numerically when both parse as integers, otherwise +/// bytewise (a numeric segment sorts before a non-numeric one); a longer tag +/// that shares its prefix sorts after the shorter one. +fn order_pre(lhs: &str, rhs: &str) -> Ordering { + let mut lhs_itr = lhs.split('.'); + let mut rhs_itr = rhs.split('.'); + loop { + match (lhs_itr.next(), rhs_itr.next()) { + (None, None) => return Ordering::Equal, + (Some(_), None) => return Ordering::Greater, + (None, Some(_)) => return Ordering::Less, + (Some(l), Some(r)) => { + let l_uint: Option = l.parse().ok(); + let r_uint: Option = r.parse().ok(); + match (l_uint, r_uint) { + (Some(_), None) => return Ordering::Less, + (None, Some(_)) => return Ordering::Greater, + (Some(l), Some(r)) => match l.cmp(&r) { + Ordering::Equal => continue, + not_equal => return not_equal, + }, + (None, None) => match l.cmp(r) { + Ordering::Equal => continue, + not_equal => return not_equal, + }, + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_version_release() { + let p = parse_version("1.2.3"); + assert_eq!((p.major, p.minor, p.patch), (1, 2, 3)); + assert_eq!(p.pre, ""); + assert_eq!(p.build, ""); + } + + #[test] + fn parse_version_pre() { + let p = parse_version("3.0.1-alpha.1"); + assert_eq!((p.major, p.minor, p.patch), (3, 0, 1)); + assert_eq!(p.pre, "alpha.1"); + assert_eq!(p.build, ""); + } + + #[test] + fn parse_version_pre_and_build() { + let p = parse_version("1.2.3-beta.2+exp.sha5114f8"); + assert_eq!((p.major, p.minor, p.patch), (1, 2, 3)); + assert_eq!(p.pre, "beta.2"); + assert_eq!(p.build, "exp.sha5114f8"); + } + + // bun keeps hyphens after the first: `--canary.67e7966.0` is the pre tag + // `-canary.67e7966.0` (see the comment in bun's Tag.parse). + #[test] + fn parse_version_double_hyphen() { + let p = parse_version("1.0.0--canary.67e7966.0"); + assert_eq!((p.major, p.minor, p.patch), (1, 0, 0)); + assert_eq!(p.pre, "-canary.67e7966.0"); + } + + #[test] + fn parse_version_build_only() { + let p = parse_version("1.0.0+20130313144700"); + assert_eq!(p.pre, ""); + assert_eq!(p.build, "20130313144700"); + } + + // The example from bun's order_pre comment: + // 1.0.0-canary.0.0.0.0.0.0 < 1.0.0-canary.0.0.0.0.0.1 + #[test] + fn order_pre_numeric_segments() { + assert_eq!(order_pre("canary.0.0.0.0.0.0", "canary.0.0.0.0.0.1"), Ordering::Less); + } + + #[test] + fn order_pre_numeric_before_alpha() { + // A segment that parses as an integer sorts before one that doesn't. + assert_eq!(order_pre("1", "alpha"), Ordering::Less); + assert_eq!(order_pre("alpha", "1"), Ordering::Greater); + } + + #[test] + fn order_pre_prefix_is_less() { + assert_eq!(order_pre("alpha", "alpha.1"), Ordering::Less); + assert_eq!(order_pre("alpha.1", "alpha"), Ordering::Greater); + } + + #[test] + fn order_pre_bytewise_alpha() { + assert_eq!(order_pre("alpha", "beta"), Ordering::Less); + assert_eq!(order_pre("beta.11", "beta.2"), Ordering::Greater); + } + + fn strs(list: &[&str]) -> Vec { + list.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn negatable_empty_is_all() { + assert_eq!(parse_os(&[]).0, OperatingSystem::ALL_VALUE); + assert_eq!(parse_cpu(&[]).0, Architecture::ALL_VALUE); + } + + // bun serializes a NONE bitset back to bun.lock as the string "none". + #[test] + fn negatable_none_token() { + assert_eq!(parse_os(&strs(&["none"])).0, 0); + assert_eq!(parse_cpu(&strs(&["none"])).0, 0); + } + + #[test] + fn negatable_single_and_negated() { + assert_eq!(parse_os(&strs(&["linux"])).0, OperatingSystem::LINUX); + assert_eq!( + parse_os(&strs(&["!win32"])).0, + OperatingSystem::ALL_VALUE & !OperatingSystem::WIN32 + ); + assert_eq!( + parse_cpu(&strs(&["x64", "!arm64"])).0, + Architecture::X64 + ); + } + + // ["any", "linux"] collapses to LINUX: a recognized token resets the + // wildcard flag (bun's Negatable.apply). + #[test] + fn negatable_wildcard_reset() { + assert_eq!(parse_os(&strs(&["any", "linux"])).0, OperatingSystem::LINUX); + assert_eq!(parse_os(&strs(&["any"])).0, OperatingSystem::ALL_VALUE); + } + + #[test] + fn negatable_unrecognized_is_none() { + assert_eq!(parse_os(&strs(&["macos"])).0, 0); + } } diff --git a/programs/bun2nix-core/src/manifest/mod.rs b/programs/bun2nix-core/src/manifest/mod.rs index 38de42d..7cd9193 100644 --- a/programs/bun2nix-core/src/manifest/mod.rs +++ b/programs/bun2nix-core/src/manifest/mod.rs @@ -433,20 +433,35 @@ impl ReadManifest { self.pkg.public_max_age } - /// Look up a release version by its semver string (e.g. `"2.2.2"`). - /// - /// Returns `None` if the version is not present in `releases`. Pre-release - /// versions (tag present) are not searched. + /// Look up a version by its semver string (e.g. `"2.2.2"` or + /// `"3.0.1-alpha.1"`), mirroring bun's `find_by_version`: the map is + /// chosen by whether the version carries a pre-release tag, and equality + /// compares the numeric triple plus the wyhash of the pre tag. pub fn find_version(&self, version_str: &str) -> Option> { - let (major, minor, patch) = parse_semver_str(version_str)?; - - let keys_slice = &self.versions[self.pkg.releases.keys.off as usize - ..(self.pkg.releases.keys.off + self.pkg.releases.keys.len) as usize]; - let pvs_slice = &self.package_versions[self.pkg.releases.values.off as usize - ..(self.pkg.releases.values.off + self.pkg.releases.values.len) as usize]; + let v = build::parse_version(version_str); + let map = if v.pre.is_empty() { + self.pkg.releases + } else { + self.pkg.prereleases + }; + // Hash 0 is what bun's parser leaves in `tag.pre` for tagless versions. + let pre_hash = if v.pre.is_empty() { + 0 + } else { + wyhash11(0, v.pre.as_bytes()) + }; - for (i, v) in keys_slice.iter().enumerate() { - if v.major == major && v.minor == minor && v.patch == patch { + let keys_slice = + &self.versions[map.keys.off as usize..(map.keys.off + map.keys.len) as usize]; + let pvs_slice = &self.package_versions + [map.values.off as usize..(map.values.off + map.values.len) as usize]; + + for (i, k) in keys_slice.iter().enumerate() { + if k.major == v.major + && k.minor == v.minor + && k.patch == v.patch + && k.tag.pre.hash == pre_hash + { return Some(ReadPackageVersion { manifest: self, pv: pvs_slice[i], @@ -456,15 +471,3 @@ impl ReadManifest { None } } - -/// Parse `"major.minor.patch"` into `(major, minor, patch)`. Returns `None` -/// if the string cannot be parsed. -fn parse_semver_str(s: &str) -> Option<(u64, u64, u64)> { - let mut parts = s.splitn(3, '.'); - let major: u64 = parts.next()?.parse().ok()?; - let minor: u64 = parts.next()?.parse().ok()?; - // Strip any pre-release suffix after the patch number. - let patch_str = parts.next()?; - let patch: u64 = patch_str.split('-').next().unwrap_or("").parse().ok()?; - Some((major, minor, patch)) -} diff --git a/programs/bun2nix-core/tests/golden.rs b/programs/bun2nix-core/tests/golden.rs index d90e5e3..547a907 100644 --- a/programs/bun2nix-core/tests/golden.rs +++ b/programs/bun2nix-core/tests/golden.rs @@ -254,6 +254,124 @@ fn peer_dep_optional_ordering_round_trip() { } } +/// Minimal `VersionMeta` with just a version string and a synthetic tarball URL. +fn bare_version(version: &str) -> meta::VersionMeta { + use std::collections::BTreeMap; + meta::VersionMeta { + version: version.to_string(), + tarball_url: format!("https://registry.npmjs.org/fake-pkg/-/fake-pkg-{version}.tgz"), + integrity: String::new(), + dependencies: BTreeMap::new(), + optional_dependencies: BTreeMap::new(), + peer_dependencies: BTreeMap::new(), + optional_peers: vec![], + bin: BTreeMap::new(), + os: vec![], + cpu: vec![], + has_install_script: false, + } +} + +fn write_default_registry(built: &serialize::PackageManifest) -> Vec { + let mut out = Vec::new(); + serialize::write( + built, + manifest::DEFAULT_URL_HASH, + manifest::DEFAULT_REGISTRY_HREF_LEN, + &mut out, + ) + .unwrap(); + out +} + +/// A pre-release version must land in the `prereleases` map (bun's +/// `find_by_version` only searches there when the queried version has a tag) +/// with `tag.pre` hashed the way bun hashes it: `wyhash11(0, pre_bytes)`. +#[test] +fn prerelease_round_trip() { + use bun2nix_core::wyhash::wyhash11; + + let pkg = meta::PackageMeta { + name: "nitro".to_string(), + versions: vec![bare_version("3.0.1-alpha.1")], + }; + + let built = build::build_manifest(&pkg); + let parsed = read(&write_default_registry(&built)).expect("manifest must parse"); + + assert_eq!(parsed.pkg.releases.keys.len, 0, "no release versions expected"); + assert_eq!(parsed.pkg.prereleases.keys.len, 1, "the pre-release must be in prereleases"); + + let key = parsed.versions[parsed.pkg.prereleases.keys.off as usize]; + assert_eq!((key.major, key.minor, key.patch), (3, 0, 1)); + assert_eq!( + key.tag.pre.hash, + wyhash11(0, b"alpha.1"), + "pre tag hash must match bun's wyhash of the pre span (Tag.eql compares hashes)" + ); + assert_eq!( + resolve_str(&key.tag.pre, &parsed.string_buf), + b"alpha.1", + "pre tag text must be recoverable for version-string display" + ); + + let v = parsed + .find_version("3.0.1-alpha.1") + .expect("pre-release must be findable by its full version string"); + assert_eq!(v.tarball_url(), pkg.versions[0].tarball_url); + + // The tag-stripped triple must NOT satisfy a release lookup. + assert!( + parsed.find_version("3.0.1").is_none(), + "bare 3.0.1 must not resolve — the only version is the pre-release" + ); +} + +/// Releases and prereleases share one versions buffer: releases first +/// (ascending), then prereleases. Every version must be findable and the two +/// maps must index disjoint slices. +#[test] +fn mixed_release_prerelease_round_trip() { + let pkg = meta::PackageMeta { + name: "fake-pkg".to_string(), + versions: vec![ + bare_version("2.0.0-beta.2"), + bare_version("1.5.0"), + bare_version("2.0.0-beta.11"), + bare_version("1.0.0"), + ], + }; + + let built = build::build_manifest(&pkg); + let parsed = read(&write_default_registry(&built)).expect("manifest must parse"); + + assert_eq!(parsed.pkg.releases.keys.off, 0); + assert_eq!(parsed.pkg.releases.keys.len, 2); + assert_eq!(parsed.pkg.prereleases.keys.off, 2); + assert_eq!(parsed.pkg.prereleases.keys.len, 2); + + // Releases ascending: 1.0.0 then 1.5.0. + assert_eq!(parsed.versions[0].minor, 0); + assert_eq!(parsed.versions[1].minor, 5); + + // Prereleases ascending by bun's numeric-aware tag order: beta.2 < beta.11. + assert_eq!( + resolve_str(&parsed.versions[2].tag.pre, &parsed.string_buf), + b"beta.2" + ); + assert_eq!( + resolve_str(&parsed.versions[3].tag.pre, &parsed.string_buf), + b"beta.11" + ); + + for vm in &pkg.versions { + let v = parsed + .find_version(&vm.version) + .unwrap_or_else(|| panic!("version {} must be present", vm.version)); + assert_eq!(v.tarball_url(), vm.tarball_url, "tarball_url for {}", vm.version); + } +} + /// Tripwire: assert that the manifest cache format version string is still /// `bun-npm-manifest-cache-v0.0.7`. /// From 3489482505adf9eafd4498c8e53015b5da980e13 Mon Sep 17 00:00:00 2001 From: Alexis Williams Date: Thu, 23 Jul 2026 19:32:20 -0700 Subject: [PATCH 13/13] fix(bun2nix-core): mark synthesized manifests as extended MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With minimumReleaseAge configured (e.g. opencode's bunfig.toml), bun's manifest cache load rejects any manifest whose has_extended_manifest flag is unset — it demotes the entry to expired and schedules a network refetch, which is fatal in the sandbox. Set the flag: our zeroed publish_timestamp_ms values read as "published at epoch", which passes every age gate — the right answer for lockfile-pinned versions. --- programs/bun2nix-core/src/manifest/build.rs | 58 ++++++++++++------- programs/bun2nix-core/src/manifest/mod.rs | 35 +++++++++--- programs/bun2nix-core/tests/golden.rs | 63 ++++++++++++++++----- 3 files changed, 115 insertions(+), 41 deletions(-) diff --git a/programs/bun2nix-core/src/manifest/build.rs b/programs/bun2nix-core/src/manifest/build.rs index 74f8b59..b59f479 100644 --- a/programs/bun2nix-core/src/manifest/build.rs +++ b/programs/bun2nix-core/src/manifest/build.rs @@ -69,7 +69,9 @@ pub fn build_manifest(pkg: &PackageMeta) -> PackageManifest { // version-string tiebreak keeps output deterministic when two tags compare // equal (e.g. "1" vs "01"). releases.sort_by(|a, b| { - triple(&a.1).cmp(&triple(&b.1)).then_with(|| a.0.version.cmp(&b.0.version)) + triple(&a.1) + .cmp(&triple(&b.1)) + .then_with(|| a.0.version.cmp(&b.0.version)) }); prereleases.sort_by(|a, b| { triple(&a.1) @@ -114,6 +116,12 @@ pub fn build_manifest(pkg: &PackageMeta) -> PackageManifest { // Never expire — so bun treats this manifest as fresh indefinitely // under BUN_MANIFEST_CACHE=2 (verified in the Task 3 spike). public_max_age: u32::MAX, + // With `minimumReleaseAge` configured, bun rejects any cached manifest + // lacking extended data (publish timestamps) and refetches it — fatal + // offline. Our zeroed `publish_timestamp_ms` reads as "published at + // epoch", which passes every age gate; correct for lockfile-pinned + // versions. + has_extended_manifest: true, ..NpmPackage::default() }; @@ -170,7 +178,9 @@ fn build_one_version( arena, names, values, - vm.optional_dependencies.iter().map(|(k, v)| (k.clone(), v.clone())), + vm.optional_dependencies + .iter() + .map(|(k, v)| (k.clone(), v.clone())), ); // Peer dependencies: bun's ABI places **optional** peers at the FRONT of @@ -193,16 +203,20 @@ fn build_one_version( // `non_optional_peer_dependencies_start` = number of optional peers // (= the index at which non-optional peers begin). - let opt_count = - vm.peer_dependencies.keys().filter(|k| optional_peer_set.contains(k.as_str())).count() - as u32; + let opt_count = vm + .peer_dependencies + .keys() + .filter(|k| optional_peer_set.contains(k.as_str())) + .count() as u32; // Build the combined peer group in one pass (optional then non-optional). let peer_dependencies = build_dep_group( arena, names, values, - opt_peers.chain(non_opt_peers).map(|(k, v)| (k.clone(), v.clone())), + opt_peers + .chain(non_opt_peers) + .map(|(k, v)| (k.clone(), v.clone())), ); // Integrity: decode SRI string → raw tag + digest bytes. @@ -278,7 +292,11 @@ fn decode_base64(input: &str) -> Option> { while i < bytes.len() { let remaining = bytes.len() - i; let a = b64_val(bytes[i])?; - let b = if remaining > 1 { b64_val(bytes[i + 1])? } else { return None }; + let b = if remaining > 1 { + b64_val(bytes[i + 1])? + } else { + return None; + }; out.push((a << 2) | (b >> 4)); if remaining > 2 { let c = b64_val(bytes[i + 2])?; @@ -444,12 +462,7 @@ fn build_bin( tag: 2, // NamedFile _padding_tag: [0; 3], value: BinValue { - raw: [ - ss_lo(&k_ss), - ss_hi(&k_ss), - ss_lo(&v_ss), - ss_hi(&v_ss), - ], + raw: [ss_lo(&k_ss), ss_hi(&k_ss), ss_lo(&v_ss), ss_hi(&v_ss)], }, } } @@ -464,7 +477,9 @@ fn build_bin( Bin { tag: 4, // Map _padding_tag: [0; 3], - value: BinValue { raw: [off, count, 0, 0] }, + value: BinValue { + raw: [off, count, 0, 0], + }, } } } @@ -505,7 +520,10 @@ pub(crate) fn parse_version(s: &str) -> ParsedVersion<'_> { let major: u64 = parts.next().and_then(|p| p.parse().ok()).unwrap_or(0); let minor: u64 = parts.next().and_then(|p| p.parse().ok()).unwrap_or(0); let rest = parts.next().unwrap_or(""); - let digits_end = rest.bytes().position(|b| !b.is_ascii_digit()).unwrap_or(rest.len()); + let digits_end = rest + .bytes() + .position(|b| !b.is_ascii_digit()) + .unwrap_or(rest.len()); let patch: u64 = rest[..digits_end].parse().unwrap_or(0); let (pre, build) = parse_tag(&rest[digits_end..]); ParsedVersion { @@ -668,7 +686,10 @@ mod tests { // 1.0.0-canary.0.0.0.0.0.0 < 1.0.0-canary.0.0.0.0.0.1 #[test] fn order_pre_numeric_segments() { - assert_eq!(order_pre("canary.0.0.0.0.0.0", "canary.0.0.0.0.0.1"), Ordering::Less); + assert_eq!( + order_pre("canary.0.0.0.0.0.0", "canary.0.0.0.0.0.1"), + Ordering::Less + ); } #[test] @@ -714,10 +735,7 @@ mod tests { parse_os(&strs(&["!win32"])).0, OperatingSystem::ALL_VALUE & !OperatingSystem::WIN32 ); - assert_eq!( - parse_cpu(&strs(&["x64", "!arm64"])).0, - Architecture::X64 - ); + assert_eq!(parse_cpu(&strs(&["x64", "!arm64"])).0, Architecture::X64); } // ["any", "linux"] collapses to LINUX: a recognized token resets the diff --git a/programs/bun2nix-core/src/manifest/mod.rs b/programs/bun2nix-core/src/manifest/mod.rs index 7cd9193..6560771 100644 --- a/programs/bun2nix-core/src/manifest/mod.rs +++ b/programs/bun2nix-core/src/manifest/mod.rs @@ -168,19 +168,28 @@ pub fn build_single_version(input: &SingleVersionInput) -> PackageManifest { &mut arena, &mut names, &mut values, - input.dependencies.iter().map(|d| (d.name.clone(), d.range.clone())), + input + .dependencies + .iter() + .map(|d| (d.name.clone(), d.range.clone())), ); let optional_dependencies = build_dep_group( &mut arena, &mut names, &mut values, - input.optional_dependencies.iter().map(|d| (d.name.clone(), d.range.clone())), + input + .optional_dependencies + .iter() + .map(|d| (d.name.clone(), d.range.clone())), ); let peer_dependencies = build_dep_group( &mut arena, &mut names, &mut values, - input.peer_dependencies.iter().map(|d| (d.name.clone(), d.range.clone())), + input + .peer_dependencies + .iter() + .map(|d| (d.name.clone(), d.range.clone())), ); let integrity = match input.sha512 { @@ -219,6 +228,10 @@ pub fn build_single_version(input: &SingleVersionInput) -> PackageManifest { // the network: `by_name_hash` only returns a non-expired manifest when // `public_max_age > timestamp_for_manifest_cache_control` (current time). public_max_age: u32::MAX, + // Required when `minimumReleaseAge` is configured — bun refetches any + // cached manifest without extended data. Zeroed publish timestamps + // pass every age gate. + has_extended_manifest: true, ..NpmPackage::default() }; // releases: keys index the `versions` buffer, values index `package_versions`. @@ -298,7 +311,12 @@ impl<'a> Reader<'a> { for i in 0..n { // SAFETY: region is `byte_len` long, n elements of size_of::. let v = unsafe { - std::ptr::read_unaligned(region.as_ptr().add(i * std::mem::size_of::()).cast::()) + std::ptr::read_unaligned( + region + .as_ptr() + .add(i * std::mem::size_of::()) + .cast::(), + ) }; out.push(v); } @@ -310,7 +328,8 @@ impl<'a> Reader<'a> { /// Deserialize a `.npm` byte buffer back into a [`ReadManifest`]. Returns `None` /// if the header does not match. pub fn read(bytes: &[u8]) -> Option { - if bytes.len() < serialize::HEADER.len() || &bytes[..serialize::HEADER.len()] != serialize::HEADER + if bytes.len() < serialize::HEADER.len() + || &bytes[..serialize::HEADER.len()] != serialize::HEADER { return None; } @@ -407,8 +426,10 @@ impl<'a> ReadPackageVersion<'a> { let n_off = map.name.off as usize; let v_off = map.value.off as usize; for i in 0..map.name.len as usize { - let name_bytes = - resolve_str(&self.manifest.external_strings[n_off + i], &self.manifest.string_buf); + let name_bytes = resolve_str( + &self.manifest.external_strings[n_off + i], + &self.manifest.string_buf, + ); let val_bytes = resolve_str( &self.manifest.external_strings_for_versions[v_off + i], &self.manifest.string_buf, diff --git a/programs/bun2nix-core/tests/golden.rs b/programs/bun2nix-core/tests/golden.rs index 547a907..475ad60 100644 --- a/programs/bun2nix-core/tests/golden.rs +++ b/programs/bun2nix-core/tests/golden.rs @@ -94,9 +94,11 @@ fn neoconfetti_round_trip() { /// Construct a `PackageMeta` for @neoconfetti/svelte 2.2.2 from the fixture. fn neoconfetti_meta() -> meta::PackageMeta { - let fixture = - std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/neoconfetti.json")) - .expect("neoconfetti.json fixture missing"); + let fixture = std::fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/neoconfetti.json" + )) + .expect("neoconfetti.json fixture missing"); serde_json::from_str(&fixture).expect("failed to parse neoconfetti.json") } @@ -130,14 +132,20 @@ fn builder_round_trips() { pkg.versions[0].tarball_url, "tarball_url must survive round-trip" ); - assert_eq!(parsed.public_max_age(), u32::MAX, "manifest must never expire"); + assert_eq!( + parsed.public_max_age(), + u32::MAX, + "manifest must never expire" + ); } /// Construct a `PackageMeta` for `ms` with two versions from the fixture. fn ms_meta() -> meta::PackageMeta { - let fixture = - std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/ms.json")) - .expect("ms.json fixture missing"); + let fixture = std::fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/ms.json" + )) + .expect("ms.json fixture missing"); serde_json::from_str(&fixture).expect("failed to parse ms.json") } @@ -159,7 +167,15 @@ fn builder_multi_version() { let parsed = read(&out).expect("header must parse"); assert_eq!(parsed.name(), b"ms"); - assert_eq!(parsed.public_max_age(), u32::MAX, "manifest must never expire"); + assert_eq!( + parsed.public_max_age(), + u32::MAX, + "manifest must never expire" + ); + assert!( + parsed.pkg.has_extended_manifest, + "manifests must claim extended data or bun refetches them when minimumReleaseAge is set" + ); // Both versions must be findable and their tarball URLs must survive. for vm in &pkg.versions { @@ -222,7 +238,9 @@ fn peer_dep_optional_ordering_round_trip() { .unwrap(); let parsed = read(&out).expect("manifest must parse"); - let v = parsed.find_version("1.0.0").expect("version 1.0.0 must be present"); + let v = parsed + .find_version("1.0.0") + .expect("version 1.0.0 must be present"); let total = v.pv.peer_dependencies.name.len as usize; let start = v.pv.non_optional_peer_dependencies_start as usize; @@ -299,8 +317,14 @@ fn prerelease_round_trip() { let built = build::build_manifest(&pkg); let parsed = read(&write_default_registry(&built)).expect("manifest must parse"); - assert_eq!(parsed.pkg.releases.keys.len, 0, "no release versions expected"); - assert_eq!(parsed.pkg.prereleases.keys.len, 1, "the pre-release must be in prereleases"); + assert_eq!( + parsed.pkg.releases.keys.len, 0, + "no release versions expected" + ); + assert_eq!( + parsed.pkg.prereleases.keys.len, 1, + "the pre-release must be in prereleases" + ); let key = parsed.versions[parsed.pkg.prereleases.keys.off as usize]; assert_eq!((key.major, key.minor, key.patch), (3, 0, 1)); @@ -368,7 +392,12 @@ fn mixed_release_prerelease_round_trip() { let v = parsed .find_version(&vm.version) .unwrap_or_else(|| panic!("version {} must be present", vm.version)); - assert_eq!(v.tarball_url(), vm.tarball_url, "tarball_url for {}", vm.version); + assert_eq!( + v.tarball_url(), + vm.tarball_url, + "tarball_url for {}", + vm.version + ); } } @@ -418,13 +447,19 @@ fn npmmirror_url_hash_and_filename() { let href = "https://registry.npmmirror.com"; assert_eq!(manifest::url_hash(href), 0x02200d3777602379); // Trailing slash is stripped before hashing. - assert_eq!(manifest::url_hash("https://registry.npmmirror.com/"), 0x02200d3777602379); + assert_eq!( + manifest::url_hash("https://registry.npmmirror.com/"), + 0x02200d3777602379 + ); assert_eq!(manifest::registry_href_len(href), 30); assert_eq!( manifest::manifest_file_name("react", Some(href)), "94c49019ded8e790-02200d3777602379.npm" ); - assert_eq!(manifest::manifest_file_name("react", None), "94c49019ded8e790.npm"); + assert_eq!( + manifest::manifest_file_name("react", None), + "94c49019ded8e790.npm" + ); } /// The generalized hash must reproduce bun's DEFAULT_URL_HASH for the