Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions cli/factory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ use crate::npm::CliNpmInstaller;
use crate::npm::CliNpmInstallerFactory;
use crate::npm::CliNpmResolver;
use crate::npm::DenoTaskLifeCycleScriptsExecutor;
use crate::npm::NpmPackumentFormat;
use crate::resolver::CliCjsTracker;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::CliResolver;
Expand Down Expand Up @@ -579,11 +580,21 @@ impl CliFactory {
self.services.npm_installer_factory.get_or_try_init(|| {
let cli_options = self.cli_options()?;
let resolver_factory = self.resolver_factory()?;
let needs_full_packument = resolver_factory
.minimum_dependency_age_config()
.ok()
.and_then(|c| c.age.as_ref().and_then(|d| d.into_option()))
.is_some();
Ok(CliNpmInstallerFactory::new(
resolver_factory.clone(),
Arc::new(CliNpmCacheHttpClient::new(
self.http_client_provider().clone(),
self.text_only_progress_bar().clone(),
if needs_full_packument {
NpmPackumentFormat::Full
} else {
NpmPackumentFormat::Abbreviated
},
)),
match resolver_factory.npm_resolver()?.as_managed() {
Some(managed_npm_resolver) => {
Expand Down
2 changes: 2 additions & 0 deletions cli/lsp/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ use crate::file_fetcher::CliFileFetcher;
use crate::http_util::HttpClientProvider;
use crate::lsp::logging::lsp_warn;
use crate::npm::CliNpmCacheHttpClient;
use crate::npm::NpmPackumentFormat;
use crate::sys::CliSys;
use crate::util::fs::canonicalize_path;
use crate::util::fs::canonicalize_path_maybe_not_exists;
Expand Down Expand Up @@ -1483,6 +1484,7 @@ impl ConfigData {
// will only happen in the tests
.unwrap_or_else(|| Arc::new(HttpClientProvider::new(None, None))),
pb.clone(),
NpmPackumentFormat::Abbreviated,
)),
Arc::new(NullLifecycleScriptsExecutor),
pb,
Expand Down
3 changes: 3 additions & 0 deletions cli/lsp/resolver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ use crate::npm::CliNpmInstaller;
use crate::npm::CliNpmRegistryInfoProvider;
use crate::npm::CliNpmResolver;
use crate::npm::CliNpmResolverCreateOptions;
use crate::npm::NpmPackumentFormat;
use crate::resolver::CliIsCjsResolver;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::CliResolver;
Expand Down Expand Up @@ -906,11 +907,13 @@ impl<'a> ResolverFactory<'a> {
let npm_client = Arc::new(CliNpmCacheHttpClient::new(
http_client_provider.clone(),
pb.clone(),
NpmPackumentFormat::Abbreviated,
));
let registry_info_provider = Arc::new(CliNpmRegistryInfoProvider::new(
npm_cache.clone(),
npm_client.clone(),
npmrc.clone(),
NpmPackumentFormat::Abbreviated,
));
let link_packages: WorkspaceNpmLinkPackagesRc = self
.config_data
Expand Down
21 changes: 21 additions & 0 deletions cli/npm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,20 +64,25 @@ pub type CliNpmGraphResolver = deno_npm_installer::graph::NpmDenoGraphResolver<
CliSys,
>;

pub use deno_npm_cache::NpmPackumentFormat;

#[derive(Debug)]
pub struct CliNpmCacheHttpClient {
http_client_provider: Arc<HttpClientProvider>,
progress_bar: ProgressBar,
packument_format: NpmPackumentFormat,
}

impl CliNpmCacheHttpClient {
pub fn new(
http_client_provider: Arc<HttpClientProvider>,
progress_bar: ProgressBar,
packument_format: NpmPackumentFormat,
) -> Self {
Self {
http_client_provider,
progress_bar,
packument_format,
}
}
}
Expand Down Expand Up @@ -110,6 +115,22 @@ impl deno_npm_cache::NpmCacheHttpClient for CliNpmCacheHttpClient {
http::header::HeaderValue::try_from(etag).unwrap(),
);
}
if self.packument_format == NpmPackumentFormat::Abbreviated {
// Request the abbreviated install manifest when possible. This is 2-5x
// smaller than the full packument (e.g. @types/node: 2.3 MB vs 10.9 MB).
// Uses content negotiation with quality factors for registry compatibility
// (some registries like older Artifactory don't support the abbreviated
// format and need the JSON fallback).
//
// Not used when minimumDependencyAge is configured, because the
// abbreviated format omits the `time` field needed for date filtering.
headers.insert(
http::header::ACCEPT,
http::header::HeaderValue::from_static(
"application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*",
),
);
}
client
.download_with_progress_and_retries(url, &headers, &guard)
.await
Expand Down
6 changes: 6 additions & 0 deletions libs/npm/registry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,11 @@ pub struct NpmPackageVersionInfo {
#[serde(default, skip_serializing_if = "HashMap::is_empty")]
#[serde(deserialize_with = "deserializers::hashmap")]
pub scripts: HashMap<SmallStackString, String>,
/// From the abbreviated install manifest format. When `true`, this version
/// has preinstall/install/postinstall lifecycle scripts. This field is used
/// when the full `scripts` map is not available (abbreviated packument).
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub has_install_script: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[serde(deserialize_with = "deserializers::string")]
pub deprecated: Option<String>,
Expand Down Expand Up @@ -1498,6 +1503,7 @@ mod test {
os: Default::default(),
cpu: Default::default(),
scripts: Default::default(),
has_install_script: Default::default(),
deprecated: Default::default(),
};
let text = serde_json::to_string(&data).unwrap();
Expand Down
3 changes: 2 additions & 1 deletion libs/npm/resolution/graph.rs
Original file line number Diff line number Diff line change
Expand Up @@ -790,7 +790,8 @@ impl Graph {
}),
is_deprecated: version_info.deprecated.is_some(),
has_bin: version_info.bin.is_some(),
has_scripts: version_info.scripts.contains_key("preinstall")
has_scripts: version_info.has_install_script
|| version_info.scripts.contains_key("preinstall")
|| version_info.scripts.contains_key("install")
|| version_info.scripts.contains_key("postinstall"),
optional_peer_dependencies: version_info
Expand Down
8 changes: 8 additions & 0 deletions libs/npm_cache/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,14 @@ impl std::fmt::Display for DownloadError {
}
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum NpmPackumentFormat {
/// Request the abbreviated install manifest (smaller, but omits `time` and `scripts`).
Abbreviated,
/// Request the full packument (needed when `minimumDependencyAge` is configured).
Full,
}

pub enum NpmCacheHttpClientResponse {
NotFound,
NotModified,
Expand Down
14 changes: 13 additions & 1 deletion libs/npm_cache/registry_info.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ use crate::NpmCacheHttpClient;
use crate::NpmCacheHttpClientResponse;
use crate::NpmCacheSetting;
use crate::NpmCacheSys;
use crate::NpmPackumentFormat;
use crate::remote::maybe_auth_header_value_for_npm_registry;
use crate::rt::MultiRuntimeAsyncValueCreator;
use crate::rt::spawn_blocking;
Expand Down Expand Up @@ -140,6 +141,7 @@ struct RegistryInfoProviderInner<
cache: Arc<NpmCache<TSys>>,
http_client: Arc<THttpClient>,
npmrc: Arc<ResolvedNpmRc>,
packument_format: NpmPackumentFormat,
force_reload_flag: AtomicFlag,
memory_cache: Mutex<MemoryCache>,
previously_loaded_packages: Mutex<HashSet<String>>,
Expand Down Expand Up @@ -251,7 +253,15 @@ impl<THttpClient: NpmCacheHttpClient, TSys: NpmCacheSys>
{
// attempt to load from the file cache
match downloader.cache.load_package_info(&name).await.map_err(JsErrorBox::from_err)? { Some(cached_info) => {
return Ok(FutureResult::SavedFsCache(Arc::new(cached_info.info)));
if downloader.packument_format == NpmPackumentFormat::Full && cached_info.info.time.is_empty() && !cached_info.info.versions.is_empty() {
// Cached data is from the abbreviated install manifest which
// doesn't include the `time` field. Since minimumDependencyAge
// is configured, we need to re-fetch the full packument.
// Don't use the etag since it corresponds to the abbreviated format.
Some(SerializedCachedPackageInfo { etag: None, ..cached_info })
} else {
return Ok(FutureResult::SavedFsCache(Arc::new(cached_info.info)));
}
} _ => {
None
}}
Expand Down Expand Up @@ -353,11 +363,13 @@ impl<THttpClient: NpmCacheHttpClient, TSys: NpmCacheSys>
cache: Arc<NpmCache<TSys>>,
http_client: Arc<THttpClient>,
npmrc: Arc<ResolvedNpmRc>,
packument_format: NpmPackumentFormat,
) -> Self {
Self(Arc::new(RegistryInfoProviderInner {
cache,
http_client,
npmrc,
packument_format,
force_reload_flag: AtomicFlag::lowered(),
memory_cache: Default::default(),
previously_loaded_packages: Default::default(),
Expand Down
23 changes: 19 additions & 4 deletions libs/npm_installer/extra_info.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// Copyright 2018-2026 the Deno authors. MIT license.

use std::collections::HashMap;
use std::path::Path;
use std::sync::Arc;

Expand Down Expand Up @@ -106,14 +107,28 @@ impl NpmPackageExtraInfoProvider {
self.fetch_from_registry(package_nv).await
} else {
match self.fetch_from_package_json(package_path).await {
Ok(extra_info) => {
Ok(mut extra_info) => {
// some packages that use "directories.bin" have a "bin" entry in
// the packument, but not in package.json (e.g. esbuild-wasm)
if (expected.bin && extra_info.bin.is_none())
|| (expected.scripts && extra_info.scripts.is_empty())
{
if expected.bin && extra_info.bin.is_none() {
self.fetch_from_registry(package_nv).await
} else {
// When a package has a binding.gyp and no install/preinstall script,
// npm injects `"install": "node-gyp rebuild"` at publish time. This
// script appears in the packument but not in the tarball's package.json.
// Match pnpm's behavior and detect this case by checking for the file.
if expected.scripts
&& extra_info.scripts.is_empty()
&& self
.sys
.base_fs_read(&package_path.join("binding.gyp"))
.is_ok()
{
extra_info.scripts = HashMap::from([(
"install".into(),
"node-gyp rebuild".to_string(),
)]);
}
Ok(extra_info)
}
}
Expand Down
13 changes: 13 additions & 0 deletions libs/npm_installer/factory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
use deno_npm_cache::NpmCache;
use deno_npm_cache::NpmCacheHttpClient;
use deno_npm_cache::NpmCacheSetting;
use deno_npm_cache::NpmPackumentFormat;
use deno_npm_cache::RegistryInfoProvider;
use deno_npm_cache::TarballCache;
use deno_resolver::factory::ResolverFactory;
Expand Down Expand Up @@ -393,10 +394,22 @@ impl<
anyhow::Error,
> {
self.registry_info_provider.get_or_try_init(|| {
let packument_format = if self
.resolver_factory
.minimum_dependency_age_config()
.ok()
.and_then(|c| c.age.as_ref().and_then(|d| d.into_option()))
.is_some()
{
NpmPackumentFormat::Full
} else {
NpmPackumentFormat::Abbreviated
};
Ok(Arc::new(RegistryInfoProvider::new(
self.npm_cache()?.clone(),
self.http_client().clone(),
self.workspace_factory().npmrc()?.clone(),
packument_format,
)))
})
}
Expand Down
45 changes: 31 additions & 14 deletions libs/npm_installer/local.rs
Original file line number Diff line number Diff line change
Expand Up @@ -376,32 +376,49 @@ impl<
Ok::<_, SyncResolutionWithFsError>(())
}
});
let extra_fut = if (package.has_bin
let needs_extra_from_disk = package.extra.is_none()
// When using abbreviated packument format, has_scripts may
// be true (from hasInstallScript) while extra.scripts is
// empty. In that case, read from disk to get real scripts.
|| (package.has_scripts
&& package
.extra
.as_ref()
.is_some_and(|e| e.scripts.is_empty()));
let extra = if (package.has_bin
|| package.has_scripts
|| package.is_deprecated)
&& package.extra.is_none()
&& needs_extra_from_disk
{
// Wait for extraction to complete first, since
// get_package_extra_info may read from the on-disk
// package.json which doesn't exist until extraction finishes.
handle
.await
.map_err(JsErrorBox::from_err)?
.map_err(JsErrorBox::from_err)?;
extra_info_provider
.get_package_extra_info(
&package.id.nv,
&package_path,
ExpectedExtraInfo::from_package(package),
)
.boxed_local()
.await
.map_err(JsErrorBox::from_err)?
} else {
std::future::ready(Ok(
package.extra.clone().unwrap_or_default(),
))
.boxed_local()
let (result, extra) = futures::future::join(
handle,
std::future::ready(Ok::<_, JsErrorBox>(
package.extra.clone().unwrap_or_default(),
)),
)
.await;
result
.map_err(JsErrorBox::from_err)?
.map_err(JsErrorBox::from_err)?;
extra?
};

let (result, extra) =
futures::future::join(handle, extra_fut).await;
result
.map_err(JsErrorBox::from_err)?
.map_err(JsErrorBox::from_err)?;
let extra = extra.map_err(JsErrorBox::from_err)?;

if package.has_bin {
bin_entries_to_setup.borrow_mut().add(
package,
Expand Down
2 changes: 2 additions & 0 deletions libs/resolver/workspace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1740,6 +1740,7 @@ fn pkg_json_to_version_info(
.collect()
})
.unwrap_or_default(),
has_install_script: false,
// not worth increasing memory for showing a deprecated
// message for linked packages
deprecated: None,
Expand Down Expand Up @@ -3233,6 +3234,7 @@ mod test {
NpmPackageVersionInfo {
version: Version::parse_from_npm("1.0.0").unwrap(),
dist: None,
has_install_script: false,
bin: Some(deno_npm::registry::NpmPackageVersionBinEntry::String(
"./bin.js".to_string()
)),
Expand Down
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,14 @@
"tarball": "http://localhost:4260/denotest-packagejson-missing-info/denotest-packagejson-missing-info-1.0.0-missingbin.tgz"
}
},
"0.5.0-missingscripts": {
"0.7.0-bindinggyp": {
"name": "denotest-packagejson-missing-info",
"version": "0.5.0-missingscripts",
"version": "0.7.0-bindinggyp",
"scripts": {
"postinstall": "echo 'postinstall'"
"install": "node-gyp rebuild"
},
"dist": {
"tarball": "http://localhost:4260/denotest-packagejson-missing-info/denotest-packagejson-missing-info-0.5.0-missingscripts.tgz"
"tarball": "http://localhost:4260/denotest-packagejson-missing-info/denotest-packagejson-missing-info-0.7.0-bindinggyp.tgz"
}
},
"0.2.5-missingdeprecated": {
Expand Down
10 changes: 3 additions & 7 deletions tests/specs/install/packagejson_missing_extra/__test__.jsonc
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,11 @@
}
]
},
"missing_scripts": {
"binding_gyp_scripts": {
"steps": [
{
"args": "install npm:denotest-packagejson-missing-info@0.5.0-missingscripts",
"output": "missingscripts.out"
},
{
"args": "install --allow-scripts",
"output": "[WILDCARD]running 'postinstall' script\n[WILDCARD]"
"args": "install npm:denotest-packagejson-missing-info@0.7.0-bindinggyp",
"output": "bindinggyp.out"
}
]
},
Expand Down
Loading
Loading